From 306c5ae742d781fd22c6e894d36da498da7dc9dc Mon Sep 17 00:00:00 2001 From: busya Date: Tue, 12 May 2026 23:54:55 +0300 Subject: [PATCH] semantics: complete DEF-to-region migration, fix regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Convert legacy [DEF:id:Type] anchors to #region/#endregion across 329 files - Reinstate _normalize_timestamp_value in sql_generator.py - Fix MarkerLogger→logger migration in events.py (molecular CoT markers) - Fix dataset_review orchestrator dependencies (_build_execution_snapshot) - Fix config_manager stale-record deletion (moved to save path only) - Add 77 missing [/DEF:] closers in 5 unbalanced test files - Update assistant_chat.integration.test.js for #region format - Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods --- .opencode/agents/qa-tester.md | 2 +- backend/_convert_defs.py | 593 ++++++------------ backend/src/api/auth.py | 41 +- backend/src/api/routes/__init__.py | 8 +- backend/src/api/routes/__tests__/conftest.py | 7 +- .../routes/__tests__/test_assistant_api.py | 163 ++--- .../routes/__tests__/test_assistant_authz.py | 176 +++--- .../__tests__/test_clean_release_api.py | 52 +- .../test_clean_release_legacy_compat.py | 39 +- .../test_clean_release_source_policy.py | 30 +- .../__tests__/test_clean_release_v2_api.py | 35 +- .../test_clean_release_v2_release_api.py | 35 +- .../__tests__/test_connections_routes.py | 27 +- .../api/routes/__tests__/test_dashboards.py | 205 +++--- .../__tests__/test_dataset_review_api.py | 242 +++---- .../src/api/routes/__tests__/test_datasets.py | 90 +-- .../src/api/routes/__tests__/test_git_api.py | 122 ++-- .../routes/__tests__/test_git_status_route.py | 180 +++--- .../routes/__tests__/test_migration_routes.py | 29 +- .../api/routes/__tests__/test_profile_api.py | 120 ++-- .../api/routes/__tests__/test_reports_api.py | 73 +-- .../__tests__/test_reports_detail_api.py | 57 +- .../test_reports_openapi_conformance.py | 57 +- .../api/routes/__tests__/test_tasks_logs.py | 44 +- backend/src/api/routes/admin.py | 109 ++-- backend/src/api/routes/assistant/__init__.py | 4 +- .../src/api/routes/assistant/_admin_routes.py | 23 +- .../api/routes/assistant/_command_parser.py | 14 +- .../api/routes/assistant/_dataset_review.py | 18 +- .../assistant/_dataset_review_dispatch.py | 12 +- backend/src/api/routes/assistant/_dispatch.py | 24 +- backend/src/api/routes/assistant/_history.py | 70 +-- .../src/api/routes/assistant/_llm_planner.py | 22 +- .../routes/assistant/_llm_planner_intent.py | 12 +- .../src/api/routes/assistant/_resolvers.py | 44 +- backend/src/api/routes/assistant/_routes.py | 25 +- backend/src/api/routes/assistant/_schemas.py | 44 +- backend/src/api/routes/clean_release.py | 42 +- backend/src/api/routes/clean_release_v2.py | 22 +- backend/src/api/routes/connections.py | 46 +- backend/src/api/routes/dashboards/__init__.py | 14 +- .../api/routes/dashboards/_action_routes.py | 38 +- .../api/routes/dashboards/_detail_routes.py | 27 +- backend/src/api/routes/dashboards/_helpers.py | 26 +- .../api/routes/dashboards/_listing_routes.py | 55 +- .../src/api/routes/dashboards/_projection.py | 4 +- backend/src/api/routes/dashboards/_schemas.py | 2 +- backend/src/api/routes/dataset_review.py | 6 +- .../dataset_review_pkg/_dependencies.py | 11 +- .../api/routes/dataset_review_pkg/_routes.py | 26 +- backend/src/api/routes/datasets.py | 77 +-- backend/src/api/routes/environments.py | 35 +- backend/src/api/routes/git/__init__.py | 2 +- backend/src/api/routes/git/_config_routes.py | 2 +- backend/src/api/routes/git/_deps.py | 4 +- .../src/api/routes/git/_environment_routes.py | 2 +- backend/src/api/routes/git/_gitea_routes.py | 2 +- backend/src/api/routes/git/_helpers.py | 32 +- backend/src/api/routes/git/_merge_routes.py | 2 +- .../api/routes/git/_repo_lifecycle_routes.py | 2 +- .../api/routes/git/_repo_operations_routes.py | 41 +- backend/src/api/routes/git/_repo_routes.py | 18 +- backend/src/api/routes/git/_router.py | 2 +- backend/src/api/routes/git_schemas.py | 8 +- backend/src/api/routes/health.py | 16 +- backend/src/api/routes/llm.py | 201 +----- backend/src/api/routes/mappings.py | 20 +- backend/src/api/routes/migration.py | 81 +-- backend/src/api/routes/plugins.py | 7 +- backend/src/api/routes/profile.py | 51 +- backend/src/api/routes/reports.py | 34 +- backend/src/api/routes/settings.py | 144 ++--- backend/src/api/routes/storage.py | 59 +- backend/src/api/routes/tasks.py | 67 +- backend/src/api/routes/translate/__init__.py | 11 +- .../routes/translate/_correction_routes.py | 39 +- .../routes/translate/_dictionary_routes.py | 145 ++--- backend/src/api/routes/translate/_helpers.py | 13 +- .../src/api/routes/translate/_job_routes.py | 112 ++-- .../api/routes/translate/_metrics_routes.py | 30 +- .../api/routes/translate/_preview_routes.py | 73 +-- backend/src/api/routes/translate/_router.py | 8 +- .../api/routes/translate/_run_list_routes.py | 47 +- .../src/api/routes/translate/_run_routes.py | 127 ++-- .../api/routes/translate/_schedule_routes.py | 83 +-- backend/src/app.py | 95 ++- .../__tests__/test_config_manager_compat.py | 154 ++++- .../src/core/__tests__/test_native_filters.py | 190 +++--- .../test_superset_preview_pipeline.py | 110 ++-- .../__tests__/test_superset_profile_lookup.py | 79 +-- .../__tests__/test_throttled_scheduler.py | 57 +- backend/src/core/async_superset_client.py | 151 ++--- backend/src/core/auth/__tests__/test_auth.py | 93 +-- backend/src/core/auth/config.py | 18 +- backend/src/core/auth/jwt.py | 22 +- backend/src/core/auth/logger.py | 21 +- backend/src/core/auth/oauth.py | 31 +- backend/src/core/auth/repository.py | 108 ++-- backend/src/core/auth/security.py | 27 +- backend/src/core/config_manager.py | 330 +++++----- backend/src/core/config_models.py | 4 +- backend/src/core/cot_logger.py | 4 +- backend/src/core/database.py | 122 +--- backend/src/core/encryption_key.py | 18 +- backend/src/core/logger.py | 418 ++++++------ .../src/core/logger/__tests__/test_logger.py | 336 ++++------ backend/src/core/mapping_service.py | 97 +-- backend/src/core/middleware/trace.py | 5 +- backend/src/core/migration/archive_parser.py | 40 +- .../core/migration/dry_run_orchestrator.py | 74 +-- backend/src/core/migration/risk_assessor.py | 81 ++- backend/src/core/migration_engine.py | 74 ++- backend/src/core/plugin_base.py | 98 +-- backend/src/core/plugin_loader.py | 74 +-- backend/src/core/scheduler.py | 132 ++-- backend/src/core/superset_client/__init__.py | 4 +- backend/src/core/superset_client/_base.py | 164 +++-- backend/src/core/superset_client/_charts.py | 28 +- .../core/superset_client/_dashboards_crud.py | 118 ++-- .../superset_client/_dashboards_filters.py | 84 ++- .../core/superset_client/_dashboards_list.py | 80 ++- .../src/core/superset_client/_databases.py | 54 +- backend/src/core/superset_client/_datasets.py | 82 +-- .../core/superset_client/_datasets_preview.py | 58 +- .../_datasets_preview_filters.py | 25 +- .../core/superset_client/_user_projection.py | 24 +- backend/src/core/superset_profile_lookup.py | 56 +- .../task_manager/__tests__/test_context.py | 22 +- .../__tests__/test_task_logger.py | 56 +- backend/src/core/task_manager/cleanup.py | 34 +- backend/src/core/task_manager/context.py | 101 ++- backend/src/core/task_manager/manager.py | 418 ++++++------ backend/src/core/task_manager/models.py | 16 +- backend/src/core/task_manager/persistence.py | 261 ++++---- backend/src/core/task_manager/task_logger.py | 88 ++- backend/src/core/utils/async_network.py | 200 +++--- backend/src/core/utils/dataset_mapper.py | 99 ++- backend/src/core/utils/fileio.py | 208 +++--- backend/src/core/utils/matching.py | 18 +- backend/src/core/utils/network.py | 309 ++++----- .../utils/superset_compilation_adapter.py | 127 ++-- .../superset_context_extractor/__init__.py | 18 +- .../utils/superset_context_extractor/_base.py | 81 +-- .../superset_context_extractor/_filters.py | 9 +- .../superset_context_extractor/_parsing.py | 28 +- .../utils/superset_context_extractor/_pii.py | 2 +- .../superset_context_extractor/_recovery.py | 26 +- .../superset_context_extractor/_templates.py | 47 +- backend/src/dependencies.py | 72 +-- backend/src/models/__init__.py | 3 +- .../models/__tests__/test_clean_release.py | 72 +-- backend/src/models/__tests__/test_models.py | 23 +- .../models/__tests__/test_report_models.py | 11 +- backend/src/models/assistant.py | 24 +- backend/src/models/auth.py | 8 +- backend/src/models/clean_release.py | 24 +- backend/src/models/config.py | 24 +- backend/src/models/connection.py | 12 +- backend/src/models/dashboard.py | 6 +- backend/src/models/dataset_review.py | 8 +- .../src/models/dataset_review_pkg/__init__.py | 2 +- .../_clarification_models.py | 4 +- .../src/models/dataset_review_pkg/_enums.py | 4 +- .../dataset_review_pkg/_execution_models.py | 2 +- .../dataset_review_pkg/_filter_models.py | 2 +- .../dataset_review_pkg/_finding_models.py | 2 +- .../dataset_review_pkg/_mapping_models.py | 4 +- .../dataset_review_pkg/_profile_models.py | 2 +- .../dataset_review_pkg/_semantic_models.py | 6 +- .../dataset_review_pkg/_session_models.py | 6 +- backend/src/models/filter_state.py | 20 +- backend/src/models/llm.py | 6 +- backend/src/models/mapping.py | 15 +- backend/src/models/profile.py | 12 +- backend/src/models/report.py | 42 +- backend/src/models/storage.py | 2 +- backend/src/models/task.py | 16 +- backend/src/models/translate.py | 48 +- backend/src/plugins/backup.py | 84 +-- backend/src/plugins/debug.py | 102 ++- backend/src/plugins/git/llm_extension.py | 14 +- backend/src/plugins/git_plugin.py | 137 ++-- backend/src/plugins/llm_analysis/__init__.py | 4 +- .../__tests__/test_client_headers.py | 22 +- .../__tests__/test_screenshot_service.py | 94 +-- .../llm_analysis/__tests__/test_service.py | 34 +- backend/src/plugins/llm_analysis/models.py | 24 +- backend/src/plugins/llm_analysis/plugin.py | 36 +- backend/src/plugins/llm_analysis/scheduler.py | 13 +- backend/src/plugins/llm_analysis/service.py | 190 +++--- backend/src/plugins/mapper.py | 82 ++- backend/src/plugins/migration.py | 167 +++-- backend/src/plugins/search.py | 88 ++- backend/src/plugins/storage/plugin.py | 188 +++--- backend/src/plugins/translate/__init__.py | 3 +- .../plugins/translate/__tests__/__init__.py | 6 +- .../test_clickhouse_insert_integration.py | 33 +- .../translate/__tests__/test_dictionary.py | 173 ++--- .../translate/__tests__/test_orchestrator.py | 58 +- .../translate/__tests__/test_preview.py | 114 ++-- .../translate/__tests__/test_sql_generator.py | 140 ++--- backend/src/plugins/translate/_utils.py | 17 +- backend/src/plugins/translate/dictionary.py | 227 ++++--- backend/src/plugins/translate/events.py | 156 ++--- backend/src/plugins/translate/executor.py | 285 +++------ backend/src/plugins/translate/metrics.py | 36 +- backend/src/plugins/translate/orchestrator.py | 241 ++++--- backend/src/plugins/translate/plugin.py | 10 +- backend/src/plugins/translate/preview.py | 242 +++---- backend/src/plugins/translate/scheduler.py | 150 +++-- backend/src/plugins/translate/service.py | 197 +++--- .../src/plugins/translate/sql_generator.py | 78 ++- .../plugins/translate/superset_executor.py | 196 +++--- backend/src/schemas/__init__.py | 3 +- .../test_settings_and_health_schemas.py | 33 +- backend/src/schemas/auth.py | 10 +- backend/src/schemas/dataset_review.py | 6 +- .../schemas/dataset_review_pkg/_composites.py | 2 +- .../src/schemas/dataset_review_pkg/_dtos.py | 2 +- backend/src/schemas/health.py | 4 +- backend/src/schemas/profile.py | 26 +- backend/src/schemas/settings.py | 4 +- backend/src/schemas/translate.py | 106 ++-- backend/src/scripts/clean_release_cli.py | 52 +- backend/src/scripts/clean_release_tui.py | 18 +- backend/src/scripts/create_admin.py | 15 +- backend/src/scripts/init_auth_db.py | 22 +- .../src/scripts/migrate_sqlite_to_postgres.py | 35 +- backend/src/scripts/seed_permissions.py | 30 +- .../src/scripts/seed_superset_load_test.py | 69 +- .../__tests__/test_encryption_manager.py | 96 +-- .../services/__tests__/test_health_service.py | 33 +- .../__tests__/test_llm_plugin_persistence.py | 111 ++-- .../__tests__/test_llm_prompt_templates.py | 80 +-- .../services/__tests__/test_llm_provider.py | 102 +-- .../__tests__/test_rbac_permission_catalog.py | 62 +- .../__tests__/test_resource_service.py | 88 +-- backend/src/services/auth_service.py | 92 +-- .../src/services/clean_release/__init__.py | 2 +- .../__tests__/test_audit_service.py | 36 +- .../__tests__/test_compliance_orchestrator.py | 46 +- .../__tests__/test_manifest_builder.py | 34 +- .../__tests__/test_policy_engine.py | 56 +- .../__tests__/test_preparation_service.py | 70 ++- .../__tests__/test_report_builder.py | 62 +- .../__tests__/test_source_isolation.py | 36 +- .../clean_release/__tests__/test_stages.py | 44 +- .../clean_release/approval_service.py | 34 +- .../clean_release/artifact_catalog_loader.py | 14 +- .../services/clean_release/audit_service.py | 10 +- .../clean_release/candidate_service.py | 22 +- .../compliance_execution_service.py | 64 +- .../clean_release/compliance_orchestrator.py | 68 +- .../clean_release/demo_data_service.py | 16 +- backend/src/services/clean_release/dto.py | 6 +- backend/src/services/clean_release/enums.py | 6 +- .../src/services/clean_release/exceptions.py | 6 +- backend/src/services/clean_release/facade.py | 6 +- .../clean_release/manifest_builder.py | 12 +- .../clean_release/manifest_service.py | 16 +- backend/src/services/clean_release/mappers.py | 6 +- .../services/clean_release/policy_engine.py | 16 +- .../policy_resolution_service.py | 18 +- .../clean_release/preparation_service.py | 8 +- .../clean_release/publication_service.py | 36 +- .../services/clean_release/report_builder.py | 4 +- .../clean_release/repositories/__init__.py | 4 +- .../repositories/approval_repository.py | 6 +- .../repositories/artifact_repository.py | 6 +- .../repositories/audit_repository.py | 6 +- .../repositories/candidate_repository.py | 6 +- .../repositories/compliance_repository.py | 6 +- .../repositories/manifest_repository.py | 79 +-- .../repositories/policy_repository.py | 6 +- .../repositories/publication_repository.py | 6 +- .../repositories/report_repository.py | 6 +- .../src/services/clean_release/repository.py | 4 +- .../clean_release/source_isolation.py | 4 +- .../services/clean_release/stages/__init__.py | 20 +- .../src/services/clean_release/stages/base.py | 12 +- .../clean_release/stages/data_purity.py | 8 +- .../stages/internal_sources_only.py | 8 +- .../stages/manifest_consistency.py | 8 +- .../stages/no_external_endpoints.py | 8 +- .../src/services/dataset_review/__init__.py | 6 +- .../dataset_review/clarification_engine.py | 103 +-- .../clarification_pkg/_helpers.py | 2 +- .../services/dataset_review/event_logger.py | 83 +-- .../services/dataset_review/orchestrator.py | 169 ++--- .../orchestrator_pkg/_commands.py | 2 +- .../orchestrator_pkg/_helpers.py | 23 +- .../__tests__/test_session_repository.py | 108 ++-- .../repositories/repository_pkg/_mutations.py | 51 +- .../repositories/session_repository.py | 167 ++--- .../dataset_review/semantic_resolver.py | 170 ++--- backend/src/services/git/__init__.py | 22 +- backend/src/services/git/_base.py | 145 ++--- backend/src/services/git/_branch.py | 101 +-- backend/src/services/git/_gitea.py | 138 ++-- backend/src/services/git/_merge.py | 62 +- backend/src/services/git/_remote_providers.py | 56 +- backend/src/services/git/_status.py | 49 +- backend/src/services/git/_sync.py | 97 ++- backend/src/services/git/_url.py | 106 ++-- backend/src/services/git_service.py | 7 +- backend/src/services/health_service.py | 103 +-- backend/src/services/llm_prompt_templates.py | 28 +- backend/src/services/llm_provider.py | 154 ++--- backend/src/services/mapping_service.py | 57 +- .../__tests__/test_notification_service.py | 9 +- .../src/services/notifications/providers.py | 18 +- backend/src/services/notifications/service.py | 115 ++-- backend/src/services/profile_service.py | 302 +++++---- .../src/services/rbac_permission_catalog.py | 40 +- .../__tests__/test_report_normalizer.py | 38 +- .../reports/__tests__/test_report_service.py | 17 +- .../reports/__tests__/test_type_profiles.py | 40 +- backend/src/services/reports/normalizer.py | 42 +- .../src/services/reports/report_service.py | 130 ++-- backend/src/services/reports/type_profiles.py | 14 +- backend/src/services/resource_service.py | 169 ++--- .../src/lib/api/__tests__/reports_api.test.js | 5 + .../lib/auth/__tests__/permissions.test.js | 5 + .../assistant_chat.integration.test.js | 4 +- .../lib/stores/__tests__/mocks/env_public.js | 3 + .../lib/stores/__tests__/mocks/environment.js | 4 + .../lib/stores/__tests__/mocks/navigation.js | 4 + .../src/lib/stores/__tests__/mocks/state.js | 6 +- .../src/lib/stores/__tests__/sidebar.test.js | 5 + .../lib/stores/__tests__/taskDrawer.test.js | 5 + .../src/services/__tests__/gitService.test.js | 5 + 331 files changed, 9630 insertions(+), 10312 deletions(-) diff --git a/.opencode/agents/qa-tester.md b/.opencode/agents/qa-tester.md index 42fbf971..295bc187 100644 --- a/.opencode/agents/qa-tester.md +++ b/.opencode/agents/qa-tester.md @@ -1,7 +1,7 @@ --- description: QA & Semantic Auditor — verification cycle for ss-tools: pytest + vitest coverage, contract validation, invariant traceability, and rejected-path regression defense. mode: subagent -model: opencode-go/mimo-2.5-pro +model: opencode-go/deepseek-v4-pro temperature: 0.1 permission: edit: allow diff --git a/backend/_convert_defs.py b/backend/_convert_defs.py index 966a99fe..39695f5f 100644 --- a/backend/_convert_defs.py +++ b/backend/_convert_defs.py @@ -1,409 +1,232 @@ #!/usr/bin/env python3 """ -Mass migration: [DEF:id:Type] -> #region id [C:N] [TYPE Type] [SEMANTICS ...] +Convert legacy # [DEF:...] / # [/DEF:...] annotations to #region/#endregion format. -Processes all Python files under backend/src/ that contain [DEF: annotations. - -STRATEGY: For each [DEF:...[/DEF:...] block, only replace the two boundary lines -and remove the intermediate metadata tag lines. This preserves any already-converted -inner #region blocks when processing nested DEF blocks. - -Consolidation rules: -- COMPLEXITY + [C:N]: keep only inline [C:N], remove @COMPLEXITY line -- Only @COMPLEXITY: move to inline [C:N], remove @COMPLEXITY line -- BRIEF + PURPOSE: keep @BRIEF, remove @PURPOSE line -- Only @PURPOSE: rename to @BRIEF -- @SEMANTICS: ... -> [SEMANTICS ...] inline in region header -- @LAYER: ... -> @LAYER ... (normalized format) -- @RELATION: ..., @PRE:..., @POST:..., etc -> normalize format +ONLY modifies comment lines (starting with '#'). +NEVER touches code, strings, imports, or non-comment lines. +Preserves original line endings (CRLF vs LF). """ import re -import os -import sys +from pathlib import Path -SOURCE_DIR = os.path.join(os.path.dirname(__file__), "src") - -# Matches: # [DEF:ContractId:Type] -DEF_OPEN_RE = re.compile(r'^(\s*)#\s*\[DEF:([^:]+):([^\]]+)\]') - -# Matches: # [/DEF:ContractId] or # [/DEF:ContractId:Type] -DEF_CLOSE_RE = re.compile(r'^(\s*)#\s*\[/DEF:([^:\]]+)') - -# Inline [C:N] pattern -C_INLINE_RE = re.compile(r'\[C:\s*(\d+)\]') - -# Tag lines to match -TAG_LINE_RE = re.compile( - r'^(\s*)#\s*@(COMPLEXITY|BRIEF|PURPOSE|SEMANTICS|LAYER|RELATION|' - r'PRE|POST|SIDE_EFFECT|DATA_CONTRACT|INVARIANT|RETURN|RATIONALE|REJECTED|' - r'STATUS|DEPRECATED)\s*:\s*(.+?)\s*$', - re.IGNORECASE -) +SRC_DIR = Path(__file__).resolve().parent / "src" -def _ids_match(close_id, open_id): - """Check if close_id matches open_id, handling fully qualified paths.""" - if close_id == open_id: - return True - # Case-insensitive comparison - c_lower = close_id.lower().rstrip('.') - o_lower = open_id.lower().rstrip('.') - if c_lower == o_lower: - return True - # Fully qualified close e.g. backend.src.services.X.Y - if c_lower.endswith('.' + o_lower): - return True - if o_lower.endswith('.' + c_lower): - return True - # Last component matches - if c_lower.split('.')[-1] == o_lower: - return True - if o_lower.split('.')[-1] == c_lower: - return True - # One contains the other - if o_lower in c_lower or c_lower in o_lower: - return True - return False +def has_crlf(content: bytes) -> bool: + """Check if content uses CRLF line endings.""" + return b'\r\n' in content -def normalize_relation(text): - text = text.strip() - # [PRED] -> [Target] - m = re.match(r'\[(\w+)\]\s*->\s*\[(.+?)\]$', text) - if m: - return f"{m.group(1)} -> [{m.group(2)}]" - # PRED -> [Target] - m = re.match(r'(\w+)\s*->\s*\[(.+?)\]$', text) - if m: - return f"{m.group(1)} -> [{m.group(2)}]" - # PRED -> Target - m = re.match(r'(\w+)\s*->\s*(.+)$', text) - if m: - target = m.group(2).strip() - if not target.startswith('['): - target = f'[{target}]' - return f"{m.group(1)} -> {target}" - return text +def process_file(filepath: Path) -> tuple[int, int]: + """Process a single file. Returns (blocks_converted, metadata_merged).""" + # Read as bytes to detect original line endings + with open(filepath, "rb") as f: + raw_bytes = f.read() + original_crlf = has_crlf(raw_bytes) -def process_file(filepath): - """Process a single file, converting [DEF:... blocks to #region/#endregion. - - Only replaces boundary lines and removes metadata tags between them. - Does NOT replace the body content, preserving nested conversions. - """ - with open(filepath, 'r', encoding='utf-8') as f: - lines = f.readlines() - - if not lines: - return False - - # Quick check - if not any('[DEF:' in line for line in lines): - return False - - ############################################################### - # PASS 1: Find all opens and match them to closes - ############################################################### - opens = [] # (line_idx, contract_id, type_str, indent) - for i, line in enumerate(lines): - m = DEF_OPEN_RE.match(line) + # Decode text (Python converts CRLF to LF automatically on decode in text mode, + # but we decoded from bytes, so handle it manually) + text = raw_bytes.decode('utf-8') + if original_crlf: + text = text.replace('\r\n', '\n') + + lines = text.split('\n') + + result = [] + blocks_converted = 0 + metadata_merged = 0 + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + stripped = line.rstrip('\r') + + # Check for [SECTION: ...] or [/SECTION] lines + if re.match(r'^#\s*\[SECTION:\s*.*\]\s*$', stripped): + i += 1 + continue + if re.match(r'^#\s*\[/SECTION\]\s*$', stripped): + i += 1 + continue + + # Check for [/DEF:Name] or [/DEF:Name:Type] + m = re.match(r'^#\s*\[/DEF:(\w+(?:\.\w+)*)(?::\w+)?\]\s*$', stripped) if m: - opens.append((i, m.group(2).strip(), m.group(3).strip(), m.group(1))) - - # Match closes to opens using stack - close_for_open = {} # open_idx -> close_idx - open_stack = [] - for i, line in enumerate(lines): - m = DEF_CLOSE_RE.match(line) + name = m.group(1) + result.append(f"# #endregion {name}") + blocks_converted += 1 + i += 1 + continue + + # Check for [DEF:Name:Type] + m = re.match(r'^#\s*\[DEF:(\w+(?:\.\w+)*):(\w+)\]\s*$', stripped) if m: - close_id = m.group(2).strip() - matched = False - for j in range(len(open_stack) - 1, -1, -1): - oi, oid, otype, oindent = open_stack[j] - if _ids_match(close_id, oid): - close_for_open[oi] = i - open_stack.pop(j) - matched = True + name = m.group(1) + typ = m.group(2) + + # Collect subsequent metadata lines + complexity = None + semantics = None + metadata_lines = [] # other metadata lines to preserve + j = i + 1 + + while j < n: + nxt = lines[j].rstrip('\r') + + # Stop if not a comment + if not nxt.startswith('#'): break - if not matched: - pass # Silently skip unmatched closes - else: - m2 = DEF_OPEN_RE.match(line) - if m2: - open_stack.append((i, m2.group(2).strip(), m2.group(3).strip(), m2.group(1))) - - if not close_for_open: - return False - - ############################################################### - # PASS 2: For each matched block, find metadata and plan edits - ############################################################### - # Edits: list of (line_idx, action, data) - # action = 'replace_header' | 'replace_footer' | 'remove_line' - edits = [] - - for open_idx, contract_id, type_str, indent in opens: - close_idx = close_for_open.get(open_idx) - if close_idx is None: - continue - - # Find nested opens to know where to stop scanning for metadata - nested_opens = [oi for oi, _, _, _ in opens if open_idx < oi < close_idx] - - # Scan for metadata lines between open and first code/nested open - metadata_indices = set() - metadata = {} # line_idx -> (tagname, content) - - for i in range(open_idx + 1, close_idx): - # Stop at nested DEF open - if i in nested_opens: - break - - stripped = lines[i].strip() - - # Stop at code line - if stripped and not stripped.startswith('#'): - break - - # Skip blank lines, bare comments, and inline comments - if not stripped or stripped == '#': - continue - - # Check for tag - m = TAG_LINE_RE.match(lines[i]) - if m: - tagname = m.group(2).upper() - content = m.group(3).strip() - metadata_indices.add(i) - metadata[i] = (tagname, content) - else: - # Non-tag comment - stop here - break - - # Check for inline [C:N] - c_val = None - c_inline = C_INLINE_RE.search(lines[open_idx]) - if c_inline: - c_val = int(c_inline.group(1)) - - # Process metadata with deduplication - brief_val = None - purpose_val = None - sem_val = None - layer_val = None - relations = [] - pres = [] - posts = [] - side_effects = [] - data_contracts = [] - invariants = [] - returns = [] - rationales = [] - rejecteds = [] - statuses = [] - - for line_idx, (tagname, content) in sorted(metadata.items()): - if tagname == 'COMPLEXITY': - if c_val is None: - try: - c_val = int(content) - except ValueError: - pass - elif tagname == 'BRIEF': - if brief_val is None: - brief_val = content - elif tagname == 'PURPOSE': - if brief_val is not None: - pass # Drop purpose when brief exists - elif purpose_val is None: - purpose_val = content - elif tagname == 'SEMANTICS': - if sem_val is None: - sem_val = content - elif tagname == 'LAYER': - if layer_val is None: - layer_val = content - elif tagname == 'RELATION': - relations.append(normalize_relation(content)) - elif tagname == 'PRE': - pres.append(content) - elif tagname == 'POST': - posts.append(content) - elif tagname == 'SIDE_EFFECT': - side_effects.append(content) - elif tagname == 'DATA_CONTRACT': - data_contracts.append(content) - elif tagname == 'INVARIANT': - invariants.append(content) - elif tagname == 'RETURN': - returns.append(content) - elif tagname == 'RATIONALE': - rationales.append(content) - elif tagname == 'REJECTED': - rejecteds.append(content) - elif tagname in ('STATUS', 'DEPRECATED'): - statuses.append(f'{tagname.upper()} {content}') - - # Consolidate brief/purpose - if purpose_val and not brief_val: - brief_val = purpose_val - # If both existed, purpose was already discarded above - - # Build region header - c_part = f' [C:{c_val}]' if c_val is not None else '' - sem_part = f' [SEMANTICS {sem_val}]' if sem_val else '' - region_header = f'{indent}# #region {contract_id}{c_part} [TYPE {type_str}]{sem_part}' - - # Build metadata body - meta_lines = [] - if brief_val: - meta_lines.append(f'{indent}# @BRIEF {brief_val}') - if layer_val: - meta_lines.append(f'{indent}# @LAYER {layer_val}') - for pre in pres: - meta_lines.append(f'{indent}# @PRE {pre}') - for post in posts: - meta_lines.append(f'{indent}# @POST {post}') - for se in side_effects: - meta_lines.append(f'{indent}# @SIDE_EFFECT {se}') - for dc in data_contracts: - meta_lines.append(f'{indent}# @DATA_CONTRACT {dc}') - for inv in invariants: - meta_lines.append(f'{indent}# @INVARIANT {inv}') - for ret in returns: - meta_lines.append(f'{indent}# @RETURN {ret}') - for rel in relations: - meta_lines.append(f'{indent}# @RELATION {rel}') - for st in statuses: - meta_lines.append(f'{indent}# @{st}') - for rat in rationales: - meta_lines.append(f'{indent}# @RATIONALE {rat}') - for rej in rejecteds: - meta_lines.append(f'{indent}# @REJECTED {rej}') - - region_footer = f'{indent}# #endregion {contract_id}' - - # Plan edits: - # 1. Replace [DEF: line with region_header + meta_lines - edits.append(('replace_multi', open_idx, [region_header] + meta_lines)) - - # 2. Remove each metadata tag line - for mi in metadata_indices: - edits.append(('remove', mi, None)) - - # 3. Replace [/DEF: line with #endregion - edits.append(('replace_single', close_idx, region_footer)) - - if not edits: - return False - - # Process from bottom to top (descending line number) to preserve indices - edits.sort(key=lambda e: e[1], reverse=True) - - for action, line_idx, data in edits: - if action == 'replace_multi': - lines[line_idx] = data[0] + '\n' - # Insert remaining lines after - for j, extra_line in enumerate(data[1:], 1): - lines.insert(line_idx + j, extra_line + '\n') - elif action == 'replace_single': - lines[line_idx] = data + '\n' - elif action == 'remove': - lines[line_idx] = None - - # Strip None lines - lines = [l for l in lines if l is not None] - - # Remove empty bare comment lines that might have been left - cleaned = [] - for line in lines: - if line.strip() == '#': - continue - cleaned.append(line) - - with open(filepath, 'w', encoding='utf-8') as f: - f.writelines(cleaned) - - return True + # Stop if it's another DEF or SECTION + if re.match(r'^#\s*\[', nxt): + break + # Skip blank comment lines (just '#') but continue collecting + if re.match(r'^#\s*$', nxt): + metadata_lines.append(lines[j]) + j += 1 + continue + # Check for @COMPLEXITY + cm = re.match(r'^#\s*@COMPLEXITY:\s*(\d+)\s*$', nxt) + if cm: + complexity = int(cm.group(1)) + j += 1 + continue -def find_remaining_defs(): - """Find all files that still have [DEF: patterns.""" - remaining = [] - for root, dirs, files in os.walk(SOURCE_DIR): - dirs[:] = [d for d in dirs if d not in ('__pycache__', '.venv', 'node_modules')] - for f in files: - if f.endswith('.py'): - fp = os.path.join(root, f) - try: - with open(fp, 'r', encoding='utf-8') as fh: - content = fh.read() - if '[DEF:' in content: - remaining.append(fp) - except Exception: - pass - return remaining + # Check for @SEMANTICS + sm = re.match(r'^#\s*@SEMANTICS:\s*(.+?)\s*$', nxt) + if sm: + semantics = sm.group(1).strip() + j += 1 + continue + + # Skip @PARAM, @RETURN, @THROW + if re.match(r'^#\s*@(PARAM|RETURN|THROW)\b', nxt): + j += 1 + continue + + # Convert @PURPOSE -> @BRIEF + pm = re.match(r'^#\s*@PURPOSE:\s*(.*?)\s*$', nxt) + if pm: + metadata_lines.append(f"# @BRIEF {pm.group(1).strip()}") + j += 1 + continue + + # Normalize @RELATION + rm = re.match(r'^#\s*@RELATION:\s*(.+?)\s*$', nxt) + if rm: + rel_text = rm.group(1).strip() + # Remove brackets ONLY from predicate (word before ->) + rel_text = re.sub(r'\[(\w+)\]\s*->', r'\1 ->', rel_text) + # Normalize whitespace around -> + rel_text = re.sub(r'\s*->\s*', ' -> ', rel_text) + metadata_lines.append(f"# @RELATION {rel_text}") + j += 1 + continue + + # Pass through any other tags as-is + if re.match(r'^#\s*@\w+', nxt): + metadata_lines.append(lines[j]) + j += 1 + continue + + # If it doesn't match any known pattern, keep it but stop collecting + break + + # Build the header + header_parts = [f"# #region {name}"] + if complexity is not None: + header_parts.append(f"[C:{complexity}]") + metadata_merged += 1 + header_parts.append(f"[TYPE {typ}]") + if semantics is not None: + header_parts.append(f"[SEMANTICS {semantics}]") + metadata_merged += 1 + + result.append(" ".join(header_parts)) + # Add preserved metadata lines + result.extend(metadata_lines) + + blocks_converted += 1 + i = j + continue + + # Check for standalone @PURPOSE (not in a DEF block) + pm = re.match(r'^#\s*@PURPOSE:\s*(.*?)\s*$', stripped) + if pm: + result.append(f"# @BRIEF {pm.group(1).strip()}") + i += 1 + continue + + # Check for standalone @RELATION normalization + rm = re.match(r'^#\s*@RELATION:\s*(.+?)\s*$', stripped) + if rm: + rel_text = rm.group(1).strip() + # Remove brackets ONLY from predicate (word before ->) + rel_text = re.sub(r'\[(\w+)\]\s*->', r'\1 ->', rel_text) + # Normalize whitespace around -> + rel_text = re.sub(r'\s*->\s*', ' -> ', rel_text) + result.append(f"# @RELATION {rel_text}") + i += 1 + continue + + # Skip @PARAM, @RETURN, @THROW lines (standalone) + if re.match(r'^#\s*@(PARAM|RETURN|THROW)\b', stripped): + i += 1 + continue + + # Pass through everything else unchanged + result.append(line) + i += 1 + + # Build new content with preserved line endings + new_text = '\n'.join(result) + + if new_text == text: + return 0, 0 + + # Convert back to CRLF if original had it + if original_crlf: + new_raw = new_text.replace('\n', '\r\n').encode('utf-8') + else: + new_raw = new_text.encode('utf-8') + + with open(filepath, "wb") as f: + f.write(new_raw) + return blocks_converted, metadata_merged def main(): - all_files = [] - for root, dirs, files in os.walk(SOURCE_DIR): - dirs[:] = [d for d in dirs if d not in ('__pycache__', '.venv', 'node_modules')] - for f in files: - if f.endswith('.py'): - all_files.append(os.path.join(root, f)) - - print(f"Scanning {len(all_files)} Python files...") - - # Multi-pass until stable - max_passes = 5 - for pass_num in range(1, max_passes + 1): - remaining = find_remaining_defs() - if not remaining: - print(f"\n✅ All DEF annotations migrated after pass {pass_num - 1}!") - return - - print(f"\nPass {pass_num}: Processing {len(remaining)} remaining files...") - modified = 0 - errors = 0 - - for fp in sorted(remaining): - rel_path = os.path.relpath(fp, SOURCE_DIR) - try: - changed = process_file(fp) - if changed: - print(f" ✓ {rel_path}") - modified += 1 - except Exception as e: - print(f" ✗ {rel_path}: {e}") - errors += 1 - import traceback - traceback.print_exc() - - print(f" Modified: {modified}, Errors: {errors}") - - if modified == 0 and errors == 0: - # No changes and no errors - files might have DEF patterns - # that can't be matched (e.g. orphaned opens/closes) - break - - # Final report - remaining = find_remaining_defs() - if remaining: - print(f"\n⚠️ {len(remaining)} files still have unconverted [DEF: patterns:") - for fp in sorted(remaining): - rel = os.path.relpath(fp, SOURCE_DIR) - # Count the DEF patterns - with open(fp, 'r') as f: - content = f.read() - opens = content.count('[DEF:') - closes = content.count('[/DEF:') - print(f" {rel} ({opens} opens, {closes} closes)") - else: - print(f"\n✅ All [DEF:] annotations have been migrated to #region format!") + total_blocks = 0 + total_merged = 0 + total_files = 0 + + # Collect all Python files + py_files = sorted(SRC_DIR.rglob("*.py")) + + for filepath in py_files: + # Skip __pycache__, .venv, tests + rel = filepath.relative_to(SRC_DIR) + parts = list(rel.parts) + if "__pycache__" in parts or ".venv" in parts: + continue + if any(p.startswith("__tests__") or p == "tests" for p in parts): + continue + + try: + blocks, merged = process_file(filepath) + if blocks > 0 or merged > 0: + total_files += 1 + total_blocks += blocks + total_merged += merged + print(f" {rel}: {blocks} blocks, {merged} metadata merged") + except Exception as e: + print(f" ERROR processing {rel}: {e}") + + print(f"\nTotal: {total_files} files processed, {total_blocks} DEF blocks converted, {total_merged} metadata merged") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/backend/src/api/auth.py b/backend/src/api/auth.py index 3c19c27a..620030f1 100755 --- a/backend/src/api/auth.py +++ b/backend/src/api/auth.py @@ -1,14 +1,13 @@ # #region AuthApi [C:3] [TYPE Module] [SEMANTICS api, auth, routes, login, logout] +# # @BRIEF Authentication API endpoints. -# @LAYER API -# @INVARIANT All auth endpoints must return consistent error codes. +# @LAYER: API # @RELATION DEPENDS_ON -> [AuthService] # @RELATION DEPENDS_ON -> [get_auth_db] # @RELATION DEPENDS_ON -> [get_current_user] # @RELATION DEPENDS_ON -> [is_adfs_configured] -# +# @INVARIANT: All auth endpoints must return consistent error codes. -# [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordRequestForm from sqlalchemy.orm import Session @@ -20,25 +19,20 @@ from ..core.auth.oauth import oauth, is_adfs_configured from ..core.auth.logger import log_security_event from ..core.logger import belief_scope import starlette.requests -# [/SECTION] # #region router [C:1] [TYPE Variable] -# @BRIEF APIRouter instance for authentication routes. # @RELATION DEPENDS_ON -> [fastapi.APIRouter] +# @BRIEF APIRouter instance for authentication routes. router = APIRouter(prefix="/api/auth", tags=["auth"]) # #endregion router # #region login_for_access_token [C:3] [TYPE Function] # @BRIEF Authenticates a user and returns a JWT access token. -# @PRE form_data contains username and password. -# @POST Returns a Token object on success. -# @THROW: HTTPException 401 if authentication fails. -# @PARAM: form_data (OAuth2PasswordRequestForm) - Login credentials. -# @PARAM: db (Session) - Auth database session. -# @RETURN: Token - The generated JWT token. -# @RELATION: CALLS -> [AuthService.authenticate_user] -# @RELATION: CALLS -> [AuthService.create_session] +# @PRE: form_data contains username and password. +# @POST: Returns a Token object on success. +# @RELATION CALLS -> [AuthService.authenticate_user] +# @RELATION CALLS -> [AuthService.create_session] @router.post("/login", response_model=Token) async def login_for_access_token( form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_auth_db) @@ -64,11 +58,9 @@ async def login_for_access_token( # #region read_users_me [C:3] [TYPE Function] # @BRIEF Retrieves the profile of the currently authenticated user. -# @PRE Valid JWT token provided. -# @POST Returns the current user's data. -# @PARAM: current_user (UserSchema) - The user extracted from the token. -# @RETURN: UserSchema - The current user profile. -# @RELATION: DEPENDS_ON -> [get_current_user] +# @PRE: Valid JWT token provided. +# @POST: Returns the current user's data. +# @RELATION DEPENDS_ON -> [get_current_user] @router.get("/me", response_model=UserSchema) async def read_users_me(current_user: UserSchema = Depends(get_current_user)): with belief_scope("api.auth.me"): @@ -80,10 +72,9 @@ async def read_users_me(current_user: UserSchema = Depends(get_current_user)): # #region logout [C:3] [TYPE Function] # @BRIEF Logs out the current user (placeholder for session revocation). -# @PRE Valid JWT token provided. -# @POST Returns success message. -# @PARAM: current_user (UserSchema) - The user extracted from the token. -# @RELATION: DEPENDS_ON -> [get_current_user] +# @PRE: Valid JWT token provided. +# @POST: Returns success message. +# @RELATION DEPENDS_ON -> [get_current_user] @router.post("/logout") async def logout(current_user: UserSchema = Depends(get_current_user)): with belief_scope("api.auth.logout"): @@ -98,7 +89,7 @@ async def logout(current_user: UserSchema = Depends(get_current_user)): # #region login_adfs [C:3] [TYPE Function] # @BRIEF Initiates the ADFS OIDC login flow. -# @POST Redirects the user to ADFS. +# @POST: Redirects the user to ADFS. # @RELATION USES -> [is_adfs_configured] @router.get("/login/adfs") async def login_adfs(request: starlette.requests.Request): @@ -117,7 +108,7 @@ async def login_adfs(request: starlette.requests.Request): # #region auth_callback_adfs [C:3] [TYPE Function] # @BRIEF Handles the callback from ADFS after successful authentication. -# @POST Provisions user JIT and returns session token. +# @POST: Provisions user JIT and returns session token. # @RELATION DEPENDS_ON -> [is_adfs_configured] # @RELATION CALLS -> [AuthService.provision_adfs_user] # @RELATION CALLS -> [AuthService.create_session] diff --git a/backend/src/api/routes/__init__.py b/backend/src/api/routes/__init__.py index ca993c24..03cf0bfc 100755 --- a/backend/src/api/routes/__init__.py +++ b/backend/src/api/routes/__init__.py @@ -1,9 +1,9 @@ # #region ApiRoutesModule [C:3] [TYPE Module] [SEMANTICS routes, lazy-import, module-registry] # @BRIEF Provide lazy route module loading to avoid heavyweight imports during tests. -# @LAYER API -# @INVARIANT Only names listed in __all__ are importable via __getattr__. +# @LAYER: API # @RELATION CALLS -> [ApiRoutesGetAttr] # @RELATION BINDS_TO -> [Route_Group_Contracts] +# @INVARIANT: Only names listed in __all__ are importable via __getattr__. # #region Route_Group_Contracts [C:3] [TYPE Block] # @BRIEF Declare the canonical route-module registry used by lazy imports and app router inclusion. @@ -41,9 +41,9 @@ __all__ = [ # #region ApiRoutesGetAttr [C:3] [TYPE Function] # @BRIEF Lazily import route module by attribute name. -# @PRE name is module candidate exposed in __all__. -# @POST Returns imported submodule or raises AttributeError. # @RELATION DEPENDS_ON -> [ApiRoutesModule] +# @PRE: name is module candidate exposed in __all__. +# @POST: Returns imported submodule or raises AttributeError. def __getattr__(name): if name in __all__: import importlib diff --git a/backend/src/api/routes/__tests__/conftest.py b/backend/src/api/routes/__tests__/conftest.py index 266ed26b..3fc1280d 100644 --- a/backend/src/api/routes/__tests__/conftest.py +++ b/backend/src/api/routes/__tests__/conftest.py @@ -1,5 +1,6 @@ -# #region RoutesTestsConftest [C:1] [TYPE Module] -# @BRIEF Shared low-fidelity test doubles for API route test modules. +# [DEF:RoutesTestsConftest:Module] +# @COMPLEXITY: 1 +# @PURPOSE: Shared low-fidelity test doubles for API route test modules. class FakeQuery: @@ -41,4 +42,4 @@ class FakeQuery: return len(self._rows) -# #endregion RoutesTestsConftest +# [/DEF:RoutesTestsConftest:Module] diff --git a/backend/src/api/routes/__tests__/test_assistant_api.py b/backend/src/api/routes/__tests__/test_assistant_api.py index 0a6978e5..379b42c9 100644 --- a/backend/src/api/routes/__tests__/test_assistant_api.py +++ b/backend/src/api/routes/__tests__/test_assistant_api.py @@ -1,10 +1,12 @@ import os os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" -# #region AssistantApiTests [C:3] [TYPE Module] [SEMANTICS tests, assistant, api] -# @BRIEF Validate assistant API endpoint logic via direct async handler invocation. -# @INVARIANT Every test clears assistant in-memory state before execution. -# @RELATION DEPENDS_ON -> [AssistantApi] +# [DEF:AssistantApiTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, assistant, api +# @PURPOSE: Validate assistant API endpoint logic via direct async handler invocation. +# @RELATION: DEPENDS_ON -> [AssistantApi] +# @INVARIANT: Every test clears assistant in-memory state before execution. import asyncio import uuid @@ -35,19 +37,20 @@ from src.models.dataset_review import ( ) -# #region _run_async [TYPE Function] -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_run_async:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] def _run_async(coro): return asyncio.run(coro) -# #endregion _run_async +# [/DEF:_run_async:Function] -# #region _FakeTask [C:1] [TYPE Class] -# @BRIEF Lightweight task model stub used as return value from _FakeTaskManager.create_task in assistant route tests. -# @INVARIANT status is a bare string not a TaskStatus enum; callers must not depend on enum semantics. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_FakeTask:Class] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Lightweight task model stub used as return value from _FakeTaskManager.create_task in assistant route tests. +# @INVARIANT: status is a bare string not a TaskStatus enum; callers must not depend on enum semantics. class _FakeTask: def __init__( self, @@ -68,14 +71,15 @@ class _FakeTask: self.finished_at = datetime.utcnow() -# #endregion _FakeTask +# [/DEF:_FakeTask:Class] # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). -# #region _FakeTaskManager [C:2] [TYPE Class] -# @BRIEF In-memory task manager stub that records created tasks for route-level assertions. -# @INVARIANT create_task stores tasks retrievable by get_task/get_tasks without external side effects. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: In-memory task manager stub that records created tasks for route-level assertions. +# @INVARIANT: create_task stores tasks retrievable by get_task/get_tasks without external side effects. class _FakeTaskManager: def __init__(self): self.tasks = {} @@ -104,13 +108,14 @@ class _FakeTaskManager: return list(self.tasks.values()) -# #endregion _FakeTaskManager +# [/DEF:_FakeTaskManager:Class] -# #region _FakeConfigManager [C:2] [TYPE Class] -# @BRIEF Deterministic config stub providing hardcoded dev/prod environments and minimal settings shape for assistant route tests. -# @INVARIANT get_config() returns anonymous inner classes, not real GlobalSettings; only default_environment_id and llm fields are safe to access. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_FakeConfigManager:Class] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: Deterministic config stub providing hardcoded dev/prod environments and minimal settings shape for assistant route tests. +# @INVARIANT: get_config() returns anonymous inner classes, not real GlobalSettings; only default_environment_id and llm fields are safe to access. class _FakeConfigManager: class _Env: def __init__(self, id, name): @@ -132,12 +137,13 @@ class _FakeConfigManager: return _Config() -# #endregion _FakeConfigManager +# [/DEF:_FakeConfigManager:Class] -# #region _admin_user [C:1] [TYPE Function] -# @BRIEF Build admin principal with spec=User for assistant route authorization tests. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_admin_user:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Build admin principal with spec=User for assistant route authorization tests. def _admin_user(): user = MagicMock(spec=User) user.id = "u-admin" @@ -148,12 +154,13 @@ def _admin_user(): return user -# #endregion _admin_user +# [/DEF:_admin_user:Function] -# #region _limited_user [C:1] [TYPE Function] -# @BRIEF Build limited user principal with empty roles for assistant route denial tests. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_limited_user:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Build limited user principal with empty roles for assistant route denial tests. def _limited_user(): user = MagicMock(spec=User) user.id = "u-limited" @@ -162,13 +169,14 @@ def _limited_user(): return user -# #endregion _limited_user +# [/DEF:_limited_user:Function] -# #region _FakeQuery [C:2] [TYPE Class] -# @BRIEF Chainable SQLAlchemy-like query stub returning fixed item lists for assistant message persistence paths. -# @INVARIANT filter() ignores all predicate arguments and returns self; no predicate-based filtering is emulated. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_FakeQuery:Class] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: Chainable SQLAlchemy-like query stub returning fixed item lists for assistant message persistence paths. +# @INVARIANT: filter() ignores all predicate arguments and returns self; no predicate-based filtering is emulated. class _FakeQuery: def __init__(self, items): self.items = items @@ -204,13 +212,14 @@ class _FakeQuery: return len(self.items) -# #endregion _FakeQuery +# [/DEF:_FakeQuery:Class] -# #region _FakeDb [C:2] [TYPE Class] -# @BRIEF Explicit in-memory DB session double limited to assistant message persistence paths. -# @INVARIANT query() always returns _FakeQuery with intentionally non-evaluated predicates; add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_FakeDb:Class] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 2 +# @PURPOSE: Explicit in-memory DB session double limited to assistant message persistence paths. +# @INVARIANT: query() always returns _FakeQuery with intentionally non-evaluated predicates; add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior. class _FakeDb: def __init__(self): self.added = [] @@ -239,11 +248,11 @@ class _FakeDb: pass -# #endregion _FakeDb +# [/DEF:_FakeDb:Class] -# #region _clear_assistant_state [TYPE Function] -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_clear_assistant_state:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] def _clear_assistant_state(): assistant_routes.CONVERSATIONS.clear() assistant_routes.USER_ACTIVE_CONVERSATION.clear() @@ -251,12 +260,13 @@ def _clear_assistant_state(): assistant_routes.ASSISTANT_AUDIT.clear() -# #endregion _clear_assistant_state +# [/DEF:_clear_assistant_state:Function] -# #region _dataset_review_session [C:1] [TYPE Function] -# @BRIEF Build minimal owned dataset-review session fixture for assistant scoped routing tests. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_dataset_review_session:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Build minimal owned dataset-review session fixture for assistant scoped routing tests. def _dataset_review_session(): session = DatasetReviewSession( session_id="sess-1", @@ -354,22 +364,23 @@ def _dataset_review_session(): return session -# #endregion _dataset_review_session +# [/DEF:_dataset_review_session:Function] -# #region _await_none [C:1] [TYPE Function] -# @BRIEF Async helper returning None for planner fallback tests. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:_await_none:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @COMPLEXITY: 1 +# @PURPOSE: Async helper returning None for planner fallback tests. async def _await_none(*args, **kwargs): return None -# #endregion _await_none +# [/DEF:_await_none:Function] -# #region test_unknown_command_returns_needs_clarification [TYPE Function] -# @BRIEF Unknown command should return clarification state and unknown intent. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_unknown_command_returns_needs_clarification:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Unknown command should return clarification state and unknown intent. def test_unknown_command_returns_needs_clarification(monkeypatch): _clear_assistant_state() req = assistant_routes.AssistantMessageRequest(message="some random gibberish") @@ -391,12 +402,12 @@ def test_unknown_command_returns_needs_clarification(monkeypatch): assert "уточните" in resp.text.lower() or "неоднозначна" in resp.text.lower() -# #endregion test_unknown_command_returns_needs_clarification +# [/DEF:test_unknown_command_returns_needs_clarification:Function] -# #region test_capabilities_question_returns_successful_help [TYPE Function] -# @BRIEF Capability query should return deterministic help response. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_capabilities_question_returns_successful_help:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Capability query should return deterministic help response. def test_capabilities_question_returns_successful_help(monkeypatch): _clear_assistant_state() req = assistant_routes.AssistantMessageRequest(message="что ты умеешь?") @@ -415,12 +426,12 @@ def test_capabilities_question_returns_successful_help(monkeypatch): assert "я могу сделать" in resp.text.lower() -# #endregion test_capabilities_question_returns_successful_help +# [/DEF:test_capabilities_question_returns_successful_help:Function] -# #region test_assistant_message_request_accepts_dataset_review_session_binding [TYPE Function] -# @BRIEF Assistant request schema should accept active dataset review session binding for scoped orchestration. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_assistant_message_request_accepts_dataset_review_session_binding:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Assistant request schema should accept active dataset review session binding for scoped orchestration. def test_assistant_message_request_accepts_dataset_review_session_binding(): request = assistant_routes.AssistantMessageRequest( message="approve mappings", @@ -430,12 +441,12 @@ def test_assistant_message_request_accepts_dataset_review_session_binding(): assert request.dataset_review_session_id == "sess-1" -# #endregion test_assistant_message_request_accepts_dataset_review_session_binding +# [/DEF:test_assistant_message_request_accepts_dataset_review_session_binding:Function] -# #region test_dataset_review_scoped_message_uses_masked_filter_context [TYPE Function] -# @BRIEF Session-scoped assistant context should mask imported-filter raw values before assistant-visible metadata is persisted. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_dataset_review_scoped_message_uses_masked_filter_context:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Session-scoped assistant context should mask imported-filter raw values before assistant-visible metadata is persisted. def test_dataset_review_scoped_message_uses_masked_filter_context(monkeypatch): _clear_assistant_state() db = _FakeDb() @@ -478,12 +489,12 @@ def test_dataset_review_scoped_message_uses_masked_filter_context(monkeypatch): assert imported_filters[0]["raw_value_masked"] is True -# #endregion test_dataset_review_scoped_message_uses_masked_filter_context +# [/DEF:test_dataset_review_scoped_message_uses_masked_filter_context:Function] -# #region test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval [TYPE Function] -# @BRIEF Session-scoped assistant commands should route dataset-review mapping approvals into confirmation workflow with bound session metadata. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Session-scoped assistant commands should route dataset-review mapping approvals into confirmation workflow with bound session metadata. def test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval(): _clear_assistant_state() db = _FakeDb() @@ -511,12 +522,12 @@ def test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval assert resp.intent["entities"]["mapping_ids"] == ["map-1"] -# #endregion test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval +# [/DEF:test_dataset_review_scoped_command_returns_confirmation_for_mapping_approval:Function] -# #region test_dataset_review_scoped_command_routes_field_semantics_update [TYPE Function] -# @BRIEF Session-scoped assistant commands should route semantic field updates through explicit confirmation metadata. -# @RELATION BINDS_TO -> [AssistantApiTests] +# [DEF:test_dataset_review_scoped_command_routes_field_semantics_update:Function] +# @RELATION: BINDS_TO -> [AssistantApiTests] +# @PURPOSE: Session-scoped assistant commands should route semantic field updates through explicit confirmation metadata. def test_dataset_review_scoped_command_routes_field_semantics_update(): _clear_assistant_state() db = _FakeDb() @@ -545,7 +556,7 @@ def test_dataset_review_scoped_command_routes_field_semantics_update(): assert resp.intent["entities"]["lock_field"] is True -# #endregion test_dataset_review_scoped_command_routes_field_semantics_update +# [/DEF:test_dataset_review_scoped_command_routes_field_semantics_update:Function] -# #endregion AssistantApiTests +# [/DEF:AssistantApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_assistant_authz.py b/backend/src/api/routes/__tests__/test_assistant_authz.py index 9f9e6be2..ad8e9078 100644 --- a/backend/src/api/routes/__tests__/test_assistant_authz.py +++ b/backend/src/api/routes/__tests__/test_assistant_authz.py @@ -1,12 +1,14 @@ import os os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w=" -# #region TestAssistantAuthz [C:3] [TYPE Module] [SEMANTICS tests, assistant, authz, confirmation, rbac] -# @BRIEF Verify assistant confirmation ownership, expiration, and deny behavior for restricted users. -# @LAYER UI (API Tests) -# @INVARIANT Security-sensitive flows fail closed for unauthorized actors. -# @RELATION DEPENDS_ON -> [AssistantApi] +# [DEF:TestAssistantAuthz:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, assistant, authz, confirmation, rbac +# @PURPOSE: Verify assistant confirmation ownership, expiration, and deny behavior for restricted users. +# @LAYER: UI (API Tests) +# @RELATION: DEPENDS_ON -> AssistantApi +# @INVARIANT: Security-sensitive flows fail closed for unauthorized actors. import os import asyncio @@ -33,23 +35,25 @@ from src.models.assistant import ( ) -# #region _run_async [C:1] [TYPE Function] -# @BRIEF Execute async endpoint handler in synchronous test context. -# @PRE coroutine is awaitable endpoint invocation. -# @POST Returns coroutine result or raises propagated exception. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:_run_async:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Execute async endpoint handler in synchronous test context. +# @PRE: coroutine is awaitable endpoint invocation. +# @POST: Returns coroutine result or raises propagated exception. def _run_async(coroutine): return asyncio.run(coroutine) -# #endregion _run_async +# [/DEF:_run_async:Function] -# #region _FakeTask [C:1] [TYPE Class] -# @BRIEF Lightweight task model used for assistant authz tests. -# @PRE task_id is non-empty string. -# @POST Returns task with provided id, status, and user_id accessible as attributes. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:_FakeTask:Class] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Lightweight task model used for assistant authz tests. +# @PRE: task_id is non-empty string. +# @POST: Returns task with provided id, status, and user_id accessible as attributes. class _FakeTask: def __init__(self, task_id: str, status: str = "RUNNING", user_id: str = "u-admin"): self.id = task_id @@ -57,12 +61,13 @@ class _FakeTask: self.user_id = user_id -# #endregion _FakeTask +# [/DEF:_FakeTask:Class] # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). -# #region _FakeTaskManager [C:2] [TYPE Class] -# @BRIEF In-memory task manager double that records assistant-created tasks deterministically. -# @INVARIANT Only create_task/get_task/get_tasks behavior used by assistant authz routes is emulated. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 2 +# @PURPOSE: In-memory task manager double that records assistant-created tasks deterministically. +# @INVARIANT: Only create_task/get_task/get_tasks behavior used by assistant authz routes is emulated. class _FakeTaskManager: def __init__(self): self._created = [] @@ -88,14 +93,15 @@ class _FakeTaskManager: ) -# #endregion _FakeTaskManager +# [/DEF:_FakeTaskManager:Class] # @CONTRACT: Partial ConfigManager stub for authz tests. Missing: get_config(). -# #region _FakeConfigManager [C:1] [TYPE Class] -# @BRIEF Provide deterministic environment aliases required by intent parsing. -# @PRE No external config or DB state is required. -# @POST get_environments() returns two deterministic SimpleNamespace stubs with id/name. -# @INVARIANT get_config() is absent; only get_environments() is emulated. Safe only for routes that do not invoke get_config() on the injected ConfigManager — verify against assistant.py route handler code before adding new test cases that use this fake. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:_FakeConfigManager:Class] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Provide deterministic environment aliases required by intent parsing. +# @PRE: No external config or DB state is required. +# @POST: get_environments() returns two deterministic SimpleNamespace stubs with id/name. +# @INVARIANT: get_config() is absent; only get_environments() is emulated. Safe only for routes that do not invoke get_config() on the injected ConfigManager — verify against assistant.py route handler code before adding new test cases that use this fake. class _FakeConfigManager: def get_environments(self): return [ @@ -109,46 +115,50 @@ class _FakeConfigManager: ) -# #endregion _FakeConfigManager -# #region _admin_user [C:1] [TYPE Function] -# @BRIEF Build admin principal fixture. -# @PRE Test requires privileged principal for risky operations. -# @POST Returns admin-like user stub with Admin role. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [/DEF:_FakeConfigManager:Class] +# [DEF:_admin_user:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Build admin principal fixture. +# @PRE: Test requires privileged principal for risky operations. +# @POST: Returns admin-like user stub with Admin role. def _admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(id="u-admin", username="admin", roles=[role]) -# #endregion _admin_user -# #region _other_admin_user [C:1] [TYPE Function] -# @BRIEF Build second admin principal fixture for ownership tests. -# @PRE Ownership mismatch scenario needs distinct authenticated actor. -# @POST Returns alternate admin-like user stub. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [/DEF:_admin_user:Function] +# [DEF:_other_admin_user:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Build second admin principal fixture for ownership tests. +# @PRE: Ownership mismatch scenario needs distinct authenticated actor. +# @POST: Returns alternate admin-like user stub. def _other_admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(id="u-admin-2", username="admin2", roles=[role]) -# #endregion _other_admin_user -# #region _limited_user [C:1] [TYPE Function] -# @BRIEF Build limited principal without required assistant execution privileges. -# @PRE Permission denial scenario needs non-admin actor. -# @POST Returns restricted user stub. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [/DEF:_other_admin_user:Function] +# [DEF:_limited_user:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Build limited principal without required assistant execution privileges. +# @PRE: Permission denial scenario needs non-admin actor. +# @POST: Returns restricted user stub. def _limited_user(): role = SimpleNamespace(name="Operator", permissions=[]) return SimpleNamespace(id="u-limited", username="limited", roles=[role]) -# #endregion _limited_user +# [/DEF:_limited_user:Function] -# #region _FakeQuery [C:1] [TYPE Class] -# @BRIEF Minimal chainable query object for fake DB interactions. -# @INVARIANT filter() deliberately discards predicate args and returns self; tests must not assume predicate evaluation. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:_FakeQuery:Class] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal chainable query object for fake DB interactions. +# @INVARIANT: filter() deliberately discards predicate args and returns self; tests must not assume predicate evaluation. class _FakeQuery: def __init__(self, rows): self._rows = list(rows) @@ -178,11 +188,12 @@ class _FakeQuery: return len(self._rows) -# #endregion _FakeQuery -# #region _FakeDb [C:2] [TYPE Class] -# @BRIEF In-memory DB session double constrained to assistant message/confirmation/audit persistence paths. -# @INVARIANT query/add/merge are intentionally narrow and must not claim full SQLAlchemy Session semantics. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [/DEF:_FakeQuery:Class] +# [DEF:_FakeDb:Class] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 2 +# @PURPOSE: In-memory DB session double constrained to assistant message/confirmation/audit persistence paths. +# @INVARIANT: query/add/merge are intentionally narrow and must not claim full SQLAlchemy Session semantics. class _FakeDb: def __init__(self): self._messages = [] @@ -226,12 +237,13 @@ class _FakeDb: return None -# #endregion _FakeDb -# #region _clear_assistant_state [C:1] [TYPE Function] -# @BRIEF Reset assistant process-local state between test cases. -# @PRE Assistant globals may contain state from prior tests. -# @POST Assistant in-memory state dictionaries are cleared. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [/DEF:_FakeDb:Class] +# [DEF:_clear_assistant_state:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @COMPLEXITY: 1 +# @PURPOSE: Reset assistant process-local state between test cases. +# @PRE: Assistant globals may contain state from prior tests. +# @POST: Assistant in-memory state dictionaries are cleared. def _clear_assistant_state(): assistant_module.CONVERSATIONS.clear() assistant_module.USER_ACTIVE_CONVERSATION.clear() @@ -239,14 +251,14 @@ def _clear_assistant_state(): assistant_module.ASSISTANT_AUDIT.clear() -# #endregion _clear_assistant_state +# [/DEF:_clear_assistant_state:Function] -# #region test_confirmation_owner_mismatch_returns_403 [TYPE Function] -# @BRIEF Confirm endpoint should reject requests from user that does not own the confirmation token. -# @PRE Confirmation token is created by first admin actor. -# @POST Second actor receives 403 on confirm operation. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:test_confirmation_owner_mismatch_returns_403:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @PURPOSE: Confirm endpoint should reject requests from user that does not own the confirmation token. +# @PRE: Confirmation token is created by first admin actor. +# @POST: Second actor receives 403 on confirm operation. def test_confirmation_owner_mismatch_returns_403(): _clear_assistant_state() task_manager = _FakeTaskManager() @@ -278,14 +290,14 @@ def test_confirmation_owner_mismatch_returns_403(): assert exc.value.status_code == 403 -# #endregion test_confirmation_owner_mismatch_returns_403 +# [/DEF:test_confirmation_owner_mismatch_returns_403:Function] -# #region test_expired_confirmation_cannot_be_confirmed [TYPE Function] -# @BRIEF Expired confirmation token should be rejected and not create task. -# @PRE Confirmation token exists and is manually expired before confirm request. -# @POST Confirm endpoint raises 400 and no task is created. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:test_expired_confirmation_cannot_be_confirmed:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @PURPOSE: Expired confirmation token should be rejected and not create task. +# @PRE: Confirmation token exists and is manually expired before confirm request. +# @POST: Confirm endpoint raises 400 and no task is created. def test_expired_confirmation_cannot_be_confirmed(): _clear_assistant_state() task_manager = _FakeTaskManager() @@ -320,14 +332,14 @@ def test_expired_confirmation_cannot_be_confirmed(): assert task_manager.get_tasks(limit=10, offset=0) == [] -# #endregion test_expired_confirmation_cannot_be_confirmed +# [/DEF:test_expired_confirmation_cannot_be_confirmed:Function] -# #region test_limited_user_cannot_launch_restricted_operation [TYPE Function] -# @BRIEF Limited user should receive denied state for privileged operation. -# @PRE Restricted user attempts dangerous deploy command. -# @POST Assistant returns denied state and does not execute operation. -# @RELATION BINDS_TO -> [TestAssistantAuthz] +# [DEF:test_limited_user_cannot_launch_restricted_operation:Function] +# @RELATION: BINDS_TO -> [TestAssistantAuthz] +# @PURPOSE: Limited user should receive denied state for privileged operation. +# @PRE: Restricted user attempts dangerous deploy command. +# @POST: Assistant returns denied state and does not execute operation. def test_limited_user_cannot_launch_restricted_operation(): _clear_assistant_state() response = _run_async( @@ -344,5 +356,5 @@ def test_limited_user_cannot_launch_restricted_operation(): assert response.state == "denied" -# #endregion test_limited_user_cannot_launch_restricted_operation -# #endregion TestAssistantAuthz +# [/DEF:test_limited_user_cannot_launch_restricted_operation:Function] +# [/DEF:TestAssistantAuthz:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_api.py index bbdbe570..8c0bb70b 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_api.py @@ -1,8 +1,10 @@ -# #region TestCleanReleaseApi [C:3] [TYPE Module] [SEMANTICS tests, api, clean-release, checks, reports] -# @BRIEF Contract tests for clean release checks and reports endpoints. -# @LAYER Domain -# @INVARIANT API returns deterministic payload shapes for checks and reports. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestCleanReleaseApi:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, api, clean-release, checks, reports +# @PURPOSE: Contract tests for clean release checks and reports endpoints. +# @LAYER: Domain +# @INVARIANT: API returns deterministic payload shapes for checks and reports. from datetime import datetime, timezone @@ -23,8 +25,8 @@ from src.models.clean_release import ( from src.services.clean_release.repository import CleanReleaseRepository -# #region _repo_with_seed_data [TYPE Function] -# @RELATION BINDS_TO -> [TestCleanReleaseApi] +# [DEF:_repo_with_seed_data:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseApi def _repo_with_seed_data() -> CleanReleaseRepository: repo = CleanReleaseRepository() repo.save_candidate( @@ -72,12 +74,12 @@ def _repo_with_seed_data() -> CleanReleaseRepository: return repo -# #endregion _repo_with_seed_data +# [/DEF:_repo_with_seed_data:Function] -# #region test_start_check_and_get_status_contract [TYPE Function] -# @BRIEF Validate checks start endpoint returns expected identifiers and status endpoint reflects the same run. -# @RELATION BINDS_TO -> [TestCleanReleaseApi] +# [DEF:test_start_check_and_get_status_contract:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate checks start endpoint returns expected identifiers and status endpoint reflects the same run. def test_start_check_and_get_status_contract(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -110,12 +112,12 @@ def test_start_check_and_get_status_contract(): app.dependency_overrides.clear() -# #endregion test_start_check_and_get_status_contract +# [/DEF:test_start_check_and_get_status_contract:Function] -# #region test_get_report_not_found_returns_404 [TYPE Function] -# @BRIEF Validate reports endpoint returns 404 for an unknown report identifier. -# @RELATION BINDS_TO -> [TestCleanReleaseApi] +# [DEF:test_get_report_not_found_returns_404:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate reports endpoint returns 404 for an unknown report identifier. def test_get_report_not_found_returns_404(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -127,12 +129,12 @@ def test_get_report_not_found_returns_404(): app.dependency_overrides.clear() -# #endregion test_get_report_not_found_returns_404 +# [/DEF:test_get_report_not_found_returns_404:Function] -# #region test_get_report_success [TYPE Function] -# @BRIEF Validate reports endpoint returns persisted report payload for an existing report identifier. -# @RELATION BINDS_TO -> [TestCleanReleaseApi] +# [DEF:test_get_report_success:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate reports endpoint returns persisted report payload for an existing report identifier. def test_get_report_success(): repo = _repo_with_seed_data() report = ComplianceReport( @@ -157,12 +159,12 @@ def test_get_report_success(): app.dependency_overrides.clear() -# #endregion test_get_report_success +# [/DEF:test_get_report_success:Function] -# #region test_prepare_candidate_api_success [TYPE Function] -# @BRIEF Validate candidate preparation endpoint returns prepared status and manifest identifier on valid input. -# @RELATION BINDS_TO -> [TestCleanReleaseApi] +# [DEF:test_prepare_candidate_api_success:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseApi +# @PURPOSE: Validate candidate preparation endpoint returns prepared status and manifest identifier on valid input. def test_prepare_candidate_api_success(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -187,5 +189,5 @@ def test_prepare_candidate_api_success(): app.dependency_overrides.clear() -# #endregion test_prepare_candidate_api_success -# #endregion TestCleanReleaseApi +# [/DEF:test_prepare_candidate_api_success:Function] +# [/DEF:TestCleanReleaseApi:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py index 43767f99..90a02f0e 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py +++ b/backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py @@ -1,7 +1,8 @@ -# #region TestCleanReleaseLegacyCompat [C:3] [TYPE Module] -# @BRIEF Compatibility tests for legacy clean-release API paths retained during v2 migration. -# @LAYER Tests -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestCleanReleaseLegacyCompat:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @PURPOSE: Compatibility tests for legacy clean-release API paths retained during v2 migration. +# @LAYER: Tests from __future__ import annotations @@ -29,11 +30,11 @@ from src.models.clean_release import ( from src.services.clean_release.repository import CleanReleaseRepository -# #region _seed_legacy_repo [TYPE Function] -# @BRIEF Seed in-memory repository with minimum trusted data for legacy endpoint contracts. -# @PRE Repository is empty. -# @POST Candidate, policy, registry and manifest are available for legacy checks flow. -# @RELATION BINDS_TO -> [TestCleanReleaseLegacyCompat] +# [DEF:_seed_legacy_repo:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat +# @PURPOSE: Seed in-memory repository with minimum trusted data for legacy endpoint contracts. +# @PRE: Repository is empty. +# @POST: Candidate, policy, registry and manifest are available for legacy checks flow. def _seed_legacy_repo() -> CleanReleaseRepository: repo = CleanReleaseRepository() now = datetime.now(timezone.utc) @@ -115,12 +116,12 @@ def _seed_legacy_repo() -> CleanReleaseRepository: return repo -# #endregion _seed_legacy_repo +# [/DEF:_seed_legacy_repo:Function] -# #region test_legacy_prepare_endpoint_still_available [TYPE Function] -# @BRIEF Verify legacy prepare endpoint remains reachable and returns a status payload. -# @RELATION BINDS_TO -> [TestCleanReleaseLegacyCompat] +# [DEF:test_legacy_prepare_endpoint_still_available:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat +# @PURPOSE: Verify legacy prepare endpoint remains reachable and returns a status payload. def test_legacy_prepare_endpoint_still_available() -> None: repo = _seed_legacy_repo() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -145,12 +146,12 @@ def test_legacy_prepare_endpoint_still_available() -> None: app.dependency_overrides.clear() -# #endregion test_legacy_prepare_endpoint_still_available +# [/DEF:test_legacy_prepare_endpoint_still_available:Function] -# #region test_legacy_checks_endpoints_still_available [TYPE Function] -# @BRIEF Verify legacy checks start/status endpoints remain available during v2 transition. -# @RELATION BINDS_TO -> [TestCleanReleaseLegacyCompat] +# [DEF:test_legacy_checks_endpoints_still_available:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat +# @PURPOSE: Verify legacy checks start/status endpoints remain available during v2 transition. def test_legacy_checks_endpoints_still_available() -> None: repo = _seed_legacy_repo() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -182,5 +183,5 @@ def test_legacy_checks_endpoints_still_available() -> None: app.dependency_overrides.clear() -# #endregion test_legacy_checks_endpoints_still_available -# #endregion TestCleanReleaseLegacyCompat +# [/DEF:test_legacy_checks_endpoints_still_available:Function] +# [/DEF:TestCleanReleaseLegacyCompat:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py index 298876eb..4544bf6f 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_source_policy.py +++ b/backend/src/api/routes/__tests__/test_clean_release_source_policy.py @@ -1,8 +1,10 @@ -# #region TestCleanReleaseSourcePolicy [C:3] [TYPE Module] [SEMANTICS tests, api, clean-release, source-policy] -# @BRIEF Validate API behavior for source isolation violations in clean release preparation. -# @LAYER Domain -# @INVARIANT External endpoints must produce blocking violation entries. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestCleanReleaseSourcePolicy:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, api, clean-release, source-policy +# @PURPOSE: Validate API behavior for source isolation violations in clean release preparation. +# @LAYER: Domain +# @INVARIANT: External endpoints must produce blocking violation entries. from datetime import datetime, timezone from fastapi.testclient import TestClient @@ -20,9 +22,9 @@ from src.models.clean_release import ( from src.services.clean_release.repository import CleanReleaseRepository -# #region _repo_with_seed_data [TYPE Function] -# @BRIEF Seed repository with candidate, registry, and active policy for source isolation test flow. -# @RELATION BINDS_TO -> [TestCleanReleaseSourcePolicy] +# [DEF:_repo_with_seed_data:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy +# @PURPOSE: Seed repository with candidate, registry, and active policy for source isolation test flow. def _repo_with_seed_data() -> CleanReleaseRepository: repo = CleanReleaseRepository() @@ -73,12 +75,12 @@ def _repo_with_seed_data() -> CleanReleaseRepository: return repo -# #endregion _repo_with_seed_data +# [/DEF:_repo_with_seed_data:Function] -# #region test_prepare_candidate_blocks_external_source [TYPE Function] -# @BRIEF Verify candidate preparation is blocked when at least one source host is external to the trusted registry. -# @RELATION BINDS_TO -> [TestCleanReleaseSourcePolicy] +# [DEF:test_prepare_candidate_blocks_external_source:Function] +# @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy +# @PURPOSE: Verify candidate preparation is blocked when at least one source host is external to the trusted registry. def test_prepare_candidate_blocks_external_source(): repo = _repo_with_seed_data() app.dependency_overrides[get_clean_release_repository] = lambda: repo @@ -108,5 +110,5 @@ def test_prepare_candidate_blocks_external_source(): app.dependency_overrides.clear() -# #endregion test_prepare_candidate_blocks_external_source -# #endregion TestCleanReleaseSourcePolicy +# [/DEF:test_prepare_candidate_blocks_external_source:Function] +# [/DEF:TestCleanReleaseSourcePolicy:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py index d5b233e7..afc7bed7 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_api.py @@ -1,7 +1,8 @@ -# #region CleanReleaseV2ApiTests [C:3] [TYPE Module] -# @BRIEF API contract tests for redesigned clean release endpoints. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [CleanReleaseV2Api] +# [DEF:CleanReleaseV2ApiTests:Module] +# @COMPLEXITY: 3 +# @PURPOSE: API contract tests for redesigned clean release endpoints. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [CleanReleaseV2Api] from datetime import datetime, timezone from types import SimpleNamespace @@ -24,9 +25,9 @@ client = TestClient(app) # [REASON] Implementing API contract tests for candidate/artifact/manifest endpoints (T012). -# #region test_candidate_registration_contract [TYPE Function] -# @BRIEF Validate candidate registration endpoint creates a draft candidate with expected identifier contract. -# @RELATION BINDS_TO -> [CleanReleaseV2ApiTests] +# [DEF:test_candidate_registration_contract:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate candidate registration endpoint creates a draft candidate with expected identifier contract. def test_candidate_registration_contract(): """ @TEST_SCENARIO: candidate_registration -> Should return 201 and candidate DTO. @@ -45,12 +46,12 @@ def test_candidate_registration_contract(): assert data["status"] == CandidateStatus.DRAFT.value -# #endregion test_candidate_registration_contract +# [/DEF:test_candidate_registration_contract:Function] -# #region test_artifact_import_contract [TYPE Function] -# @BRIEF Validate artifact import endpoint accepts candidate artifacts and returns success status payload. -# @RELATION BINDS_TO -> [CleanReleaseV2ApiTests] +# [DEF:test_artifact_import_contract:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate artifact import endpoint accepts candidate artifacts and returns success status payload. def test_artifact_import_contract(): """ @TEST_SCENARIO: artifact_import -> Should return 200 and success status. @@ -80,12 +81,12 @@ def test_artifact_import_contract(): assert response.json()["status"] == "success" -# #endregion test_artifact_import_contract +# [/DEF:test_artifact_import_contract:Function] -# #region test_manifest_build_contract [TYPE Function] -# @BRIEF Validate manifest build endpoint produces manifest payload linked to the target candidate. -# @RELATION BINDS_TO -> [CleanReleaseV2ApiTests] +# [DEF:test_manifest_build_contract:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests +# @PURPOSE: Validate manifest build endpoint produces manifest payload linked to the target candidate. def test_manifest_build_contract(): """ @TEST_SCENARIO: manifest_build -> Should return 201 and manifest DTO. @@ -110,5 +111,5 @@ def test_manifest_build_contract(): assert data["candidate_id"] == candidate_id -# #endregion test_manifest_build_contract -# #endregion CleanReleaseV2ApiTests +# [/DEF:test_manifest_build_contract:Function] +# [/DEF:CleanReleaseV2ApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py index 66127e1e..11a7441a 100644 --- a/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py +++ b/backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py @@ -1,7 +1,8 @@ -# #region CleanReleaseV2ReleaseApiTests [C:3] [TYPE Module] -# @BRIEF API contract test scaffolding for clean release approval and publication endpoints. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [CleanReleaseV2Api] +# [DEF:CleanReleaseV2ReleaseApiTests:Module] +# @COMPLEXITY: 3 +# @PURPOSE: API contract test scaffolding for clean release approval and publication endpoints. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> [CleanReleaseV2Api] """Contract tests for redesigned approval/publication API endpoints.""" @@ -22,9 +23,9 @@ test_app.include_router(clean_release_v2_router) client = TestClient(test_app) -# #region _seed_candidate_and_passed_report [TYPE Function] -# @BRIEF Seed repository with approvable candidate and passed report for release endpoint contracts. -# @RELATION BINDS_TO -> [CleanReleaseV2ReleaseApiTests] +# [DEF:_seed_candidate_and_passed_report:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Seed repository with approvable candidate and passed report for release endpoint contracts. def _seed_candidate_and_passed_report() -> tuple[str, str]: repository = get_clean_release_repository() candidate_id = f"api-release-candidate-{uuid4()}" @@ -58,12 +59,12 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]: return candidate_id, report_id -# #endregion _seed_candidate_and_passed_report +# [/DEF:_seed_candidate_and_passed_report:Function] -# #region test_release_approve_and_publish_revoke_contract [TYPE Function] -# @BRIEF Verify approve, publish, and revoke endpoints preserve expected release lifecycle contract. -# @RELATION BINDS_TO -> [CleanReleaseV2ReleaseApiTests] +# [DEF:test_release_approve_and_publish_revoke_contract:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Verify approve, publish, and revoke endpoints preserve expected release lifecycle contract. def test_release_approve_and_publish_revoke_contract() -> None: """Contract for approve -> publish -> revoke lifecycle endpoints.""" candidate_id, report_id = _seed_candidate_and_passed_report() @@ -102,12 +103,12 @@ def test_release_approve_and_publish_revoke_contract() -> None: assert revoke_payload["publication"]["status"] == "REVOKED" -# #endregion test_release_approve_and_publish_revoke_contract +# [/DEF:test_release_approve_and_publish_revoke_contract:Function] -# #region test_release_reject_contract [TYPE Function] -# @BRIEF Verify reject endpoint returns successful rejection decision payload. -# @RELATION BINDS_TO -> [CleanReleaseV2ReleaseApiTests] +# [DEF:test_release_reject_contract:Function] +# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests +# @PURPOSE: Verify reject endpoint returns successful rejection decision payload. def test_release_reject_contract() -> None: """Contract for reject endpoint.""" candidate_id, report_id = _seed_candidate_and_passed_report() @@ -122,5 +123,5 @@ def test_release_reject_contract() -> None: assert payload["decision"] == "REJECTED" -# #endregion test_release_reject_contract -# #endregion CleanReleaseV2ReleaseApiTests +# [/DEF:test_release_reject_contract:Function] +# [/DEF:CleanReleaseV2ReleaseApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_connections_routes.py b/backend/src/api/routes/__tests__/test_connections_routes.py index 9f7082e1..3e352d64 100644 --- a/backend/src/api/routes/__tests__/test_connections_routes.py +++ b/backend/src/api/routes/__tests__/test_connections_routes.py @@ -1,7 +1,8 @@ -# #region ConnectionsRoutesTests [C:3] [TYPE Module] -# @BRIEF Verifies connection routes bootstrap their table before CRUD access. -# @LAYER API -# @RELATION DEPENDS_ON -> [ConnectionsRouter] +# [DEF:ConnectionsRoutesTests:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Verifies connection routes bootstrap their table before CRUD access. +# @LAYER: API +# @RELATION: DEPENDS_ON -> ConnectionsRouter import os import sys @@ -42,9 +43,9 @@ def db_session(): session.close() -# #region test_list_connections_bootstraps_missing_table [TYPE Function] -# @BRIEF Ensure listing connections auto-creates missing table and returns empty payload. -# @RELATION BINDS_TO -> [ConnectionsRoutesTests] +# [DEF:test_list_connections_bootstraps_missing_table:Function] +# @RELATION: BINDS_TO -> ConnectionsRoutesTests +# @PURPOSE: Ensure listing connections auto-creates missing table and returns empty payload. def test_list_connections_bootstraps_missing_table(db_session): from src.api.routes.connections import list_connections @@ -55,12 +56,12 @@ def test_list_connections_bootstraps_missing_table(db_session): assert "connection_configs" in inspector.get_table_names() -# #endregion test_list_connections_bootstraps_missing_table +# [/DEF:test_list_connections_bootstraps_missing_table:Function] -# #region test_create_connection_bootstraps_missing_table [TYPE Function] -# @BRIEF Ensure connection creation bootstraps table and persists returned connection fields. -# @RELATION BINDS_TO -> [ConnectionsRoutesTests] +# [DEF:test_create_connection_bootstraps_missing_table:Function] +# @RELATION: BINDS_TO -> ConnectionsRoutesTests +# @PURPOSE: Ensure connection creation bootstraps table and persists returned connection fields. def test_create_connection_bootstraps_missing_table(db_session): from src.api.routes.connections import ConnectionCreate, create_connection @@ -82,5 +83,5 @@ def test_create_connection_bootstraps_missing_table(db_session): assert "connection_configs" in inspector.get_table_names() -# #endregion test_create_connection_bootstraps_missing_table -# #endregion ConnectionsRoutesTests +# [/DEF:test_create_connection_bootstraps_missing_table:Function] +# [/DEF:ConnectionsRoutesTests:Module] diff --git a/backend/src/api/routes/__tests__/test_dashboards.py b/backend/src/api/routes/__tests__/test_dashboards.py index 6e52a743..e6a6231d 100644 --- a/backend/src/api/routes/__tests__/test_dashboards.py +++ b/backend/src/api/routes/__tests__/test_dashboards.py @@ -1,7 +1,8 @@ -# #region DashboardsApiTests [C:3] [TYPE Module] -# @BRIEF Unit tests for dashboards API endpoints. -# @LAYER API -# @RELATION DEPENDS_ON -> [DashboardsApi] +# [DEF:DashboardsApiTests:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for dashboards API endpoints. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [DashboardsApi] import pytest from unittest.mock import MagicMock, patch, AsyncMock @@ -70,9 +71,9 @@ def mock_deps(): client = TestClient(app) -# #region test_get_dashboards_success [TYPE Function] -# @BRIEF Validate dashboards listing returns a populated response that satisfies the schema contract. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing returns a populated response that satisfies the schema contract. # @TEST: GET /api/dashboards returns 200 and valid schema # @PRE: env_id exists # @POST: Response matches DashboardsResponse schema @@ -109,12 +110,12 @@ def test_get_dashboards_success(mock_deps): DashboardsResponse(**data) -# #endregion test_get_dashboards_success +# [/DEF:test_get_dashboards_success:Function] -# #region test_get_dashboards_with_search [TYPE Function] -# @BRIEF Validate dashboards listing applies the search filter and returns only matching rows. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_with_search:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing applies the search filter and returns only matching rows. # @TEST: GET /api/dashboards filters by search term # @PRE: search parameter provided # @POST: Only matching dashboards returned @@ -155,12 +156,12 @@ def test_get_dashboards_with_search(mock_deps): assert data["dashboards"][0]["title"] == "Sales Report" -# #endregion test_get_dashboards_with_search +# [/DEF:test_get_dashboards_with_search:Function] -# #region test_get_dashboards_empty [TYPE Function] -# @BRIEF Validate dashboards listing returns an empty payload for an environment without dashboards. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_empty:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing returns an empty payload for an environment without dashboards. # @TEST_EDGE: empty_dashboards -> {env_id: 'empty_env', expected_total: 0} def test_get_dashboards_empty(mock_deps): """@TEST_EDGE: empty_dashboards -> {env_id: 'empty_env', expected_total: 0}""" @@ -179,12 +180,12 @@ def test_get_dashboards_empty(mock_deps): DashboardsResponse(**data) -# #endregion test_get_dashboards_empty +# [/DEF:test_get_dashboards_empty:Function] -# #region test_get_dashboards_superset_failure [TYPE Function] -# @BRIEF Validate dashboards listing surfaces a 503 contract when Superset access fails. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_superset_failure:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing surfaces a 503 contract when Superset access fails. # @TEST_EDGE: external_superset_failure -> {env_id: 'bad_conn', status: 503} def test_get_dashboards_superset_failure(mock_deps): """@TEST_EDGE: external_superset_failure -> {env_id: 'bad_conn', status: 503}""" @@ -201,12 +202,12 @@ def test_get_dashboards_superset_failure(mock_deps): assert "Failed to fetch dashboards" in response.json()["detail"] -# #endregion test_get_dashboards_superset_failure +# [/DEF:test_get_dashboards_superset_failure:Function] -# #region test_get_dashboards_env_not_found [TYPE Function] -# @BRIEF Validate dashboards listing returns 404 when the requested environment does not exist. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_env_not_found:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing returns 404 when the requested environment does not exist. # @TEST: GET /api/dashboards returns 404 if env_id missing # @PRE: env_id does not exist # @POST: Returns 404 error @@ -218,12 +219,12 @@ def test_get_dashboards_env_not_found(mock_deps): assert "Environment not found" in response.json()["detail"] -# #endregion test_get_dashboards_env_not_found +# [/DEF:test_get_dashboards_env_not_found:Function] -# #region test_get_dashboards_invalid_pagination [TYPE Function] -# @BRIEF Validate dashboards listing rejects invalid pagination parameters with 400 responses. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_invalid_pagination:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboards listing rejects invalid pagination parameters with 400 responses. # @TEST: GET /api/dashboards returns 400 for invalid page/page_size # @PRE: page < 1 or page_size > 100 # @POST: Returns 400 error @@ -242,12 +243,12 @@ def test_get_dashboards_invalid_pagination(mock_deps): assert "Page size must be between 1 and 100" in response.json()["detail"] -# #endregion test_get_dashboards_invalid_pagination +# [/DEF:test_get_dashboards_invalid_pagination:Function] -# #region test_get_dashboard_detail_success [TYPE Function] -# @BRIEF Validate dashboard detail returns charts and datasets for an existing dashboard. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboard_detail_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboard detail returns charts and datasets for an existing dashboard. # @TEST: GET /api/dashboards/{id} returns dashboard detail with charts and datasets def test_get_dashboard_detail_success(mock_deps): with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls: @@ -298,12 +299,12 @@ def test_get_dashboard_detail_success(mock_deps): assert payload["dataset_count"] == 1 -# #endregion test_get_dashboard_detail_success +# [/DEF:test_get_dashboard_detail_success:Function] -# #region test_get_dashboard_detail_env_not_found [TYPE Function] -# @BRIEF Validate dashboard detail returns 404 when the requested environment is missing. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboard_detail_env_not_found:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboard detail returns 404 when the requested environment is missing. # @TEST: GET /api/dashboards/{id} returns 404 for missing environment def test_get_dashboard_detail_env_not_found(mock_deps): mock_deps["config"].get_environments.return_value = [] @@ -314,11 +315,11 @@ def test_get_dashboard_detail_env_not_found(mock_deps): assert "Environment not found" in response.json()["detail"] -# #endregion test_get_dashboard_detail_env_not_found +# [/DEF:test_get_dashboard_detail_env_not_found:Function] -# #region test_migrate_dashboards_success [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_migrate_dashboards_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: POST /api/dashboards/migrate creates migration task # @PRE: Valid source_env_id, target_env_id, dashboard_ids # @PURPOSE: Validate dashboard migration request creates an async task and returns its identifier. @@ -351,11 +352,11 @@ def test_migrate_dashboards_success(mock_deps): mock_deps["task"].create_task.assert_called_once() -# #endregion test_migrate_dashboards_success +# [/DEF:test_migrate_dashboards_success:Function] -# #region test_migrate_dashboards_no_ids [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_migrate_dashboards_no_ids:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: POST /api/dashboards/migrate returns 400 for empty dashboard_ids # @PRE: dashboard_ids is empty # @PURPOSE: Validate dashboard migration rejects empty dashboard identifier lists. @@ -374,13 +375,13 @@ def test_migrate_dashboards_no_ids(mock_deps): assert "At least one dashboard ID must be provided" in response.json()["detail"] -# #endregion test_migrate_dashboards_no_ids +# [/DEF:test_migrate_dashboards_no_ids:Function] -# #region test_migrate_dashboards_env_not_found [TYPE Function] -# @BRIEF Validate migration creation returns 404 when the source environment cannot be resolved. -# @PRE source_env_id and target_env_id are valid environment IDs -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_migrate_dashboards_env_not_found:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate migration creation returns 404 when the source environment cannot be resolved. +# @PRE: source_env_id and target_env_id are valid environment IDs def test_migrate_dashboards_env_not_found(mock_deps): """@PRE: source_env_id and target_env_id are valid environment IDs.""" mock_deps["config"].get_environments.return_value = [] @@ -392,11 +393,11 @@ def test_migrate_dashboards_env_not_found(mock_deps): assert "Source environment not found" in response.json()["detail"] -# #endregion test_migrate_dashboards_env_not_found +# [/DEF:test_migrate_dashboards_env_not_found:Function] -# #region test_backup_dashboards_success [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_backup_dashboards_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: POST /api/dashboards/backup creates backup task # @PRE: Valid env_id, dashboard_ids # @PURPOSE: Validate dashboard backup request creates an async backup task and returns its identifier. @@ -422,13 +423,13 @@ def test_backup_dashboards_success(mock_deps): mock_deps["task"].create_task.assert_called_once() -# #endregion test_backup_dashboards_success +# [/DEF:test_backup_dashboards_success:Function] -# #region test_backup_dashboards_env_not_found [TYPE Function] -# @BRIEF Validate backup task creation returns 404 when the target environment is missing. -# @PRE env_id is a valid environment ID -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_backup_dashboards_env_not_found:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate backup task creation returns 404 when the target environment is missing. +# @PRE: env_id is a valid environment ID def test_backup_dashboards_env_not_found(mock_deps): """@PRE: env_id is a valid environment ID.""" mock_deps["config"].get_environments.return_value = [] @@ -439,11 +440,11 @@ def test_backup_dashboards_env_not_found(mock_deps): assert "Environment not found" in response.json()["detail"] -# #endregion test_backup_dashboards_env_not_found +# [/DEF:test_backup_dashboards_env_not_found:Function] -# #region test_get_database_mappings_success [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_database_mappings_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards/db-mappings returns mapping suggestions # @PRE: Valid source_env_id, target_env_id # @PURPOSE: Validate database mapping suggestions are returned for valid source and target environments. @@ -478,13 +479,13 @@ def test_get_database_mappings_success(mock_deps): assert data["mappings"][0]["confidence"] == 0.95 -# #endregion test_get_database_mappings_success +# [/DEF:test_get_database_mappings_success:Function] -# #region test_get_database_mappings_env_not_found [TYPE Function] -# @BRIEF Validate database mapping suggestions return 404 when either environment is missing. -# @PRE source_env_id and target_env_id are valid environment IDs -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_database_mappings_env_not_found:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate database mapping suggestions return 404 when either environment is missing. +# @PRE: source_env_id and target_env_id are valid environment IDs def test_get_database_mappings_env_not_found(mock_deps): """@PRE: source_env_id must be a valid environment.""" mock_deps["config"].get_environments.return_value = [] @@ -494,12 +495,12 @@ def test_get_database_mappings_env_not_found(mock_deps): assert response.status_code == 404 -# #endregion test_get_database_mappings_env_not_found +# [/DEF:test_get_database_mappings_env_not_found:Function] -# #region test_get_dashboard_tasks_history_filters_success [TYPE Function] -# @BRIEF Validate dashboard task history returns only related backup and LLM tasks. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboard_tasks_history_filters_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboard task history returns only related backup and LLM tasks. # @TEST: GET /api/dashboards/{id}/tasks returns backup and llm tasks for dashboard def test_get_dashboard_tasks_history_filters_success(mock_deps): now = datetime.now(timezone.utc) @@ -545,12 +546,12 @@ def test_get_dashboard_tasks_history_filters_success(mock_deps): } -# #endregion test_get_dashboard_tasks_history_filters_success +# [/DEF:test_get_dashboard_tasks_history_filters_success:Function] -# #region test_get_dashboard_thumbnail_success [TYPE Function] -# @BRIEF Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboard_thumbnail_success:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Validate dashboard thumbnail endpoint proxies image bytes and content type from Superset. # @TEST: GET /api/dashboards/{id}/thumbnail proxies image bytes from Superset def test_get_dashboard_thumbnail_success(mock_deps): with patch("src.api.routes.dashboards._detail_routes.SupersetClient") as mock_client_cls: @@ -579,14 +580,14 @@ def test_get_dashboard_thumbnail_success(mock_deps): assert response.headers["content-type"].startswith("image/png") -# #endregion test_get_dashboard_thumbnail_success +# [/DEF:test_get_dashboard_thumbnail_success:Function] -# #region _build_profile_preference_stub [TYPE Function] -# @BRIEF Creates profile preference payload stub for dashboards filter contract tests. -# @PRE username can be empty; enabled indicates profile-default toggle state. -# @POST Returns object compatible with ProfileService.get_my_preference contract. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:_build_profile_preference_stub:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Creates profile preference payload stub for dashboards filter contract tests. +# @PRE: username can be empty; enabled indicates profile-default toggle state. +# @POST: Returns object compatible with ProfileService.get_my_preference contract. def _build_profile_preference_stub(username: str, enabled: bool): preference = MagicMock() preference.superset_username = username @@ -600,14 +601,14 @@ def _build_profile_preference_stub(username: str, enabled: bool): return payload -# #endregion _build_profile_preference_stub +# [/DEF:_build_profile_preference_stub:Function] -# #region _matches_actor_case_insensitive [TYPE Function] -# @BRIEF Applies trim + case-insensitive owners OR modified_by matching used by route contract tests. -# @PRE owners can be None or list-like values. -# @POST Returns True when bound username matches any owner or modified_by. -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:_matches_actor_case_insensitive:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests +# @PURPOSE: Applies trim + case-insensitive owners OR modified_by matching used by route contract tests. +# @PRE: owners can be None or list-like values. +# @POST: Returns True when bound username matches any owner or modified_by. def _matches_actor_case_insensitive(bound_username, owners, modified_by): normalized_bound = str(bound_username or "").strip().lower() if not normalized_bound: @@ -625,11 +626,11 @@ def _matches_actor_case_insensitive(bound_username, owners, modified_by): ) -# #endregion _matches_actor_case_insensitive +# [/DEF:_matches_actor_case_insensitive:Function] -# #region test_get_dashboards_profile_filter_contract_owners_or_modified_by [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_profile_filter_contract_owners_or_modified_by:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards applies profile-default filter with owners OR modified_by trim+case-insensitive semantics. # @PURPOSE: Validate profile-default filtering matches owner and modifier aliases using normalized Superset actor values. # @PRE: Current user has enabled profile-default preference and bound username. @@ -692,11 +693,11 @@ def test_get_dashboards_profile_filter_contract_owners_or_modified_by(mock_deps) assert payload["effective_profile_filter"]["match_logic"] == "owners_or_modified_by" -# #endregion test_get_dashboards_profile_filter_contract_owners_or_modified_by +# [/DEF:test_get_dashboards_profile_filter_contract_owners_or_modified_by:Function] -# #region test_get_dashboards_override_show_all_contract [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_override_show_all_contract:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards honors override_show_all and disables profile-default filter for current page. # @PURPOSE: Validate override_show_all bypasses profile-default filtering without changing dashboard list semantics. # @PRE: Profile-default preference exists but override_show_all=true query is provided. @@ -753,11 +754,11 @@ def test_get_dashboards_override_show_all_contract(mock_deps): profile_service.matches_dashboard_actor.assert_not_called() -# #endregion test_get_dashboards_override_show_all_contract +# [/DEF:test_get_dashboards_override_show_all_contract:Function] -# #region test_get_dashboards_profile_filter_no_match_results_contract [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_profile_filter_no_match_results_contract:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards returns empty result set when profile-default filter is active and no dashboard actors match. # @PURPOSE: Validate profile-default filtering returns an empty dashboard page when no actor aliases match the bound user. # @PRE: Profile-default preference is enabled with bound username and all dashboards are non-matching. @@ -816,11 +817,11 @@ def test_get_dashboards_profile_filter_no_match_results_contract(mock_deps): assert payload["effective_profile_filter"]["match_logic"] == "owners_or_modified_by" -# #endregion test_get_dashboards_profile_filter_no_match_results_contract +# [/DEF:test_get_dashboards_profile_filter_no_match_results_contract:Function] -# #region test_get_dashboards_page_context_other_disables_profile_default [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_page_context_other_disables_profile_default:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards does not auto-apply profile-default filter outside dashboards_main page context. # @PURPOSE: Validate non-dashboard page contexts suppress profile-default filtering and preserve unfiltered results. # @PRE: Profile-default preference exists but page_context=other query is provided. @@ -877,11 +878,11 @@ def test_get_dashboards_page_context_other_disables_profile_default(mock_deps): profile_service.matches_dashboard_actor.assert_not_called() -# #endregion test_get_dashboards_page_context_other_disables_profile_default +# [/DEF:test_get_dashboards_page_context_other_disables_profile_default:Function] -# #region test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards resolves Superset display-name alias once and filters without per-dashboard detail calls. # @PURPOSE: Validate profile-default filtering reuses resolved Superset display aliases without triggering per-dashboard detail fanout. # @PRE: Profile-default filter is active, bound username is `admin`, dashboard actors contain display labels. @@ -962,11 +963,11 @@ def test_get_dashboards_profile_filter_matches_display_alias_without_detail_fano superset_client.get_dashboard.assert_not_called() -# #endregion test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout +# [/DEF:test_get_dashboards_profile_filter_matches_display_alias_without_detail_fanout:Function] -# #region test_get_dashboards_profile_filter_matches_owner_object_payload_contract [TYPE Function] -# @RELATION BINDS_TO -> [DashboardsApiTests] +# [DEF:test_get_dashboards_profile_filter_matches_owner_object_payload_contract:Function] +# @RELATION: BINDS_TO -> DashboardsApiTests # @TEST: GET /api/dashboards profile-default filter matches Superset owner object payloads. # @PURPOSE: Validate profile-default filtering accepts owner object payloads once aliases resolve to the bound Superset username. # @PRE: Profile-default preference is enabled and owners list contains dict payloads. @@ -1044,7 +1045,7 @@ def test_get_dashboards_profile_filter_matches_owner_object_payload_contract(moc assert payload["dashboards"][0]["title"] == "Featured Charts" -# #endregion test_get_dashboards_profile_filter_matches_owner_object_payload_contract +# [/DEF:test_get_dashboards_profile_filter_matches_owner_object_payload_contract:Function] -# #endregion DashboardsApiTests +# [/DEF:DashboardsApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_dataset_review_api.py b/backend/src/api/routes/__tests__/test_dataset_review_api.py index 4c52982d..209d8e66 100644 --- a/backend/src/api/routes/__tests__/test_dataset_review_api.py +++ b/backend/src/api/routes/__tests__/test_dataset_review_api.py @@ -1,8 +1,10 @@ -# #region DatasetReviewApiTests [C:3] [TYPE Module] [SEMANTICS dataset_review, api, tests, lifecycle, exports, orchestration] -# @BRIEF Verify backend US1 dataset review lifecycle, export, parsing, and dictionary-resolution contracts. -# @LAYER API -# @RELATION BINDS_TO -> [DatasetReviewApi] -# @RELATION BINDS_TO -> [DatasetReviewOrchestrator] +# [DEF:DatasetReviewApiTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: dataset_review, api, tests, lifecycle, exports, orchestration +# @PURPOSE: Verify backend US1 dataset review lifecycle, export, parsing, and dictionary-resolution contracts. +# @LAYER: API +# @RELATION: [BINDS_TO] ->[DatasetReviewApi] +# @RELATION: [BINDS_TO] ->[DatasetReviewOrchestrator] from datetime import datetime, timezone import json @@ -61,6 +63,9 @@ from src.services.dataset_review.orchestrator import ( PreparePreviewResult, StartSessionCommand, ) +from src.services.dataset_review.orchestrator_pkg._helpers import ( + build_execution_snapshot, +) from src.services.dataset_review.semantic_resolver import SemanticSourceResolver from src.services.dataset_review.event_logger import SessionEventLogger from src.services.dataset_review.repositories.session_repository import ( @@ -71,8 +76,8 @@ from src.services.dataset_review.repositories.session_repository import ( client = TestClient(app) -# #region _make_user [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_user:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_user(): permissions = [ SimpleNamespace(resource="dataset:session", action="READ"), @@ -85,11 +90,11 @@ def _make_user(): return SimpleNamespace(id="user-1", username="tester", roles=[dataset_review_role]) -# #endregion _make_user +# [/DEF:_make_user:Function] -# #region _make_config_manager [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_config_manager:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_config_manager(): env = Environment( id="env-1", @@ -107,11 +112,11 @@ def _make_config_manager(): return manager -# #endregion _make_config_manager +# [/DEF:_make_config_manager:Function] -# #region _make_session [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_session:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_session(): now = datetime.now(timezone.utc) return DatasetReviewSession( @@ -134,11 +139,11 @@ def _make_session(): ) -# #endregion _make_session +# [/DEF:_make_session:Function] -# #region _make_us2_session [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_us2_session:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_us2_session(): now = datetime.now(timezone.utc) session = _make_session() @@ -252,11 +257,11 @@ def _make_us2_session(): return session -# #endregion _make_us2_session +# [/DEF:_make_us2_session:Function] -# #region _make_us3_session [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_us3_session:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_us3_session(): """Fake session factory for US3 flow tests. @@ -322,11 +327,11 @@ def _make_us3_session(): return session -# #endregion _make_us3_session +# [/DEF:_make_us3_session:Function] -# #region _make_preview_ready_session [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:_make_preview_ready_session:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests def _make_preview_ready_session(): session = _make_us3_session() session.readiness_state = ReadinessState.COMPILED_PREVIEW_READY @@ -335,11 +340,11 @@ def _make_preview_ready_session(): return session -# #endregion _make_preview_ready_session +# [/DEF:_make_preview_ready_session:Function] -# #region dataset_review_api_dependencies [TYPE Function] -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:dataset_review_api_dependencies:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests @pytest.fixture(autouse=True) def dataset_review_api_dependencies(): mock_user = _make_user() @@ -358,12 +363,12 @@ def dataset_review_api_dependencies(): app.dependency_overrides.clear() -# #endregion dataset_review_api_dependencies +# [/DEF:dataset_review_api_dependencies:Function] -# #region test_parse_superset_link_dashboard_partial_recovery [TYPE Function] -# @BRIEF Verify dashboard links recover dataset context and preserve explicit partial-recovery markers. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_parse_superset_link_dashboard_partial_recovery:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify dashboard links recover dataset context and preserve explicit partial-recovery markers. def test_parse_superset_link_dashboard_partial_recovery(): env = Environment( id="env-1", @@ -395,12 +400,12 @@ def test_parse_superset_link_dashboard_partial_recovery(): assert result.imported_filters[0]["filter_name"] == "country" -# #endregion test_parse_superset_link_dashboard_partial_recovery +# [/DEF:test_parse_superset_link_dashboard_partial_recovery:Function] -# #region test_parse_superset_link_dashboard_slug_recovery [TYPE Function] -# @BRIEF Verify dashboard slug links resolve through dashboard detail endpoints and recover dataset context. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_parse_superset_link_dashboard_slug_recovery:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify dashboard slug links resolve through dashboard detail endpoints and recover dataset context. def test_parse_superset_link_dashboard_slug_recovery(): env = Environment( id="env-1", @@ -432,12 +437,12 @@ def test_parse_superset_link_dashboard_slug_recovery(): fake_client.get_dashboard_detail.assert_called_once_with("slack") -# #endregion test_parse_superset_link_dashboard_slug_recovery +# [/DEF:test_parse_superset_link_dashboard_slug_recovery:Function] -# #region test_parse_superset_link_dashboard_permalink_partial_recovery [TYPE Function] -# @BRIEF Verify dashboard permalink links no longer fail parsing and preserve permalink filter state for partial recovery. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_parse_superset_link_dashboard_permalink_partial_recovery:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify dashboard permalink links no longer fail parsing and preserve permalink filter state for partial recovery. def test_parse_superset_link_dashboard_permalink_partial_recovery(): env = Environment( id="env-1", @@ -482,9 +487,9 @@ def test_parse_superset_link_dashboard_permalink_partial_recovery(): fake_client.get_dashboard_permalink_state.assert_called_once_with("QabXy6wG30Z") -# #region test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state [TYPE Function] -# @BRIEF Verify permalink state with nested dashboard id recovers dataset binding and keeps imported filters. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify permalink state with nested dashboard id recovers dataset binding and keeps imported filters. def test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state(): env = Environment( id="env-1", @@ -526,13 +531,13 @@ def test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_da assert result.imported_filters[0]["filter_name"] == "country" -# #endregion test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state -# #endregion test_parse_superset_link_dashboard_permalink_partial_recovery +# [/DEF:test_parse_superset_link_dashboard_permalink_recovers_dataset_from_nested_dashboard_state:Function] +# [/DEF:test_parse_superset_link_dashboard_permalink_partial_recovery:Function] -# #region test_resolve_from_dictionary_prefers_exact_match [TYPE Function] -# @BRIEF Verify trusted dictionary exact matches outrank fuzzy candidates and unresolved fields stay explicit. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_resolve_from_dictionary_prefers_exact_match:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify trusted dictionary exact matches outrank fuzzy candidates and unresolved fields stay explicit. def test_resolve_from_dictionary_prefers_exact_match(): resolver = SemanticSourceResolver() result = resolver.resolve_from_dictionary( @@ -572,12 +577,12 @@ def test_resolve_from_dictionary_prefers_exact_match(): assert result.partial_recovery is True -# #endregion test_resolve_from_dictionary_prefers_exact_match +# [/DEF:test_resolve_from_dictionary_prefers_exact_match:Function] -# #region test_orchestrator_start_session_preserves_partial_recovery [TYPE Function] -# @BRIEF Verify session start persists usable recovery-required state when Superset intake is partial. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_orchestrator_start_session_preserves_partial_recovery:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify session start persists usable recovery-required state when Superset intake is partial. def test_orchestrator_start_session_preserves_partial_recovery( dataset_review_api_dependencies, ): @@ -640,12 +645,12 @@ def test_orchestrator_start_session_preserves_partial_recovery( repository.save_profile_and_findings.assert_called_once() -# #endregion test_orchestrator_start_session_preserves_partial_recovery +# [/DEF:test_orchestrator_start_session_preserves_partial_recovery:Function] -# #region test_orchestrator_start_session_bootstraps_recovery_state [TYPE Function] -# @BRIEF Verify session start persists recovered filters, template variables, and initial execution mappings for review workspace bootstrap. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_orchestrator_start_session_bootstraps_recovery_state:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify session start persists recovered filters, template variables, and initial execution mappings for review workspace bootstrap. def test_orchestrator_start_session_bootstraps_recovery_state( dataset_review_api_dependencies, ): @@ -675,6 +680,7 @@ def test_orchestrator_start_session_bootstraps_recovery_state( partial_recovery=True, unresolved_references=["dashboard_dataset_binding_missing"], imported_filters=[{"filter_name": "country", "raw_value": ["DE"]}], + dataset_payload=None, ) fake_extractor = MagicMock() @@ -748,12 +754,12 @@ def test_orchestrator_start_session_bootstraps_recovery_state( assert saved_mappings[0].raw_input_value == ["DE"] -# #endregion test_orchestrator_start_session_bootstraps_recovery_state +# [/DEF:test_orchestrator_start_session_bootstraps_recovery_state:Function] -# #region test_start_session_endpoint_returns_created_summary [TYPE Function] -# @BRIEF Verify POST session lifecycle endpoint returns a persisted ownership-scoped summary. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_start_session_endpoint_returns_created_summary:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify POST session lifecycle endpoint returns a persisted ownership-scoped summary. def test_start_session_endpoint_returns_created_summary( dataset_review_api_dependencies, ): @@ -781,12 +787,12 @@ def test_start_session_endpoint_returns_created_summary( assert payload["environment_id"] == "env-1" -# #endregion test_start_session_endpoint_returns_created_summary +# [/DEF:test_start_session_endpoint_returns_created_summary:Function] -# #region test_get_session_detail_export_and_lifecycle_endpoints [TYPE Function] -# @BRIEF Verify lifecycle get/patch/delete plus documentation and validation exports remain ownership-scoped and usable. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_get_session_detail_export_and_lifecycle_endpoints:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Verify lifecycle get/patch/delete plus documentation and validation exports remain ownership-scoped and usable. def test_get_session_detail_export_and_lifecycle_endpoints( dataset_review_api_dependencies, ): @@ -899,11 +905,11 @@ def test_get_session_detail_export_and_lifecycle_endpoints( assert delete_response.status_code == 204 -# #endregion test_get_session_detail_export_and_lifecycle_endpoints +# [/DEF:test_get_session_detail_export_and_lifecycle_endpoints:Function] -# #region test_get_clarification_state_returns_empty_payload_when_session_has_no_record [TYPE Function] -# @BRIEF Clarification state endpoint should return a non-blocking empty payload when the session has no clarification aggregate yet. +# [DEF:test_get_clarification_state_returns_empty_payload_when_session_has_no_record:Function] +# @PURPOSE: Clarification state endpoint should return a non-blocking empty payload when the session has no clarification aggregate yet. def test_get_clarification_state_returns_empty_payload_when_session_has_no_record( dataset_review_api_dependencies, ): @@ -922,12 +928,12 @@ def test_get_clarification_state_returns_empty_payload_when_session_has_no_recor } -# #endregion test_get_clarification_state_returns_empty_payload_when_session_has_no_record +# [/DEF:test_get_clarification_state_returns_empty_payload_when_session_has_no_record:Function] -# #region test_us2_clarification_endpoints_persist_answer_and_feedback [TYPE Function] -# @BRIEF Clarification endpoints should expose one current question, persist the answer before advancement, and store feedback on the answer audit record. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us2_clarification_endpoints_persist_answer_and_feedback:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Clarification endpoints should expose one current question, persist the answer before advancement, and store feedback on the answer audit record. def test_us2_clarification_endpoints_persist_answer_and_feedback( dataset_review_api_dependencies, ): @@ -989,12 +995,12 @@ def test_us2_clarification_endpoints_persist_answer_and_feedback( assert session.clarification_sessions[0].questions[0].answer.user_feedback == "up" -# #endregion test_us2_clarification_endpoints_persist_answer_and_feedback +# [/DEF:test_us2_clarification_endpoints_persist_answer_and_feedback:Function] -# #region test_us2_field_semantic_override_lock_unlock_and_feedback [TYPE Function] -# @BRIEF Semantic field endpoints should apply manual overrides with lock/provenance invariants and persist feedback independently. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us2_field_semantic_override_lock_unlock_and_feedback:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Semantic field endpoints should apply manual overrides with lock/provenance invariants and persist feedback independently. def test_us2_field_semantic_override_lock_unlock_and_feedback( dataset_review_api_dependencies, ): @@ -1071,12 +1077,12 @@ def test_us2_field_semantic_override_lock_unlock_and_feedback( assert session.semantic_fields[0].user_feedback == "down" -# #endregion test_us2_field_semantic_override_lock_unlock_and_feedback +# [/DEF:test_us2_field_semantic_override_lock_unlock_and_feedback:Function] -# #region test_us3_mapping_patch_approval_preview_and_launch_endpoints [TYPE Function] -# @BRIEF US3 execution endpoints should persist manual overrides, preserve explicit approval semantics, return contract-shaped preview truth, and expose audited launch handoff. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: US3 execution endpoints should persist manual overrides, preserve explicit approval semantics, return contract-shaped preview truth, and expose audited launch handoff. def test_us3_mapping_patch_approval_preview_and_launch_endpoints( dataset_review_api_dependencies, ): @@ -1279,12 +1285,12 @@ def test_us3_mapping_patch_approval_preview_and_launch_endpoints( ) -# #endregion test_us3_mapping_patch_approval_preview_and_launch_endpoints +# [/DEF:test_us3_mapping_patch_approval_preview_and_launch_endpoints:Function] -# #region test_us3_preview_response_propagates_refreshed_session_version_for_launch_follow_up [TYPE Function] -# @BRIEF Preview response should expose the refreshed session version so the normal preview-then-launch UI flow can satisfy optimistic locking without a forced full reload. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us3_preview_response_propagates_refreshed_session_version_for_launch_follow_up:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Preview response should expose the refreshed session version so the normal preview-then-launch UI flow can satisfy optimistic locking without a forced full reload. def test_us3_preview_response_propagates_refreshed_session_version_for_launch_follow_up( dataset_review_api_dependencies, ): @@ -1378,12 +1384,12 @@ def test_us3_preview_response_propagates_refreshed_session_version_for_launch_fo assert launch_response.json()["run_context"]["run_context_id"] == "run-2" -# #endregion test_us3_preview_response_propagates_refreshed_session_version_for_launch_follow_up +# [/DEF:test_us3_preview_response_propagates_refreshed_session_version_for_launch_follow_up:Function] -# #region test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift [TYPE Function] -# @BRIEF Preview endpoint should preserve API contract and surface generic upstream preview failures without fabricating dashboard-not-found semantics for non-dashboard 404s. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Preview endpoint should preserve API contract and surface generic upstream preview failures without fabricating dashboard-not-found semantics for non-dashboard 404s. def test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift( dataset_review_api_dependencies, ): @@ -1432,12 +1438,12 @@ def test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not assert "Dashboard not found" not in payload["error_details"] -# #endregion test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift +# [/DEF:test_us3_preview_endpoint_returns_failed_preview_without_false_dashboard_not_found_contract_drift:Function] -# #region test_mutation_endpoints_surface_session_version_conflict_payload [TYPE Function] -# @BRIEF Dataset review mutation endpoints should return deterministic 409 conflict semantics when optimistic-lock versions are stale. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_mutation_endpoints_surface_session_version_conflict_payload:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Dataset review mutation endpoints should return deterministic 409 conflict semantics when optimistic-lock versions are stale. def test_mutation_endpoints_surface_session_version_conflict_payload( dataset_review_api_dependencies, ): @@ -1472,12 +1478,12 @@ def test_mutation_endpoints_surface_session_version_conflict_payload( assert payload["actual_version"] == 5 -# #endregion test_mutation_endpoints_surface_session_version_conflict_payload +# [/DEF:test_mutation_endpoints_surface_session_version_conflict_payload:Function] -# #region test_update_session_surfaces_commit_time_session_version_conflict_payload [TYPE Function] -# @BRIEF Session lifecycle mutation should return deterministic 409 conflict semantics when commit-time optimistic locking rejects a stale write. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_update_session_surfaces_commit_time_session_version_conflict_payload:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Session lifecycle mutation should return deterministic 409 conflict semantics when commit-time optimistic locking rejects a stale write. def test_update_session_surfaces_commit_time_session_version_conflict_payload( dataset_review_api_dependencies, ): @@ -1510,12 +1516,12 @@ def test_update_session_surfaces_commit_time_session_version_conflict_payload( assert payload["actual_version"] == 1 -# #endregion test_update_session_surfaces_commit_time_session_version_conflict_payload +# [/DEF:test_update_session_surfaces_commit_time_session_version_conflict_payload:Function] -# #region test_execution_snapshot_includes_recovered_imported_filters_without_template_mapping [TYPE Function] -# @BRIEF Recovered imported filters with values should flow into preview filter context even when no template variable mapping exists. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_execution_snapshot_includes_recovered_imported_filters_without_template_mapping:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Recovered imported filters with values should flow into preview filter context even when no template variable mapping exists. def test_execution_snapshot_includes_recovered_imported_filters_without_template_mapping( dataset_review_api_dependencies, ): @@ -1545,7 +1551,7 @@ def test_execution_snapshot_includes_recovered_imported_filters_without_template session.execution_mappings = [] session.semantic_fields = [] - snapshot = orchestrator._build_execution_snapshot(session) + snapshot = build_execution_snapshot(session) assert snapshot["template_params"] == {} assert snapshot["preview_blockers"] == [] @@ -1557,7 +1563,7 @@ def test_execution_snapshot_includes_recovered_imported_filters_without_template "value_origin": "extra_form_data.filters", } - snapshot = orchestrator._build_execution_snapshot(session) + snapshot = build_execution_snapshot(session) assert snapshot["template_params"] == {} assert snapshot["preview_blockers"] == [] @@ -1583,12 +1589,12 @@ def test_execution_snapshot_includes_recovered_imported_filters_without_template ] -# #endregion test_execution_snapshot_includes_recovered_imported_filters_without_template_mapping +# [/DEF:test_execution_snapshot_includes_recovered_imported_filters_without_template_mapping:Function] -# #region test_execution_snapshot_preserves_mapped_template_variables_and_filter_context [TYPE Function] -# @BRIEF Mapped template variables should still populate template params while contributing their effective filter context. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_execution_snapshot_preserves_mapped_template_variables_and_filter_context:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Mapped template variables should still populate template params while contributing their effective filter context. def test_execution_snapshot_preserves_mapped_template_variables_and_filter_context( dataset_review_api_dependencies, ): @@ -1604,7 +1610,7 @@ def test_execution_snapshot_preserves_mapped_template_variables_and_filter_conte ) session = _make_preview_ready_session() - snapshot = orchestrator._build_execution_snapshot(session) + snapshot = build_execution_snapshot(session) assert snapshot["template_params"] == {"country": "DE"} assert snapshot["preview_blockers"] == [] @@ -1622,12 +1628,12 @@ def test_execution_snapshot_preserves_mapped_template_variables_and_filter_conte assert snapshot["open_warning_refs"] == ["map-1"] -# #endregion test_execution_snapshot_preserves_mapped_template_variables_and_filter_context +# [/DEF:test_execution_snapshot_preserves_mapped_template_variables_and_filter_context:Function] -# #region test_execution_snapshot_skips_partial_imported_filters_without_values [TYPE Function] -# @BRIEF Partial imported filters without raw or normalized values must not emit bogus active preview filters. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_execution_snapshot_skips_partial_imported_filters_without_values:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Partial imported filters without raw or normalized values must not emit bogus active preview filters. def test_execution_snapshot_skips_partial_imported_filters_without_values( dataset_review_api_dependencies, ): @@ -1657,19 +1663,19 @@ def test_execution_snapshot_skips_partial_imported_filters_without_values( session.execution_mappings = [] session.semantic_fields = [] - snapshot = orchestrator._build_execution_snapshot(session) + snapshot = build_execution_snapshot(session) assert snapshot["template_params"] == {} assert snapshot["effective_filters"] == [] assert snapshot["preview_blockers"] == [] -# #endregion test_execution_snapshot_skips_partial_imported_filters_without_values +# [/DEF:test_execution_snapshot_skips_partial_imported_filters_without_values:Function] -# #region test_us3_launch_endpoint_requires_launch_permission [TYPE Function] -# @BRIEF Launch endpoint should enforce the contract RBAC permission instead of the generic session-manage permission. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_us3_launch_endpoint_requires_launch_permission:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Launch endpoint should enforce the contract RBAC permission instead of the generic session-manage permission. def test_us3_launch_endpoint_requires_launch_permission( dataset_review_api_dependencies, ): @@ -1720,12 +1726,12 @@ def test_us3_launch_endpoint_requires_launch_permission( ) -# #endregion test_us3_launch_endpoint_requires_launch_permission +# [/DEF:test_us3_launch_endpoint_requires_launch_permission:Function] -# #region test_semantic_source_version_propagation_preserves_locked_fields [TYPE Function] -# @BRIEF Updated semantic source versions should mark unlocked fields reviewable while preserving locked manual values. -# @RELATION BINDS_TO -> [DatasetReviewApiTests] +# [DEF:test_semantic_source_version_propagation_preserves_locked_fields:Function] +# @RELATION: BINDS_TO -> DatasetReviewApiTests +# @PURPOSE: Updated semantic source versions should mark unlocked fields reviewable while preserving locked manual values. def test_semantic_source_version_propagation_preserves_locked_fields(): resolver = SemanticSourceResolver() source = SimpleNamespace(source_id="src-1", source_version="2026.04") @@ -1759,6 +1765,6 @@ def test_semantic_source_version_propagation_preserves_locked_fields(): assert locked_field.needs_review is False -# #endregion test_semantic_source_version_propagation_preserves_locked_fields +# [/DEF:test_semantic_source_version_propagation_preserves_locked_fields:Function] -# #endregion DatasetReviewApiTests +# [/DEF:DatasetReviewApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_datasets.py b/backend/src/api/routes/__tests__/test_datasets.py index 9269d60a..a2098901 100644 --- a/backend/src/api/routes/__tests__/test_datasets.py +++ b/backend/src/api/routes/__tests__/test_datasets.py @@ -1,8 +1,10 @@ -# #region DatasetsApiTests [C:3] [TYPE Module] [SEMANTICS datasets, api, tests, pagination, mapping, docs] -# @BRIEF Unit tests for datasets API endpoints. -# @LAYER API -# @INVARIANT Endpoint contracts remain stable for success and validation failure paths. -# @RELATION DEPENDS_ON -> [DatasetsApi] +# [DEF:DatasetsApiTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: datasets, api, tests, pagination, mapping, docs +# @PURPOSE: Unit tests for datasets API endpoints. +# @LAYER: API +# @RELATION: DEPENDS_ON -> [DatasetsApi] +# @INVARIANT: Endpoint contracts remain stable for success and validation failure paths. import pytest from unittest.mock import MagicMock, patch, AsyncMock @@ -70,9 +72,9 @@ def mock_deps(): client = TestClient(app) -# #region test_get_datasets_success [TYPE Function] -# @BRIEF Validate successful datasets listing contract for an existing environment. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_get_datasets_success:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate successful datasets listing contract for an existing environment. # @TEST: GET /api/datasets returns 200 and valid schema # @PRE: env_id exists # @POST: Response matches DatasetsResponse schema @@ -106,12 +108,12 @@ def test_get_datasets_success(mock_deps): DatasetsResponse(**data) -# #endregion test_get_datasets_success +# [/DEF:test_get_datasets_success:Function] -# #region test_get_datasets_env_not_found [TYPE Function] -# @BRIEF Validate datasets listing returns 404 when the requested environment does not exist. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_get_datasets_env_not_found:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate datasets listing returns 404 when the requested environment does not exist. # @TEST: GET /api/datasets returns 404 if env_id missing # @PRE: env_id does not exist # @POST: Returns 404 error @@ -124,12 +126,12 @@ def test_get_datasets_env_not_found(mock_deps): assert "Environment not found" in response.json()["detail"] -# #endregion test_get_datasets_env_not_found +# [/DEF:test_get_datasets_env_not_found:Function] -# #region test_get_datasets_invalid_pagination [TYPE Function] -# @BRIEF Validate datasets listing rejects invalid pagination parameters with 400 responses. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_get_datasets_invalid_pagination:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate datasets listing rejects invalid pagination parameters with 400 responses. # @TEST: GET /api/datasets returns 400 for invalid page/page_size # @PRE: page < 1 or page_size > 100 # @POST: Returns 400 error @@ -154,12 +156,12 @@ def test_get_datasets_invalid_pagination(mock_deps): assert "Page size must be between 1 and 100" in response.json()["detail"] -# #endregion test_get_datasets_invalid_pagination +# [/DEF:test_get_datasets_invalid_pagination:Function] -# #region test_map_columns_success [TYPE Function] -# @BRIEF Validate map-columns request creates an async mapping task and returns its identifier. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_map_columns_success:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate map-columns request creates an async mapping task and returns its identifier. # @TEST: POST /api/datasets/map-columns creates mapping task # @PRE: Valid env_id, dataset_ids, source_type # @POST: Returns task_id @@ -186,12 +188,12 @@ def test_map_columns_success(mock_deps): mock_deps["task"].create_task.assert_called_once() -# #endregion test_map_columns_success +# [/DEF:test_map_columns_success:Function] -# #region test_map_columns_invalid_source_type [TYPE Function] -# @BRIEF Validate map-columns rejects unsupported source types with a 400 contract response. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_map_columns_invalid_source_type:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate map-columns rejects unsupported source types with a 400 contract response. # @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type # @PRE: source_type is not 'postgresql' or 'xlsx' # @POST: Returns 400 error @@ -205,11 +207,11 @@ def test_map_columns_invalid_source_type(mock_deps): assert "Source type must be 'postgresql' or 'xlsx'" in response.json()["detail"] -# #endregion test_map_columns_invalid_source_type +# [/DEF:test_map_columns_invalid_source_type:Function] -# #region test_generate_docs_success [TYPE Function] -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_generate_docs_success:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @TEST: POST /api/datasets/generate-docs creates doc generation task # @PRE: Valid env_id, dataset_ids, llm_provider # @PURPOSE: Validate generate-docs request creates an async documentation task and returns its identifier. @@ -237,12 +239,12 @@ def test_generate_docs_success(mock_deps): mock_deps["task"].create_task.assert_called_once() -# #endregion test_generate_docs_success +# [/DEF:test_generate_docs_success:Function] -# #region test_map_columns_empty_ids [TYPE Function] -# @BRIEF Validate map-columns rejects empty dataset identifier lists. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_map_columns_empty_ids:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate map-columns rejects empty dataset identifier lists. # @TEST: POST /api/datasets/map-columns returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error @@ -256,12 +258,12 @@ def test_map_columns_empty_ids(mock_deps): assert "At least one dataset ID must be provided" in response.json()["detail"] -# #endregion test_map_columns_empty_ids +# [/DEF:test_map_columns_empty_ids:Function] -# #region test_generate_docs_empty_ids [TYPE Function] -# @BRIEF Validate generate-docs rejects empty dataset identifier lists. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_generate_docs_empty_ids:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate generate-docs rejects empty dataset identifier lists. # @TEST: POST /api/datasets/generate-docs returns 400 for empty dataset_ids # @PRE: dataset_ids is empty # @POST: Returns 400 error @@ -275,11 +277,11 @@ def test_generate_docs_empty_ids(mock_deps): assert "At least one dataset ID must be provided" in response.json()["detail"] -# #endregion test_generate_docs_empty_ids +# [/DEF:test_generate_docs_empty_ids:Function] -# #region test_generate_docs_env_not_found [TYPE Function] -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_generate_docs_env_not_found:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] # @TEST: POST /api/datasets/generate-docs returns 404 for missing env # @PRE: env_id does not exist # @PURPOSE: Validate generate-docs returns 404 when the requested environment cannot be resolved. @@ -295,12 +297,12 @@ def test_generate_docs_env_not_found(mock_deps): assert "Environment not found" in response.json()["detail"] -# #endregion test_generate_docs_env_not_found +# [/DEF:test_generate_docs_env_not_found:Function] -# #region test_get_datasets_superset_failure [TYPE Function] -# @BRIEF Validate datasets listing surfaces a 503 contract when Superset access fails. -# @RELATION BINDS_TO -> [DatasetsApiTests:Module] +# [DEF:test_get_datasets_superset_failure:Function] +# @RELATION: BINDS_TO -> [DatasetsApiTests:Module] +# @PURPOSE: Validate datasets listing surfaces a 503 contract when Superset access fails. # @TEST_EDGE: external_superset_failure -> {status: 503} # @POST: Returns 503 with stable error detail when upstream dataset fetch fails. def test_get_datasets_superset_failure(mock_deps): @@ -318,7 +320,7 @@ def test_get_datasets_superset_failure(mock_deps): assert "Failed to fetch datasets" in response.json()["detail"] -# #endregion test_get_datasets_superset_failure +# [/DEF:test_get_datasets_superset_failure:Function] -# #endregion DatasetsApiTests +# [/DEF:DatasetsApiTests:Module] diff --git a/backend/src/api/routes/__tests__/test_git_api.py b/backend/src/api/routes/__tests__/test_git_api.py index d9249b02..50ef85ba 100644 --- a/backend/src/api/routes/__tests__/test_git_api.py +++ b/backend/src/api/routes/__tests__/test_git_api.py @@ -1,6 +1,7 @@ -# #region TestGitApi [C:3] [TYPE Module] -# @BRIEF API tests for Git configurations and repository operations. -# @RELATION VERIFIES -> [GitApi] +# [DEF:TestGitApi:Module] +# @COMPLEXITY: 3 +# @RELATION: VERIFIES -> [GitApi] +# @PURPOSE: API tests for Git configurations and repository operations. import pytest import asyncio @@ -10,10 +11,11 @@ from src.api.routes import git as git_routes from src.models.git import GitServerConfig, GitProvider, GitStatus, GitRepository -# #region DbMock [C:2] [TYPE Class] -# @BRIEF In-memory session double for git route tests with minimal query/filter persistence semantics. -# @INVARIANT Supports only the SQLAlchemy-like operations exercised by this test module. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:DbMock:Class] +# @RELATION: BINDS_TO -> [TestGitApi] +# @COMPLEXITY: 2 +# @PURPOSE: In-memory session double for git route tests with minimal query/filter persistence semantics. +# @INVARIANT: Supports only the SQLAlchemy-like operations exercised by this test module. class DbMock: def __init__(self, data=None): self._data = data or [] @@ -82,12 +84,12 @@ class DbMock: item.last_validated = "2026-03-08T00:00:00Z" -# #endregion DbMock +# [/DEF:DbMock:Class] -# #region test_get_git_configs_masks_pat [TYPE Function] -# @BRIEF Validate listing git configs masks stored PAT values in API-facing responses. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_get_git_configs_masks_pat:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate listing git configs masks stored PAT values in API-facing responses. def test_get_git_configs_masks_pat(): """ @PRE: Database session `db` is available. @@ -114,12 +116,12 @@ def test_get_git_configs_masks_pat(): assert result[0].name == "Test Server" -# #endregion test_get_git_configs_masks_pat +# [/DEF:test_get_git_configs_masks_pat:Function] -# #region test_create_git_config_persists_config [TYPE Function] -# @BRIEF Validate creating git config persists supplied server attributes in backing session. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_create_git_config_persists_config:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate creating git config persists supplied server attributes in backing session. def test_create_git_config_persists_config(): """ @PRE: `config` contains valid GitServerConfigCreate data. @@ -147,14 +149,14 @@ def test_create_git_config_persists_config(): ) # Note: route returns unmasked until serialized by FastAPI usually, but in tests schema might catch it or not. -# #endregion test_create_git_config_persists_config +# [/DEF:test_create_git_config_persists_config:Function] from src.api.routes.git_schemas import GitServerConfigUpdate -# #region test_update_git_config_modifies_record [TYPE Function] -# @BRIEF Validate updating git config modifies mutable fields while preserving masked PAT semantics. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_update_git_config_modifies_record:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate updating git config modifies mutable fields while preserving masked PAT semantics. def test_update_git_config_modifies_record(): """ @PRE: `config_id` corresponds to an existing configuration. @@ -188,6 +190,8 @@ def test_update_git_config_modifies_record(): def refresh(self, config): pass + # [/DEF:SingleConfigDbMock:Class] + db = SingleConfigDbMock() update_data = GitServerConfigUpdate(name="Updated Server", pat="********") @@ -204,12 +208,12 @@ def test_update_git_config_modifies_record(): assert result.pat == "********" -# #endregion test_update_git_config_modifies_record +# [/DEF:test_update_git_config_modifies_record:Function] -# #region test_update_git_config_raises_404_if_not_found [TYPE Function] -# @BRIEF Validate updating non-existent git config raises HTTP 404 contract response. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_update_git_config_raises_404_if_not_found:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate updating non-existent git config raises HTTP 404 contract response. def test_update_git_config_raises_404_if_not_found(): """ @PRE: `config_id` corresponds to a missing configuration. @@ -229,12 +233,12 @@ def test_update_git_config_raises_404_if_not_found(): assert exc_info.value.detail == "Configuration not found" -# #endregion test_update_git_config_raises_404_if_not_found +# [/DEF:test_update_git_config_raises_404_if_not_found:Function] -# #region test_delete_git_config_removes_record [TYPE Function] -# @BRIEF Validate deleting existing git config removes record and returns success payload. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_delete_git_config_removes_record:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate deleting existing git config removes record and returns success payload. def test_delete_git_config_removes_record(): """ @PRE: `config_id` corresponds to an existing configuration. @@ -259,6 +263,8 @@ def test_delete_git_config_removes_record(): def commit(self): pass + # [/DEF:SingleConfigDbMock:Class] + db = SingleConfigDbMock() result = asyncio.run(git_routes.delete_git_config(config_id="config-1", db=db)) @@ -267,12 +273,12 @@ def test_delete_git_config_removes_record(): assert result["status"] == "success" -# #endregion test_delete_git_config_removes_record +# [/DEF:test_delete_git_config_removes_record:Function] -# #region test_test_git_config_validates_connection_successfully [TYPE Function] -# @BRIEF Validate test-connection endpoint returns success when provider connectivity check passes. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_test_git_config_validates_connection_successfully:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate test-connection endpoint returns success when provider connectivity check passes. def test_test_git_config_validates_connection_successfully(monkeypatch): """ @PRE: `config` contains provider, url, and pat. @@ -284,6 +290,8 @@ def test_test_git_config_validates_connection_successfully(monkeypatch): async def test_connection(self, provider, url, pat): return True + # [/DEF:MockGitService:Class] + monkeypatch.setattr(git_routes, "git_service", MockGitService()) from src.api.routes.git_schemas import GitServerConfigCreate @@ -300,12 +308,12 @@ def test_test_git_config_validates_connection_successfully(monkeypatch): assert result["status"] == "success" -# #endregion test_test_git_config_validates_connection_successfully +# [/DEF:test_test_git_config_validates_connection_successfully:Function] -# #region test_test_git_config_fails_validation [TYPE Function] -# @BRIEF Validate test-connection endpoint raises HTTP 400 when provider connectivity check fails. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_test_git_config_fails_validation:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate test-connection endpoint raises HTTP 400 when provider connectivity check fails. def test_test_git_config_fails_validation(monkeypatch): """ @PRE: `config` contains provider, url, and pat BUT connection fails. @@ -317,6 +325,8 @@ def test_test_git_config_fails_validation(monkeypatch): async def test_connection(self, provider, url, pat): return False + # [/DEF:MockGitService:Class] + monkeypatch.setattr(git_routes, "git_service", MockGitService()) from src.api.routes.git_schemas import GitServerConfigCreate @@ -335,12 +345,12 @@ def test_test_git_config_fails_validation(monkeypatch): assert exc_info.value.detail == "Connection failed" -# #endregion test_test_git_config_fails_validation +# [/DEF:test_test_git_config_fails_validation:Function] -# #region test_list_gitea_repositories_returns_payload [TYPE Function] -# @BRIEF Validate gitea repositories endpoint returns normalized list for GITEA provider configs. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_list_gitea_repositories_returns_payload:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate gitea repositories endpoint returns normalized list for GITEA provider configs. def test_list_gitea_repositories_returns_payload(monkeypatch): """ @PRE: config_id exists and provider is GITEA. @@ -354,6 +364,8 @@ def test_list_gitea_repositories_returns_payload(monkeypatch): {"name": "test-repo", "full_name": "owner/test-repo", "private": True} ] + # [/DEF:MockGitService:Class] + monkeypatch.setattr(git_routes, "git_service", MockGitService()) existing_config = GitServerConfig( id="config-1", @@ -373,12 +385,12 @@ def test_list_gitea_repositories_returns_payload(monkeypatch): assert result[0].private is True -# #endregion test_list_gitea_repositories_returns_payload +# [/DEF:test_list_gitea_repositories_returns_payload:Function] -# #region test_list_gitea_repositories_rejects_non_gitea [TYPE Function] -# @BRIEF Validate gitea repositories endpoint rejects non-GITEA providers with HTTP 400. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_list_gitea_repositories_rejects_non_gitea:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate gitea repositories endpoint rejects non-GITEA providers with HTTP 400. def test_list_gitea_repositories_rejects_non_gitea(monkeypatch): """ @PRE: config_id exists and provider is NOT GITEA. @@ -400,12 +412,12 @@ def test_list_gitea_repositories_rejects_non_gitea(monkeypatch): assert "GITEA provider only" in exc_info.value.detail -# #endregion test_list_gitea_repositories_rejects_non_gitea +# [/DEF:test_list_gitea_repositories_rejects_non_gitea:Function] -# #region test_create_remote_repository_creates_provider_repo [TYPE Function] -# @BRIEF Validate remote repository creation endpoint maps provider response into normalized payload. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_create_remote_repository_creates_provider_repo:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate remote repository creation endpoint maps provider response into normalized payload. def test_create_remote_repository_creates_provider_repo(monkeypatch): """ @PRE: config_id exists and PAT has creation permissions. @@ -424,6 +436,8 @@ def test_create_remote_repository_creates_provider_repo(monkeypatch): "clone_url": f"{server_url}/user/{name}.git", } + # [/DEF:MockGitService:Class] + monkeypatch.setattr(git_routes, "git_service", MockGitService()) from src.api.routes.git_schemas import RemoteRepoCreateRequest @@ -448,12 +462,12 @@ def test_create_remote_repository_creates_provider_repo(monkeypatch): assert result.full_name == "user/new-repo" -# #endregion test_create_remote_repository_creates_provider_repo +# [/DEF:test_create_remote_repository_creates_provider_repo:Function] -# #region test_init_repository_initializes_and_saves_binding [TYPE Function] -# @BRIEF Validate repository initialization endpoint creates local repo and persists dashboard binding. -# @RELATION BINDS_TO -> [TestGitApi] +# [DEF:test_init_repository_initializes_and_saves_binding:Function] +# @RELATION: BINDS_TO -> [TestGitApi] +# @PURPOSE: Validate repository initialization endpoint creates local repo and persists dashboard binding. def test_init_repository_initializes_and_saves_binding(monkeypatch): """ @PRE: `dashboard_ref` exists and `init_data` contains valid config_id and remote_url. @@ -469,6 +483,8 @@ def test_init_repository_initializes_and_saves_binding(monkeypatch): def _get_repo_path(self, dashboard_id, repo_key): return f"/tmp/repos/{repo_key}" + # [/DEF:MockGitService:Class] + git_service_mock = MockGitService() monkeypatch.setattr(git_routes, "git_service", git_service_mock) monkeypatch.setattr( @@ -507,5 +523,5 @@ def test_init_repository_initializes_and_saves_binding(monkeypatch): assert db._added[0].dashboard_id == 123 -# #endregion test_init_repository_initializes_and_saves_binding -# #endregion TestGitApi +# [/DEF:test_init_repository_initializes_and_saves_binding:Function] +# [/DEF:TestGitApi:Module] diff --git a/backend/src/api/routes/__tests__/test_git_status_route.py b/backend/src/api/routes/__tests__/test_git_status_route.py index 7940d0dd..867fd706 100644 --- a/backend/src/api/routes/__tests__/test_git_status_route.py +++ b/backend/src/api/routes/__tests__/test_git_status_route.py @@ -1,7 +1,9 @@ -# #region TestGitStatusRoute [C:3] [TYPE Module] [SEMANTICS tests, git, api, status, no_repo] -# @BRIEF Validate status endpoint behavior for missing and error repository states. -# @LAYER Domain (Tests) -# @RELATION VERIFIES -> [GitApi] +# [DEF:TestGitStatusRoute:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, git, api, status, no_repo +# @PURPOSE: Validate status endpoint behavior for missing and error repository states. +# @LAYER: Domain (Tests) +# @RELATION: VERIFIES -> [GitApi] from fastapi import HTTPException import pytest @@ -11,11 +13,11 @@ from unittest.mock import MagicMock from src.api.routes import git as git_routes -# #region test_get_repository_status_returns_no_repo_payload_for_missing_repo [TYPE Function] -# @BRIEF Ensure missing local repository is represented as NO_REPO payload instead of an API error. -# @PRE GitService.get_status raises HTTPException(404). -# @POST Route returns a deterministic NO_REPO status payload. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_status_returns_no_repo_payload_for_missing_repo:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure missing local repository is represented as NO_REPO payload instead of an API error. +# @PRE: GitService.get_status raises HTTPException(404). +# @POST: Route returns a deterministic NO_REPO status payload. def test_get_repository_status_returns_no_repo_payload_for_missing_repo(monkeypatch): class MissingRepoGitService: def _get_repo_path(self, dashboard_id: int) -> str: @@ -32,14 +34,14 @@ def test_get_repository_status_returns_no_repo_payload_for_missing_repo(monkeypa assert response["sync_state"] == "NO_REPO" assert response["has_repo"] is False assert response["current_branch"] is None -# #endregion test_get_repository_status_returns_no_repo_payload_for_missing_repo +# [/DEF:test_get_repository_status_returns_no_repo_payload_for_missing_repo:Function] -# #region test_get_repository_status_propagates_non_404_http_exception [TYPE Function] -# @BRIEF Ensure HTTP exceptions other than 404 are not masked. -# @PRE GitService.get_status raises HTTPException with non-404 status. -# @POST Raised exception preserves original status and detail. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_status_propagates_non_404_http_exception:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure HTTP exceptions other than 404 are not masked. +# @PRE: GitService.get_status raises HTTPException with non-404 status. +# @POST: Raised exception preserves original status and detail. def test_get_repository_status_propagates_non_404_http_exception(monkeypatch): class ConflictGitService: def _get_repo_path(self, dashboard_id: int) -> str: @@ -56,14 +58,14 @@ def test_get_repository_status_propagates_non_404_http_exception(monkeypatch): assert exc_info.value.status_code == 409 assert exc_info.value.detail == "Conflict" -# #endregion test_get_repository_status_propagates_non_404_http_exception +# [/DEF:test_get_repository_status_propagates_non_404_http_exception:Function] -# #region test_get_repository_diff_propagates_http_exception [TYPE Function] -# @BRIEF Ensure diff endpoint preserves domain HTTP errors from GitService. -# @PRE GitService.get_diff raises HTTPException. -# @POST Endpoint raises same HTTPException values. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_diff_propagates_http_exception:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure diff endpoint preserves domain HTTP errors from GitService. +# @PRE: GitService.get_diff raises HTTPException. +# @POST: Endpoint raises same HTTPException values. def test_get_repository_diff_propagates_http_exception(monkeypatch): class DiffGitService: def get_diff(self, dashboard_id: int, file_path=None, staged: bool = False) -> str: @@ -76,14 +78,14 @@ def test_get_repository_diff_propagates_http_exception(monkeypatch): assert exc_info.value.status_code == 404 assert exc_info.value.detail == "Repository missing" -# #endregion test_get_repository_diff_propagates_http_exception +# [/DEF:test_get_repository_diff_propagates_http_exception:Function] -# #region test_get_history_wraps_unexpected_error_as_500 [TYPE Function] -# @BRIEF Ensure non-HTTP exceptions in history endpoint become deterministic 500 errors. -# @PRE GitService.get_commit_history raises ValueError. -# @POST Endpoint returns HTTPException with status 500 and route context. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_history_wraps_unexpected_error_as_500:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure non-HTTP exceptions in history endpoint become deterministic 500 errors. +# @PRE: GitService.get_commit_history raises ValueError. +# @POST: Endpoint returns HTTPException with status 500 and route context. def test_get_history_wraps_unexpected_error_as_500(monkeypatch): class HistoryGitService: def get_commit_history(self, dashboard_id: int, limit: int = 50): @@ -96,14 +98,14 @@ def test_get_history_wraps_unexpected_error_as_500(monkeypatch): assert exc_info.value.status_code == 500 assert exc_info.value.detail == "get_history failed: broken parser" -# #endregion test_get_history_wraps_unexpected_error_as_500 +# [/DEF:test_get_history_wraps_unexpected_error_as_500:Function] -# #region test_commit_changes_wraps_unexpected_error_as_500 [TYPE Function] -# @BRIEF Ensure commit endpoint does not leak unexpected errors as 400. -# @PRE GitService.commit_changes raises RuntimeError. -# @POST Endpoint raises HTTPException(500) with route context. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_commit_changes_wraps_unexpected_error_as_500:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure commit endpoint does not leak unexpected errors as 400. +# @PRE: GitService.commit_changes raises RuntimeError. +# @POST: Endpoint raises HTTPException(500) with route context. def test_commit_changes_wraps_unexpected_error_as_500(monkeypatch): class CommitGitService: def commit_changes(self, dashboard_id: int, message: str, files): @@ -120,14 +122,14 @@ def test_commit_changes_wraps_unexpected_error_as_500(monkeypatch): assert exc_info.value.status_code == 500 assert exc_info.value.detail == "commit_changes failed: index lock" -# #endregion test_commit_changes_wraps_unexpected_error_as_500 +# [/DEF:test_commit_changes_wraps_unexpected_error_as_500:Function] -# #region test_get_repository_status_batch_returns_mixed_statuses [TYPE Function] -# @BRIEF Ensure batch endpoint returns per-dashboard statuses in one response. -# @PRE Some repositories are missing and some are initialized. -# @POST Returned map includes resolved status for each requested dashboard ID. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_status_batch_returns_mixed_statuses:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure batch endpoint returns per-dashboard statuses in one response. +# @PRE: Some repositories are missing and some are initialized. +# @POST: Returned map includes resolved status for each requested dashboard ID. def test_get_repository_status_batch_returns_mixed_statuses(monkeypatch): class BatchGitService: def _get_repo_path(self, dashboard_id: int) -> str: @@ -148,14 +150,14 @@ def test_get_repository_status_batch_returns_mixed_statuses(monkeypatch): assert response.statuses["1"]["sync_status"] == "NO_REPO" assert response.statuses["2"]["sync_state"] == "SYNCED" -# #endregion test_get_repository_status_batch_returns_mixed_statuses +# [/DEF:test_get_repository_status_batch_returns_mixed_statuses:Function] -# #region test_get_repository_status_batch_marks_item_as_error_on_service_failure [TYPE Function] -# @BRIEF Ensure batch endpoint marks failed items as ERROR without failing entire request. -# @PRE GitService raises non-HTTP exception for one dashboard. -# @POST Failed dashboard status is marked as ERROR. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_status_batch_marks_item_as_error_on_service_failure:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure batch endpoint marks failed items as ERROR without failing entire request. +# @PRE: GitService raises non-HTTP exception for one dashboard. +# @POST: Failed dashboard status is marked as ERROR. def test_get_repository_status_batch_marks_item_as_error_on_service_failure(monkeypatch): class BatchErrorGitService: def _get_repo_path(self, dashboard_id: int) -> str: @@ -174,14 +176,14 @@ def test_get_repository_status_batch_marks_item_as_error_on_service_failure(monk assert response.statuses["9"]["sync_status"] == "ERROR" assert response.statuses["9"]["sync_state"] == "ERROR" -# #endregion test_get_repository_status_batch_marks_item_as_error_on_service_failure +# [/DEF:test_get_repository_status_batch_marks_item_as_error_on_service_failure:Function] -# #region test_get_repository_status_batch_deduplicates_and_truncates_ids [TYPE Function] -# @BRIEF Ensure batch endpoint protects server from oversized payloads. -# @PRE request includes duplicate IDs and more than MAX_REPOSITORY_STATUS_BATCH entries. -# @POST Result contains unique IDs up to configured cap. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_repository_status_batch_deduplicates_and_truncates_ids:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure batch endpoint protects server from oversized payloads. +# @PRE: request includes duplicate IDs and more than MAX_REPOSITORY_STATUS_BATCH entries. +# @POST: Result contains unique IDs up to configured cap. def test_get_repository_status_batch_deduplicates_and_truncates_ids(monkeypatch): class SafeBatchGitService: def _get_repo_path(self, dashboard_id: int) -> str: @@ -200,14 +202,14 @@ def test_get_repository_status_batch_deduplicates_and_truncates_ids(monkeypatch) assert len(response.statuses) == git_routes.MAX_REPOSITORY_STATUS_BATCH assert "1" in response.statuses -# #endregion test_get_repository_status_batch_deduplicates_and_truncates_ids +# [/DEF:test_get_repository_status_batch_deduplicates_and_truncates_ids:Function] -# #region test_commit_changes_applies_profile_identity_before_commit [TYPE Function] -# @BRIEF Ensure commit route configures repository identity from profile preferences before commit call. -# @PRE Profile preference contains git_username/git_email for current user. -# @POST git_service.configure_identity receives resolved identity and commit proceeds. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_commit_changes_applies_profile_identity_before_commit:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure commit route configures repository identity from profile preferences before commit call. +# @PRE: Profile preference contains git_username/git_email for current user. +# @POST: git_service.configure_identity receives resolved identity and commit proceeds. def test_commit_changes_applies_profile_identity_before_commit(monkeypatch): class IdentityGitService: def __init__(self): @@ -262,14 +264,14 @@ def test_commit_changes_applies_profile_identity_before_commit(monkeypatch): assert identity_service.configured_identity == (12, "user_1", "user1@mail.ru") assert identity_service.commit_payload == (12, "test", ["dashboards/a.yaml"]) -# #endregion test_commit_changes_applies_profile_identity_before_commit +# [/DEF:test_commit_changes_applies_profile_identity_before_commit:Function] -# #region test_pull_changes_applies_profile_identity_before_pull [TYPE Function] -# @BRIEF Ensure pull route configures repository identity from profile preferences before pull call. -# @PRE Profile preference contains git_username/git_email for current user. -# @POST git_service.configure_identity receives resolved identity and pull proceeds. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_pull_changes_applies_profile_identity_before_pull:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure pull route configures repository identity from profile preferences before pull call. +# @PRE: Profile preference contains git_username/git_email for current user. +# @POST: git_service.configure_identity receives resolved identity and pull proceeds. def test_pull_changes_applies_profile_identity_before_pull(monkeypatch): class IdentityGitService: def __init__(self): @@ -319,14 +321,14 @@ def test_pull_changes_applies_profile_identity_before_pull(monkeypatch): assert identity_service.configured_identity == (12, "user_1", "user1@mail.ru") assert identity_service.pulled_dashboard_id == 12 -# #endregion test_pull_changes_applies_profile_identity_before_pull +# [/DEF:test_pull_changes_applies_profile_identity_before_pull:Function] -# #region test_get_merge_status_returns_service_payload [TYPE Function] -# @BRIEF Ensure merge status route returns service payload as-is. -# @PRE git_service.get_merge_status returns unfinished merge payload. -# @POST Route response contains has_unfinished_merge=True. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_get_merge_status_returns_service_payload:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure merge status route returns service payload as-is. +# @PRE: git_service.get_merge_status returns unfinished merge payload. +# @POST: Route response contains has_unfinished_merge=True. def test_get_merge_status_returns_service_payload(monkeypatch): class MergeStatusGitService: def get_merge_status(self, dashboard_id: int) -> dict: @@ -352,14 +354,14 @@ def test_get_merge_status_returns_service_payload(monkeypatch): assert response["has_unfinished_merge"] is True assert response["conflicts_count"] == 2 -# #endregion test_get_merge_status_returns_service_payload +# [/DEF:test_get_merge_status_returns_service_payload:Function] -# #region test_resolve_merge_conflicts_passes_resolution_items_to_service [TYPE Function] -# @BRIEF Ensure merge resolve route forwards parsed resolutions to service. -# @PRE resolve_data has one file strategy. -# @POST Service receives normalized list and route returns resolved files. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_resolve_merge_conflicts_passes_resolution_items_to_service:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure merge resolve route forwards parsed resolutions to service. +# @PRE: resolve_data has one file strategy. +# @POST: Service receives normalized list and route returns resolved files. def test_resolve_merge_conflicts_passes_resolution_items_to_service(monkeypatch): captured = {} @@ -390,14 +392,14 @@ def test_resolve_merge_conflicts_passes_resolution_items_to_service(monkeypatch) assert captured["dashboard_id"] == 12 assert captured["resolutions"][0]["resolution"] == "mine" assert response["resolved_files"] == ["dashboards/a.yaml"] -# #endregion test_resolve_merge_conflicts_passes_resolution_items_to_service +# [/DEF:test_resolve_merge_conflicts_passes_resolution_items_to_service:Function] -# #region test_abort_merge_calls_service_and_returns_result [TYPE Function] -# @BRIEF Ensure abort route delegates to service. -# @PRE Service abort_merge returns aborted status. -# @POST Route returns aborted status. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_abort_merge_calls_service_and_returns_result:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure abort route delegates to service. +# @PRE: Service abort_merge returns aborted status. +# @POST: Route returns aborted status. def test_abort_merge_calls_service_and_returns_result(monkeypatch): class AbortGitService: def abort_merge(self, dashboard_id: int): @@ -415,14 +417,14 @@ def test_abort_merge_calls_service_and_returns_result(monkeypatch): ) assert response["status"] == "aborted" -# #endregion test_abort_merge_calls_service_and_returns_result +# [/DEF:test_abort_merge_calls_service_and_returns_result:Function] -# #region test_continue_merge_passes_message_and_returns_commit [TYPE Function] -# @BRIEF Ensure continue route passes commit message to service. -# @PRE continue_data.message is provided. -# @POST Route returns committed status and hash. -# @RELATION BINDS_TO -> [TestGitStatusRoute] +# [DEF:test_continue_merge_passes_message_and_returns_commit:Function] +# @RELATION: BINDS_TO -> TestGitStatusRoute +# @PURPOSE: Ensure continue route passes commit message to service. +# @PRE: continue_data.message is provided. +# @POST: Route returns committed status and hash. def test_continue_merge_passes_message_and_returns_commit(monkeypatch): class ContinueGitService: def continue_merge(self, dashboard_id: int, message: str): @@ -446,7 +448,7 @@ def test_continue_merge_passes_message_and_returns_commit(monkeypatch): assert response["status"] == "committed" assert response["commit_hash"] == "abc123" -# #endregion test_continue_merge_passes_message_and_returns_commit +# [/DEF:test_continue_merge_passes_message_and_returns_commit:Function] -# #endregion TestGitStatusRoute +# [/DEF:TestGitStatusRoute:Module] diff --git a/backend/src/api/routes/__tests__/test_migration_routes.py b/backend/src/api/routes/__tests__/test_migration_routes.py index e20709a6..37473204 100644 --- a/backend/src/api/routes/__tests__/test_migration_routes.py +++ b/backend/src/api/routes/__tests__/test_migration_routes.py @@ -1,8 +1,9 @@ -# #region TestMigrationRoutes [C:3] [TYPE Module] -# @BRIEF Unit tests for migration API route handlers. -# @LAYER API -# @RELATION VERIFIES -> [backend.src.api.routes.migration] +# [DEF:TestMigrationRoutes:Module] # +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for migration API route handlers. +# @LAYER: API +# @RELATION: VERIFIES -> backend.src.api.routes.migration # import pytest import sys @@ -59,8 +60,8 @@ def db_session(): session.close() -# #region _make_config_manager [TYPE Function] -# @RELATION BINDS_TO -> [TestMigrationRoutes] +# [DEF:_make_config_manager:Function] +# @RELATION: BINDS_TO -> TestMigrationRoutes def _make_config_manager(cron="0 2 * * *"): """Creates a mock config manager with a realistic AppConfig-like object.""" settings = MagicMock() @@ -75,7 +76,7 @@ def _make_config_manager(cron="0 2 * * *"): # --- get_migration_settings tests --- -# #endregion _make_config_manager +# [/DEF:_make_config_manager:Function] @pytest.mark.asyncio @@ -328,8 +329,8 @@ async def test_get_resource_mappings_filter_by_type(db_session): @pytest.fixture -# #region _mock_env [TYPE Function] -# @RELATION BINDS_TO -> [TestMigrationRoutes] +# [DEF:_mock_env:Function] +# @RELATION: BINDS_TO -> TestMigrationRoutes def _mock_env(): """Creates a mock config environment object.""" env = MagicMock() @@ -343,11 +344,11 @@ def _mock_env(): return env -# #endregion _mock_env +# [/DEF:_mock_env:Function] -# #region _make_sync_config_manager [TYPE Function] -# @RELATION BINDS_TO -> [TestMigrationRoutes] +# [DEF:_make_sync_config_manager:Function] +# @RELATION: BINDS_TO -> TestMigrationRoutes def _make_sync_config_manager(environments): """Creates a mock config manager with environments list.""" settings = MagicMock() @@ -361,7 +362,7 @@ def _make_sync_config_manager(environments): return cm -# #endregion _make_sync_config_manager +# [/DEF:_make_sync_config_manager:Function] @pytest.mark.asyncio @@ -652,4 +653,4 @@ async def test_dry_run_migration_rejects_same_environment(db_session): assert exc.value.status_code == 400 -# #endregion TestMigrationRoutes +# [/DEF:TestMigrationRoutes:Module] diff --git a/backend/src/api/routes/__tests__/test_profile_api.py b/backend/src/api/routes/__tests__/test_profile_api.py index 4c8bdd35..4a3500de 100644 --- a/backend/src/api/routes/__tests__/test_profile_api.py +++ b/backend/src/api/routes/__tests__/test_profile_api.py @@ -1,7 +1,9 @@ -# #region TestProfileApi [C:3] [TYPE Module] [SEMANTICS tests, profile, api, preferences, lookup, contract] -# @BRIEF Verifies profile API route contracts for preference read/update and Superset account lookup. -# @LAYER API -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestProfileApi:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, profile, api, preferences, lookup, contract +# @PURPOSE: Verifies profile API route contracts for preference read/update and Superset account lookup. +# @LAYER: API # [SECTION: IMPORTS] from datetime import datetime, timezone @@ -31,11 +33,11 @@ from src.services.profile_service import ( client = TestClient(app) -# #region mock_profile_route_dependencies [TYPE Function] -# @BRIEF Provides deterministic dependency overrides for profile route tests. -# @PRE App instance is initialized. -# @POST Dependencies are overridden for current test and restored afterward. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:mock_profile_route_dependencies:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Provides deterministic dependency overrides for profile route tests. +# @PRE: App instance is initialized. +# @POST: Dependencies are overridden for current test and restored afterward. def mock_profile_route_dependencies(): mock_user = MagicMock() mock_user.id = "u-1" @@ -49,14 +51,14 @@ def mock_profile_route_dependencies(): app.dependency_overrides[get_config_manager] = lambda: mock_config_manager return mock_user, mock_db, mock_config_manager -# #endregion mock_profile_route_dependencies +# [/DEF:mock_profile_route_dependencies:Function] -# #region profile_route_deps_fixture [TYPE Function] -# @BRIEF Pytest fixture wrapper for profile route dependency overrides. -# @PRE None. -# @POST Yields overridden dependencies and clears overrides after test. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:profile_route_deps_fixture:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Pytest fixture wrapper for profile route dependency overrides. +# @PRE: None. +# @POST: Yields overridden dependencies and clears overrides after test. import pytest @@ -65,14 +67,14 @@ def profile_route_deps_fixture(): yielded = mock_profile_route_dependencies() yield yielded app.dependency_overrides.clear() -# #endregion profile_route_deps_fixture +# [/DEF:profile_route_deps_fixture:Function] -# #region _build_preference_response [TYPE Function] -# @BRIEF Builds stable profile preference response payload for route tests. -# @PRE user_id is provided. -# @POST Returns ProfilePreferenceResponse object with deterministic timestamps. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:_build_preference_response:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Builds stable profile preference response payload for route tests. +# @PRE: user_id is provided. +# @POST: Returns ProfilePreferenceResponse object with deterministic timestamps. def _build_preference_response(user_id: str = "u-1") -> ProfilePreferenceResponse: now = datetime.now(timezone.utc) return ProfilePreferenceResponse( @@ -106,14 +108,14 @@ def _build_preference_response(user_id: str = "u-1") -> ProfilePreferenceRespons ], ), ) -# #endregion _build_preference_response +# [/DEF:_build_preference_response:Function] -# #region test_get_profile_preferences_returns_self_payload [TYPE Function] -# @BRIEF Verifies GET /api/profile/preferences returns stable self-scoped payload. -# @PRE Authenticated user context is available. -# @POST Response status is 200 and payload contains current user preference. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_get_profile_preferences_returns_self_payload:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies GET /api/profile/preferences returns stable self-scoped payload. +# @PRE: Authenticated user context is available. +# @POST: Response status is 200 and payload contains current user preference. def test_get_profile_preferences_returns_self_payload(profile_route_deps_fixture): mock_user, _, _ = profile_route_deps_fixture service = MagicMock() @@ -139,14 +141,14 @@ def test_get_profile_preferences_returns_self_payload(profile_route_deps_fixture assert payload["security"]["current_role"] == "Data Engineer" assert payload["security"]["permissions"][0]["key"] == "migration:run" service.get_my_preference.assert_called_once_with(mock_user) -# #endregion test_get_profile_preferences_returns_self_payload +# [/DEF:test_get_profile_preferences_returns_self_payload:Function] -# #region test_patch_profile_preferences_success [TYPE Function] -# @BRIEF Verifies PATCH /api/profile/preferences persists valid payload through route mapping. -# @PRE Valid request payload and authenticated user. -# @POST Response status is 200 with saved preference payload. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_patch_profile_preferences_success:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies PATCH /api/profile/preferences persists valid payload through route mapping. +# @PRE: Valid request payload and authenticated user. +# @POST: Response status is 200 with saved preference payload. def test_patch_profile_preferences_success(profile_route_deps_fixture): mock_user, _, _ = profile_route_deps_fixture service = MagicMock() @@ -190,14 +192,14 @@ def test_patch_profile_preferences_success(profile_route_deps_fixture): assert called_kwargs["payload"].start_page == "reports-logs" assert called_kwargs["payload"].auto_open_task_drawer is False assert called_kwargs["payload"].dashboards_table_density == "free" -# #endregion test_patch_profile_preferences_success +# [/DEF:test_patch_profile_preferences_success:Function] -# #region test_patch_profile_preferences_validation_error [TYPE Function] -# @BRIEF Verifies route maps domain validation failure to HTTP 422 with actionable details. -# @PRE Service raises ProfileValidationError. -# @POST Response status is 422 and includes validation messages. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_patch_profile_preferences_validation_error:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies route maps domain validation failure to HTTP 422 with actionable details. +# @PRE: Service raises ProfileValidationError. +# @POST: Response status is 422 and includes validation messages. def test_patch_profile_preferences_validation_error(profile_route_deps_fixture): service = MagicMock() service.update_my_preference.side_effect = ProfileValidationError( @@ -217,14 +219,14 @@ def test_patch_profile_preferences_validation_error(profile_route_deps_fixture): payload = response.json() assert "detail" in payload assert "Superset username is required when default filter is enabled." in payload["detail"] -# #endregion test_patch_profile_preferences_validation_error +# [/DEF:test_patch_profile_preferences_validation_error:Function] -# #region test_patch_profile_preferences_cross_user_denied [TYPE Function] -# @BRIEF Verifies route maps domain authorization guard failure to HTTP 403. -# @PRE Service raises ProfileAuthorizationError. -# @POST Response status is 403 with denial message. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_patch_profile_preferences_cross_user_denied:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies route maps domain authorization guard failure to HTTP 403. +# @PRE: Service raises ProfileAuthorizationError. +# @POST: Response status is 403 with denial message. def test_patch_profile_preferences_cross_user_denied(profile_route_deps_fixture): service = MagicMock() service.update_my_preference.side_effect = ProfileAuthorizationError( @@ -243,14 +245,14 @@ def test_patch_profile_preferences_cross_user_denied(profile_route_deps_fixture) assert response.status_code == 403 payload = response.json() assert payload["detail"] == "Cross-user preference mutation is forbidden" -# #endregion test_patch_profile_preferences_cross_user_denied +# [/DEF:test_patch_profile_preferences_cross_user_denied:Function] -# #region test_lookup_superset_accounts_success [TYPE Function] -# @BRIEF Verifies lookup route returns success payload with normalized candidates. -# @PRE Valid environment_id and service success response. -# @POST Response status is 200 and items list is returned. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_lookup_superset_accounts_success:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies lookup route returns success payload with normalized candidates. +# @PRE: Valid environment_id and service success response. +# @POST: Response status is 200 and items list is returned. def test_lookup_superset_accounts_success(profile_route_deps_fixture): service = MagicMock() service.lookup_superset_accounts.return_value = SupersetAccountLookupResponse( @@ -280,14 +282,14 @@ def test_lookup_superset_accounts_success(profile_route_deps_fixture): assert payload["environment_id"] == "dev" assert payload["total"] == 1 assert payload["items"][0]["username"] == "john_doe" -# #endregion test_lookup_superset_accounts_success +# [/DEF:test_lookup_superset_accounts_success:Function] -# #region test_lookup_superset_accounts_env_not_found [TYPE Function] -# @BRIEF Verifies lookup route maps missing environment to HTTP 404. -# @PRE Service raises EnvironmentNotFoundError. -# @POST Response status is 404 with explicit message. -# @RELATION BINDS_TO -> [TestProfileApi] +# [DEF:test_lookup_superset_accounts_env_not_found:Function] +# @RELATION: BINDS_TO -> TestProfileApi +# @PURPOSE: Verifies lookup route maps missing environment to HTTP 404. +# @PRE: Service raises EnvironmentNotFoundError. +# @POST: Response status is 404 with explicit message. def test_lookup_superset_accounts_env_not_found(profile_route_deps_fixture): service = MagicMock() service.lookup_superset_accounts.side_effect = EnvironmentNotFoundError( @@ -300,6 +302,6 @@ def test_lookup_superset_accounts_env_not_found(profile_route_deps_fixture): assert response.status_code == 404 payload = response.json() assert payload["detail"] == "Environment 'missing-env' not found" -# #endregion test_lookup_superset_accounts_env_not_found +# [/DEF:test_lookup_superset_accounts_env_not_found:Function] -# #endregion TestProfileApi +# [/DEF:TestProfileApi:Module] diff --git a/backend/src/api/routes/__tests__/test_reports_api.py b/backend/src/api/routes/__tests__/test_reports_api.py index 512421d0..d2ebe139 100644 --- a/backend/src/api/routes/__tests__/test_reports_api.py +++ b/backend/src/api/routes/__tests__/test_reports_api.py @@ -1,8 +1,10 @@ -# #region TestReportsApi [C:3] [TYPE Module] [SEMANTICS tests, reports, api, contract, pagination, filtering] -# @BRIEF Contract tests for GET /api/reports defaults, pagination, and filtering behavior. -# @LAYER Domain (Tests) -# @INVARIANT API response contract contains {items,total,page,page_size,has_next,applied_filters}. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestReportsApi:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, reports, api, contract, pagination, filtering +# @PURPOSE: Contract tests for GET /api/reports defaults, pagination, and filtering behavior. +# @LAYER: Domain (Tests) +# @INVARIANT: API response contract contains {items,total,page,page_size,has_next,applied_filters}. from datetime import datetime, timedelta, timezone from types import SimpleNamespace @@ -15,10 +17,11 @@ from src.dependencies import get_current_user, get_task_manager # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). -# #region _FakeTaskManager [C:1] [TYPE Class] -# @BRIEF Minimal task-manager double exposing only get_all_tasks used by reports route tests. -# @INVARIANT Returns pre-seeded tasks without mutation or side effects. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsApi] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager double exposing only get_all_tasks used by reports route tests. +# @INVARIANT: Returns pre-seeded tasks without mutation or side effects. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -27,23 +30,23 @@ class _FakeTaskManager: return self._tasks -# #endregion _FakeTaskManager +# [/DEF:_FakeTaskManager:Class] -# #region _admin_user [TYPE Function] -# @BRIEF Build deterministic admin principal accepted by reports authorization guard. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:_admin_user:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Build deterministic admin principal accepted by reports authorization guard. def _admin_user(): admin_role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[admin_role]) -# #endregion _admin_user +# [/DEF:_admin_user:Function] -# #region _make_task [TYPE Function] -# @BRIEF Build Task fixture with controlled timestamps/status for reports list/detail normalization. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:_make_task:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Build Task fixture with controlled timestamps/status for reports list/detail normalization. def _make_task( task_id: str, plugin_id: str, @@ -63,12 +66,12 @@ def _make_task( ) -# #endregion _make_task +# [/DEF:_make_task:Function] -# #region test_get_reports_default_pagination_contract [TYPE Function] -# @BRIEF Validate reports list endpoint default pagination and contract keys for mixed task statuses. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:test_get_reports_default_pagination_contract:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint default pagination and contract keys for mixed task statuses. def test_get_reports_default_pagination_contract(): now = datetime.utcnow() tasks = [ @@ -117,12 +120,12 @@ def test_get_reports_default_pagination_contract(): app.dependency_overrides.clear() -# #endregion test_get_reports_default_pagination_contract +# [/DEF:test_get_reports_default_pagination_contract:Function] -# #region test_get_reports_filter_and_pagination [TYPE Function] -# @BRIEF Validate reports list endpoint applies task-type/status filters and pagination boundaries. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:test_get_reports_filter_and_pagination:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint applies task-type/status filters and pagination boundaries. def test_get_reports_filter_and_pagination(): now = datetime.utcnow() tasks = [ @@ -171,12 +174,12 @@ def test_get_reports_filter_and_pagination(): app.dependency_overrides.clear() -# #endregion test_get_reports_filter_and_pagination +# [/DEF:test_get_reports_filter_and_pagination:Function] -# #region test_get_reports_handles_mixed_naive_and_aware_datetimes [TYPE Function] -# @BRIEF Validate reports sorting remains stable when task timestamps mix naive and timezone-aware datetimes. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:test_get_reports_handles_mixed_naive_and_aware_datetimes:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports sorting remains stable when task timestamps mix naive and timezone-aware datetimes. def test_get_reports_handles_mixed_naive_and_aware_datetimes(): naive_now = datetime.utcnow() aware_now = datetime.now(timezone.utc) @@ -211,12 +214,12 @@ def test_get_reports_handles_mixed_naive_and_aware_datetimes(): app.dependency_overrides.clear() -# #endregion test_get_reports_handles_mixed_naive_and_aware_datetimes +# [/DEF:test_get_reports_handles_mixed_naive_and_aware_datetimes:Function] -# #region test_get_reports_invalid_filter_returns_400 [TYPE Function] -# @BRIEF Validate reports list endpoint rejects unsupported task type filters with HTTP 400. -# @RELATION BINDS_TO -> [TestReportsApi] +# [DEF:test_get_reports_invalid_filter_returns_400:Function] +# @RELATION: BINDS_TO -> TestReportsApi +# @PURPOSE: Validate reports list endpoint rejects unsupported task type filters with HTTP 400. def test_get_reports_invalid_filter_returns_400(): now = datetime.utcnow() tasks = [ @@ -242,5 +245,5 @@ def test_get_reports_invalid_filter_returns_400(): app.dependency_overrides.clear() -# #endregion test_get_reports_invalid_filter_returns_400 -# #endregion TestReportsApi +# [/DEF:test_get_reports_invalid_filter_returns_400:Function] +# [/DEF:TestReportsApi:Module] diff --git a/backend/src/api/routes/__tests__/test_reports_detail_api.py b/backend/src/api/routes/__tests__/test_reports_detail_api.py index ad8b1a17..5606de4c 100644 --- a/backend/src/api/routes/__tests__/test_reports_detail_api.py +++ b/backend/src/api/routes/__tests__/test_reports_detail_api.py @@ -1,8 +1,10 @@ -# #region TestReportsDetailApi [C:3] [TYPE Module] [SEMANTICS tests, reports, api, detail, diagnostics] -# @BRIEF Contract tests for GET /api/reports/{report_id} detail endpoint behavior. -# @LAYER Domain (Tests) -# @INVARIANT Detail endpoint tests must keep deterministic assertions for success and not-found contracts. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestReportsDetailApi:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, reports, api, detail, diagnostics +# @PURPOSE: Contract tests for GET /api/reports/{report_id} detail endpoint behavior. +# @LAYER: Domain (Tests) +# @INVARIANT: Detail endpoint tests must keep deterministic assertions for success and not-found contracts. from datetime import datetime, timedelta from types import SimpleNamespace @@ -15,10 +17,11 @@ from src.dependencies import get_current_user, get_task_manager # @DEBT: Divergent _FakeTaskManager definition. Canonical version should be in conftest.py. Authz variant is missing get_all_tasks(). -# #region _FakeTaskManager [C:1] [TYPE Class] -# @BRIEF Minimal task-manager double exposing pre-seeded tasks to detail endpoint under test. -# @INVARIANT get_all_tasks returns exactly seeded tasks list. -# @RELATION BINDS_TO -> [TestReportsDetailApi] +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsDetailApi] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager double exposing pre-seeded tasks to detail endpoint under test. +# @INVARIANT: get_all_tasks returns exactly seeded tasks list. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -27,23 +30,23 @@ class _FakeTaskManager: return self._tasks -# #endregion _FakeTaskManager +# [/DEF:_FakeTaskManager:Class] -# #region _admin_user [TYPE Function] -# @BRIEF Provide admin principal fixture accepted by reports detail authorization policy. -# @RELATION BINDS_TO -> [TestReportsDetailApi] +# [DEF:_admin_user:Function] +# @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Provide admin principal fixture accepted by reports detail authorization policy. def _admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[role]) -# #endregion _admin_user +# [/DEF:_admin_user:Function] -# #region _make_task [TYPE Function] -# @BRIEF Build deterministic Task payload for reports detail endpoint contract assertions. -# @RELATION BINDS_TO -> [TestReportsDetailApi] +# [DEF:_make_task:Function] +# @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Build deterministic Task payload for reports detail endpoint contract assertions. def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None): now = datetime.utcnow() return Task( @@ -59,12 +62,12 @@ def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None): ) -# #endregion _make_task +# [/DEF:_make_task:Function] -# #region test_get_report_detail_success [TYPE Function] -# @BRIEF Validate report detail endpoint returns report body with diagnostics and next actions for existing task. -# @RELATION BINDS_TO -> [TestReportsDetailApi] +# [DEF:test_get_report_detail_success:Function] +# @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Validate report detail endpoint returns report body with diagnostics and next actions for existing task. def test_get_report_detail_success(): task = _make_task( "detail-1", @@ -95,12 +98,12 @@ def test_get_report_detail_success(): app.dependency_overrides.clear() -# #endregion test_get_report_detail_success +# [/DEF:test_get_report_detail_success:Function] -# #region test_get_report_detail_not_found [TYPE Function] -# @BRIEF Validate report detail endpoint returns 404 when requested report identifier is absent. -# @RELATION BINDS_TO -> [TestReportsDetailApi] +# [DEF:test_get_report_detail_not_found:Function] +# @RELATION: BINDS_TO -> TestReportsDetailApi +# @PURPOSE: Validate report detail endpoint returns 404 when requested report identifier is absent. def test_get_report_detail_not_found(): task = _make_task("detail-2", "superset-backup", TaskStatus.SUCCESS) @@ -115,5 +118,5 @@ def test_get_report_detail_not_found(): app.dependency_overrides.clear() -# #endregion test_get_report_detail_not_found -# #endregion TestReportsDetailApi +# [/DEF:test_get_report_detail_not_found:Function] +# [/DEF:TestReportsDetailApi:Module] diff --git a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py index 63f36598..0b425c0a 100644 --- a/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py +++ b/backend/src/api/routes/__tests__/test_reports_openapi_conformance.py @@ -1,8 +1,10 @@ -# #region TestReportsOpenapiConformance [C:3] [TYPE Module] [SEMANTICS tests, reports, openapi, conformance] -# @BRIEF Validate implemented reports payload shape against OpenAPI-required top-level contract fields. -# @LAYER Domain (Tests) -# @INVARIANT List and detail payloads include required contract keys. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestReportsOpenapiConformance:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, reports, openapi, conformance +# @PURPOSE: Validate implemented reports payload shape against OpenAPI-required top-level contract fields. +# @LAYER: Domain (Tests) +# @INVARIANT: List and detail payloads include required contract keys. from datetime import datetime from types import SimpleNamespace @@ -14,10 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus from src.dependencies import get_current_user, get_task_manager -# #region _FakeTaskManager [C:1] [TYPE Class] -# @BRIEF Minimal task-manager fake exposing static task list for OpenAPI conformance checks. -# @INVARIANT get_all_tasks returns seeded tasks unchanged. -# @RELATION BINDS_TO -> [TestReportsOpenapiConformance] +# [DEF:_FakeTaskManager:Class] +# @RELATION: BINDS_TO -> [TestReportsOpenapiConformance] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal task-manager fake exposing static task list for OpenAPI conformance checks. +# @INVARIANT: get_all_tasks returns seeded tasks unchanged. class _FakeTaskManager: def __init__(self, tasks): self._tasks = tasks @@ -26,23 +29,23 @@ class _FakeTaskManager: return self._tasks -# #endregion _FakeTaskManager +# [/DEF:_FakeTaskManager:Class] -# #region _admin_user [TYPE Function] -# @BRIEF Provide admin principal fixture required by reports routes in conformance tests. -# @RELATION BINDS_TO -> [TestReportsOpenapiConformance] +# [DEF:_admin_user:Function] +# @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Provide admin principal fixture required by reports routes in conformance tests. def _admin_user(): role = SimpleNamespace(name="Admin", permissions=[]) return SimpleNamespace(username="test-admin", roles=[role]) -# #endregion _admin_user +# [/DEF:_admin_user:Function] -# #region _task [TYPE Function] -# @BRIEF Construct deterministic task fixture consumed by reports list/detail payload assertions. -# @RELATION BINDS_TO -> [TestReportsOpenapiConformance] +# [DEF:_task:Function] +# @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Construct deterministic task fixture consumed by reports list/detail payload assertions. def _task(task_id: str, plugin_id: str, status: TaskStatus): now = datetime.utcnow() return Task( @@ -56,12 +59,12 @@ def _task(task_id: str, plugin_id: str, status: TaskStatus): ) -# #endregion _task +# [/DEF:_task:Function] -# #region test_reports_list_openapi_required_keys [TYPE Function] -# @BRIEF Verify reports list endpoint includes all required OpenAPI top-level keys. -# @RELATION BINDS_TO -> [TestReportsOpenapiConformance] +# [DEF:test_reports_list_openapi_required_keys:Function] +# @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Verify reports list endpoint includes all required OpenAPI top-level keys. def test_reports_list_openapi_required_keys(): tasks = [ _task("r-1", "superset-backup", TaskStatus.SUCCESS), @@ -89,12 +92,12 @@ def test_reports_list_openapi_required_keys(): app.dependency_overrides.clear() -# #endregion test_reports_list_openapi_required_keys +# [/DEF:test_reports_list_openapi_required_keys:Function] -# #region test_reports_detail_openapi_required_keys [TYPE Function] -# @BRIEF Verify reports detail endpoint returns payload containing the report object key. -# @RELATION BINDS_TO -> [TestReportsOpenapiConformance] +# [DEF:test_reports_detail_openapi_required_keys:Function] +# @RELATION: BINDS_TO -> TestReportsOpenapiConformance +# @PURPOSE: Verify reports detail endpoint returns payload containing the report object key. def test_reports_detail_openapi_required_keys(): tasks = [_task("r-3", "llm_dashboard_validation", TaskStatus.SUCCESS)] app.dependency_overrides[get_current_user] = lambda: _admin_user() @@ -111,5 +114,5 @@ def test_reports_detail_openapi_required_keys(): app.dependency_overrides.clear() -# #endregion test_reports_detail_openapi_required_keys -# #endregion TestReportsOpenapiConformance +# [/DEF:test_reports_detail_openapi_required_keys:Function] +# [/DEF:TestReportsOpenapiConformance:Module] diff --git a/backend/src/api/routes/__tests__/test_tasks_logs.py b/backend/src/api/routes/__tests__/test_tasks_logs.py index 827ba181..dd99fda0 100644 --- a/backend/src/api/routes/__tests__/test_tasks_logs.py +++ b/backend/src/api/routes/__tests__/test_tasks_logs.py @@ -1,7 +1,9 @@ -# #region test_tasks_logs_module [C:2] [TYPE Module] [SEMANTICS tests, tasks, logs, api, contract, validation] -# @BRIEF Contract testing for task logs API endpoints. -# @LAYER Domain (Tests) -# @RELATION VERIFIES -> [src.api.routes.tasks:Module] +# [DEF:test_tasks_logs_module:Module] +# @RELATION: VERIFIES -> [src.api.routes.tasks:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: tests, tasks, logs, api, contract, validation +# @PURPOSE: Contract testing for task logs API endpoints. +# @LAYER: Domain (Tests) import pytest from fastapi import FastAPI @@ -30,9 +32,9 @@ def client(): # @TEST_CONTRACT: get_task_logs_api -> Invariants # @TEST_FIXTURE: valid_task_logs_request -# #region test_get_task_logs_success [TYPE Function] -# @BRIEF Validate task logs endpoint returns filtered logs for an existing task. -# @RELATION BINDS_TO -> [test_tasks_logs_module] +# [DEF:test_get_task_logs_success:Function] +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint returns filtered logs for an existing task. def test_get_task_logs_success(client): tc, tm = client @@ -53,12 +55,12 @@ def test_get_task_logs_success(client): # @TEST_EDGE: task_not_found -# #endregion test_get_task_logs_success +# [/DEF:test_get_task_logs_success:Function] -# #region test_get_task_logs_not_found [TYPE Function] -# @BRIEF Validate task logs endpoint returns 404 when the task identifier is missing. -# @RELATION BINDS_TO -> [test_tasks_logs_module] +# [DEF:test_get_task_logs_not_found:Function] +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint returns 404 when the task identifier is missing. def test_get_task_logs_not_found(client): tc, tm = client tm.get_task.return_value = None @@ -69,12 +71,12 @@ def test_get_task_logs_not_found(client): # @TEST_EDGE: invalid_limit -# #endregion test_get_task_logs_not_found +# [/DEF:test_get_task_logs_not_found:Function] -# #region test_get_task_logs_invalid_limit [TYPE Function] -# @BRIEF Validate task logs endpoint enforces query validation for limit lower bound. -# @RELATION BINDS_TO -> [test_tasks_logs_module] +# [DEF:test_get_task_logs_invalid_limit:Function] +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate task logs endpoint enforces query validation for limit lower bound. def test_get_task_logs_invalid_limit(client): tc, tm = client # limit=0 is ge=1 in Query @@ -83,12 +85,12 @@ def test_get_task_logs_invalid_limit(client): # @TEST_INVARIANT: response_purity -# #endregion test_get_task_logs_invalid_limit +# [/DEF:test_get_task_logs_invalid_limit:Function] -# #region test_get_task_log_stats_success [TYPE Function] -# @BRIEF Validate log stats endpoint returns success payload for an existing task. -# @RELATION BINDS_TO -> [test_tasks_logs_module] +# [DEF:test_get_task_log_stats_success:Function] +# @RELATION: BINDS_TO -> test_tasks_logs_module +# @PURPOSE: Validate log stats endpoint returns success payload for an existing task. def test_get_task_log_stats_success(client): tc, tm = client tm.get_task.return_value = MagicMock() @@ -102,5 +104,5 @@ def test_get_task_log_stats_success(client): # assuming tm.get_task_log_stats returns something compatible with LogStats -# #endregion test_get_task_log_stats_success -# #endregion test_tasks_logs_module +# [/DEF:test_get_task_log_stats_success:Function] +# [/DEF:test_tasks_logs_module:Module] diff --git a/backend/src/api/routes/admin.py b/backend/src/api/routes/admin.py index 0d95ceee..baaa8f78 100644 --- a/backend/src/api/routes/admin.py +++ b/backend/src/api/routes/admin.py @@ -1,14 +1,13 @@ # #region AdminApi [C:3] [TYPE Module] [SEMANTICS api, admin, users, roles, permissions] +# # @BRIEF Admin API endpoints for user and role management. -# @LAYER API -# @INVARIANT All endpoints in this module require 'Admin' role or 'admin' scope. +# @LAYER: API # @RELATION DEPENDS_ON -> [AuthRepository:Class] # @RELATION DEPENDS_ON -> [get_auth_db:Function] # @RELATION DEPENDS_ON -> [has_permission:Function] # -# +# @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope. -# [SECTION: IMPORTS] from typing import List from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session @@ -28,30 +27,24 @@ from ...schemas.auth import ( ) from ...models.auth import User, Role, ADGroupMapping from ...dependencies import has_permission, get_plugin_loader -from ...core.cot_logger import MarkerLogger from ...core.logger import logger, belief_scope - -log = MarkerLogger("AdminApi") from ...services.rbac_permission_catalog import ( discover_declared_permissions, sync_permission_catalog, ) -# [/SECTION] # #region router [TYPE Variable] +# @RELATION DEPENDS_ON -> fastapi.APIRouter # @BRIEF APIRouter instance for admin routes. -# @RELATION DEPENDS_ON -> [fastapi.APIRouter] router = APIRouter(prefix="/api/admin", tags=["admin"]) # #endregion router # #region list_users [C:3] [TYPE Function] # @BRIEF Lists all registered users. -# @PRE Current user has 'Admin' role. -# @POST Returns a list of UserSchema objects. -# @PARAM: db (Session) - Auth database session. -# @RETURN: List[UserSchema] - List of users. -# @RELATION: CALLS -> User +# @PRE: Current user has 'Admin' role. +# @POST: Returns a list of UserSchema objects. +# @RELATION CALLS -> User @router.get("/users", response_model=List[UserSchema]) async def list_users( db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:users", "READ")) @@ -66,12 +59,9 @@ async def list_users( # #region create_user [C:3] [TYPE Function] # @BRIEF Creates a new local user. -# @PRE Current user has 'Admin' role. -# @POST New user is created in the database. -# @PARAM: user_in (UserCreate) - New user data. -# @PARAM: db (Session) - Auth database session. -# @RETURN: UserSchema - The created user. -# @RELATION: [CALLS] ->[AuthRepository:Class] +# @PRE: Current user has 'Admin' role. +# @POST: New user is created in the database. +# @RELATION CALLS -> [AuthRepository:Class] @router.post("/users", response_model=UserSchema, status_code=status.HTTP_201_CREATED) async def create_user( user_in: UserCreate, @@ -107,13 +97,9 @@ async def create_user( # #region update_user [C:3] [TYPE Function] # @BRIEF Updates an existing user. -# @PRE Current user has 'Admin' role. -# @POST User record is updated in the database. -# @PARAM: user_id (str) - Target user UUID. -# @PARAM: user_in (UserUpdate) - Updated user data. -# @PARAM: db (Session) - Auth database session. -# @RETURN: UserSchema - The updated user profile. -# @RELATION: CALLS -> AuthRepository +# @PRE: Current user has 'Admin' role. +# @POST: User record is updated in the database. +# @RELATION CALLS -> AuthRepository @router.put("/users/{user_id}", response_model=UserSchema) async def update_user( user_id: str, @@ -151,12 +137,9 @@ async def update_user( # #region delete_user [C:3] [TYPE Function] # @BRIEF Deletes a user. -# @PRE Current user has 'Admin' role. -# @POST User record is removed from the database. -# @PARAM: user_id (str) - Target user UUID. -# @PARAM: db (Session) - Auth database session. -# @RETURN: None -# @RELATION: CALLS -> AuthRepository +# @PRE: Current user has 'Admin' role. +# @POST: User record is removed from the database. +# @RELATION CALLS -> AuthRepository @router.delete("/users/{user_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_user( user_id: str, @@ -164,17 +147,25 @@ async def delete_user( _=Depends(has_permission("admin:users", "WRITE")), ): with belief_scope("api.admin.delete_user"): - log.reason(f"Attempting to delete user {user_id}") + logger.info( + f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}" + ) repo = AuthRepository(db) user = repo.get_user_by_id(user_id) if not user: - log.explore(f"User not found for deletion: {user_id}", error="User not found") + logger.warning( + f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}" + ) raise HTTPException(status_code=404, detail="User not found") - log.reason(f"Found user to delete: {user.username}") + logger.info( + f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}" + ) db.delete(user) db.commit() - log.reflect(f"Successfully deleted user {user_id}") + logger.info( + f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}" + ) return None @@ -183,7 +174,6 @@ async def delete_user( # #region list_roles [C:3] [TYPE Function] # @BRIEF Lists all available roles. -# @RETURN List[RoleSchema] - List of roles. # @RELATION CALLS -> [Role:Class] @router.get("/roles", response_model=List[RoleSchema]) async def list_roles( @@ -198,13 +188,10 @@ async def list_roles( # #region create_role [C:3] [TYPE Function] # @BRIEF Creates a new system role with associated permissions. -# @PRE Role name must be unique. -# @POST New Role record is created in auth.db. -# @PARAM: role_in (RoleCreate) - New role data. -# @PARAM: db (Session) - Auth database session. -# @RETURN: RoleSchema - The created role. +# @PRE: Role name must be unique. +# @POST: New Role record is created in auth.db. # @SIDE_EFFECT: Commits new role and associations to auth.db. -# @RELATION: [CALLS] ->[get_permission_by_id:Function] +# @RELATION CALLS -> [get_permission_by_id:Function] @router.post("/roles", response_model=RoleSchema, status_code=status.HTTP_201_CREATED) async def create_role( role_in: RoleCreate, @@ -238,14 +225,10 @@ async def create_role( # #region update_role [C:3] [TYPE Function] # @BRIEF Updates an existing role's metadata and permissions. -# @PRE role_id must be a valid existing role UUID. -# @POST Role record is updated in auth.db. -# @PARAM: role_id (str) - Target role identifier. -# @PARAM: role_in (RoleUpdate) - Updated role data. -# @PARAM: db (Session) - Auth database session. -# @RETURN: RoleSchema - The updated role. +# @PRE: role_id must be a valid existing role UUID. +# @POST: Role record is updated in auth.db. # @SIDE_EFFECT: Commits updates to auth.db. -# @RELATION: [CALLS] ->[get_role_by_id:Function] +# @RELATION CALLS -> [get_role_by_id:Function] @router.put("/roles/{role_id}", response_model=RoleSchema) async def update_role( role_id: str, @@ -285,13 +268,10 @@ async def update_role( # #region delete_role [C:3] [TYPE Function] # @BRIEF Removes a role from the system. -# @PRE role_id must be a valid existing role UUID. -# @POST Role record is removed from auth.db. -# @PARAM: role_id (str) - Target role identifier. -# @PARAM: db (Session) - Auth database session. -# @RETURN: None +# @PRE: role_id must be a valid existing role UUID. +# @POST: Role record is removed from auth.db. # @SIDE_EFFECT: Deletes record from auth.db and commits. -# @RELATION: [CALLS] ->[get_role_by_id:Function] +# @RELATION CALLS -> [get_role_by_id:Function] @router.delete("/roles/{role_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_role( role_id: str, @@ -314,10 +294,8 @@ async def delete_role( # #region list_permissions [C:3] [TYPE Function] # @BRIEF Lists all available system permissions for assignment. -# @POST Returns a list of all PermissionSchema objects. -# @PARAM: db (Session) - Auth database session. -# @RETURN: List[PermissionSchema] - List of permissions. -# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions +# @POST: Returns a list of all PermissionSchema objects. +# @RELATION CALLS -> backend.src.core.auth.repository.AuthRepository.list_permissions @router.get("/permissions", response_model=List[PermissionSchema]) async def list_permissions( db: Session = Depends(get_auth_db), @@ -332,7 +310,10 @@ async def list_permissions( db=db, declared_permissions=declared_permissions ) if inserted_count > 0: - log.reason(f"Synchronized {inserted_count} missing RBAC permissions into auth catalog") + logger.info( + "[api.admin.list_permissions][Action] Synchronized %s missing RBAC permissions into auth catalog", + inserted_count, + ) repo = AuthRepository(db) return repo.list_permissions() @@ -343,7 +324,7 @@ async def list_permissions( # #region list_ad_mappings [C:3] [TYPE Function] # @BRIEF Lists all AD Group to Role mappings. -# @RELATION CALLS -> [ADGroupMapping] +# @RELATION CALLS -> ADGroupMapping @router.get("/ad-mappings", response_model=List[ADGroupMappingSchema]) async def list_ad_mappings( db: Session = Depends(get_auth_db), @@ -357,10 +338,10 @@ async def list_ad_mappings( # #region create_ad_mapping [C:2] [TYPE Function] -# @BRIEF Creates a new AD Group mapping. # @RELATION DEPENDS_ON -> [ADGroupMapping:Class] # @RELATION DEPENDS_ON -> [get_auth_db:Function] # @RELATION DEPENDS_ON -> [has_permission:Function] +# @BRIEF Creates a new AD Group mapping. @router.post("/ad-mappings", response_model=ADGroupMappingSchema) async def create_ad_mapping( mapping_in: ADGroupMappingCreate, diff --git a/backend/src/api/routes/assistant/__init__.py b/backend/src/api/routes/assistant/__init__.py index a2cc545b..92639db5 100644 --- a/backend/src/api/routes/assistant/__init__.py +++ b/backend/src/api/routes/assistant/__init__.py @@ -1,11 +1,11 @@ # #region AssistantApi [C:5] [TYPE Module] [SEMANTICS api, assistant, chat, command, confirmation] # @BRIEF API routes for LLM assistant command parsing and safe execution orchestration. -# @LAYER API -# @INVARIANT Risky operations are never executed without valid confirmation token. +# @LAYER: API # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [AssistantMessageRecord] # @RELATION DEPENDS_ON -> [AssistantConfirmationRecord] # @RELATION DEPENDS_ON -> [AssistantAuditRecord] +# @INVARIANT: Risky operations are never executed without valid confirmation token. # Re-export public API for backward compatibility. from ._routes import router, send_message, confirm_operation, cancel_operation diff --git a/backend/src/api/routes/assistant/_admin_routes.py b/backend/src/api/routes/assistant/_admin_routes.py index 8b859f93..bf263a12 100644 --- a/backend/src/api/routes/assistant/_admin_routes.py +++ b/backend/src/api/routes/assistant/_admin_routes.py @@ -1,10 +1,10 @@ # #region AssistantAdminRoutes [C:5] [TYPE Module] # @BRIEF FastAPI route handlers for assistant admin operations — conversation listing, deletion, history, audit. -# @LAYER API -# @INVARIANT Audit endpoint requires tasks:READ permission. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantRoutes] # @RELATION DEPENDS_ON -> [AssistantSchemas] # @RELATION DEPENDS_ON -> [AssistantHistory] +# @INVARIANT: Audit endpoint requires tasks:READ permission. from __future__ import annotations @@ -42,9 +42,8 @@ from ._routes import router # #region list_conversations [C:2] [TYPE Function] # @BRIEF Return paginated conversation list for current user with archived flag and last message preview. -# @PRE Authenticated user context and valid pagination params. -# @POST Conversations are grouped by conversation_id sorted by latest activity descending. -# @RETURN Dict with items, paging metadata, and archive segmentation counts. +# @PRE: Authenticated user context and valid pagination params. +# @POST: Conversations are grouped by conversation_id sorted by latest activity descending. @router.get("/conversations") async def list_conversations( page: int = Query(1, ge=1), @@ -139,8 +138,8 @@ async def list_conversations( # #region delete_conversation [C:2] [TYPE Function] # @BRIEF Soft-delete or hard-delete a conversation and clear its in-memory trace. -# @PRE conversation_id belongs to current_user. -# @POST Conversation records are removed from DB and CONVERSATIONS cache. +# @PRE: conversation_id belongs to current_user. +# @POST: Conversation records are removed from DB and CONVERSATIONS cache. @router.delete("/conversations/{conversation_id}") async def delete_conversation( conversation_id: str, @@ -185,9 +184,8 @@ async def delete_conversation( @router.get("/history") # #region get_history [TYPE Function] # @BRIEF Retrieve paginated assistant conversation history for current user. -# @PRE Authenticated user is available and page params are valid. -# @POST Returns persistent messages and mirrored in-memory snapshot for diagnostics. -# @RETURN Dict with items, paging metadata, and resolved conversation_id. +# @PRE: Authenticated user is available and page params are valid. +# @POST: Returns persistent messages and mirrored in-memory snapshot for diagnostics. async def get_history( page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), @@ -257,9 +255,8 @@ async def get_history( @router.get("/audit") # #region get_assistant_audit [TYPE Function] # @BRIEF Return assistant audit decisions for current user from persistent and in-memory stores. -# @PRE User has tasks:READ permission. -# @POST Audit payload is returned in reverse chronological order from DB. -# @RETURN Dict with persistent and memory audit slices. +# @PRE: User has tasks:READ permission. +# @POST: Audit payload is returned in reverse chronological order from DB. async def get_assistant_audit( limit: int = Query(50, ge=1, le=500), current_user: User = Depends(get_current_user), diff --git a/backend/src/api/routes/assistant/_command_parser.py b/backend/src/api/routes/assistant/_command_parser.py index 8d044485..f80590ec 100644 --- a/backend/src/api/routes/assistant/_command_parser.py +++ b/backend/src/api/routes/assistant/_command_parser.py @@ -1,8 +1,8 @@ # #region AssistantCommandParser [C:4] [TYPE Module] # @BRIEF Deterministic RU/EN command text parser that converts user messages into intent payloads. -# @LAYER API -# @INVARIANT Every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantResolvers] +# @INVARIANT: Every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. from __future__ import annotations @@ -16,13 +16,13 @@ from ._resolvers import _extract_id, _is_production_env # #region _parse_command [C:4] [TYPE Function] # @BRIEF Deterministically parse RU/EN command text into intent payload. -# @PRE message contains raw user text and config manager resolves environments. -# @POST Returns intent dict with domain/operation/entities/confidence/risk fields. -# @SIDE_EFFECT None (pure parsing logic). -# @DATA_CONTRACT Input[message:str, config_manager:ConfigManager] -> Output[Dict[str,Any]{domain,operation,entities,confidence,risk_level,requires_confirmation}] -# @INVARIANT every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. +# @DATA_CONTRACT: Input[message:str, config_manager:ConfigManager] -> Output[Dict[str,Any]{domain,operation,entities,confidence,risk_level,requires_confirmation}] # @RELATION DEPENDS_ON -> [_extract_id] # @RELATION DEPENDS_ON -> [_is_production_env] +# @SIDE_EFFECT: None (pure parsing logic). +# @PRE: message contains raw user text and config manager resolves environments. +# @POST: Returns intent dict with domain/operation/entities/confidence/risk fields. +# @INVARIANT: every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation. def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any]: with belief_scope('_parse_command'): logger.reason('Belief protocol reasoning checkpoint for _parse_command') diff --git a/backend/src/api/routes/assistant/_dataset_review.py b/backend/src/api/routes/assistant/_dataset_review.py index 5c4738dd..5313fe0f 100644 --- a/backend/src/api/routes/assistant/_dataset_review.py +++ b/backend/src/api/routes/assistant/_dataset_review.py @@ -1,10 +1,10 @@ # #region AssistantDatasetReview [C:4] [TYPE Module] # @BRIEF Dataset review context loading and intent planning for the assistant API. -# @LAYER API -# @INVARIANT Dataset review operations are always scoped to the owner's session. +# @LAYER: API # @RELATION DEPENDS_ON -> [DatasetReviewOrchestrator] # @RELATION DEPENDS_ON -> [AssistantSchemas] # @RELATION DISPATCHES -> [AssistantDatasetReviewDispatch] +# @INVARIANT: Dataset review operations are always scoped to the owner's session. from __future__ import annotations @@ -37,10 +37,10 @@ from ._schemas import ( # #region _serialize_dataset_review_context [C:4] [TYPE Function] # @BRIEF Build assistant-safe dataset-review context snapshot with masked imported-filter payloads for session-scoped assistant routing. -# @PRE session_id is a valid active review session identifier. -# @POST Returns a serializable dictionary containing the complete review context. -# @SIDE_EFFECT Reads session data from the database. # @RELATION DEPENDS_ON -> [DatasetReviewSession] +# @PRE: session_id is a valid active review session identifier. +# @POST: Returns a serializable dictionary containing the complete review context. +# @SIDE_EFFECT: Reads session data from the database. def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str, Any]: with belief_scope('_serialize_dataset_review_context'): logger.reason('Belief protocol reasoning checkpoint for _serialize_dataset_review_context') @@ -57,10 +57,10 @@ def _serialize_dataset_review_context(session: DatasetReviewSession) -> Dict[str # #region _load_dataset_review_context [C:4] [TYPE Function] # @BRIEF Load owner-scoped dataset-review context for assistant planning and grounded response generation. -# @PRE session_id is a valid active review session identifier. -# @POST Returns a loaded context object with session data and findings. -# @SIDE_EFFECT Reads session data from the database. # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] +# @PRE: session_id is a valid active review session identifier. +# @POST: Returns a loaded context object with session data and findings. +# @SIDE_EFFECT: Reads session data from the database. def _load_dataset_review_context(dataset_review_session_id: Optional[str], current_user: User, db: Session) -> Optional[Dict[str, Any]]: with belief_scope('_load_dataset_review_context'): if not dataset_review_session_id: @@ -136,7 +136,7 @@ def _extract_quoted_segment(message: str, label: str) -> Optional[str]: # #region _plan_dataset_review_intent [C:3] [TYPE Function] # @BRIEF Parse session-scoped dataset-review assistant commands before falling back to generic assistant tool routing. -# @RELATION CALLS -> [DatasetReviewOrchestrator] +# @RELATION CALLS -> DatasetReviewOrchestrator def _plan_dataset_review_intent( message: str, dataset_context: Dict[str, Any], diff --git a/backend/src/api/routes/assistant/_dataset_review_dispatch.py b/backend/src/api/routes/assistant/_dataset_review_dispatch.py index 279fd3ee..44f94aa1 100644 --- a/backend/src/api/routes/assistant/_dataset_review_dispatch.py +++ b/backend/src/api/routes/assistant/_dataset_review_dispatch.py @@ -1,10 +1,10 @@ # #region AssistantDatasetReviewDispatch [C:4] [TYPE Module] # @BRIEF Dispatch and confirmation handling for dataset-review assistant intents. -# @LAYER API -# @INVARIANT Dataset review dispatch requires valid session version for write operations. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantDatasetReview] # @RELATION DEPENDS_ON -> [DatasetReviewOrchestrator] # @RELATION DEPENDS_ON -> [AssistantSchemas] +# @INVARIANT: Dataset review dispatch requires valid session version for write operations. from __future__ import annotations @@ -58,10 +58,10 @@ def _dataset_review_conflict_http_exception( # #region _dispatch_dataset_review_intent [C:4] [TYPE Function] # @BRIEF Route confirmed dataset-review assistant intents through existing backend dataset-review APIs and orchestration boundaries. -# @PRE context contains valid session data and user intent. -# @POST Returns a structured response with planned actions and confirmations. -# @SIDE_EFFECT May update session state and enqueue tasks. -# @RELATION CALLS -> [DatasetReviewOrchestrator] +# @RELATION CALLS -> DatasetReviewOrchestrator +# @PRE: context contains valid session data and user intent. +# @POST: Returns a structured response with planned actions and confirmations. +# @SIDE_EFFECT: May update session state and enqueue tasks. async def _dispatch_dataset_review_intent( intent: Dict[str, Any], current_user: User, diff --git a/backend/src/api/routes/assistant/_dispatch.py b/backend/src/api/routes/assistant/_dispatch.py index 559e4a56..e47f6251 100644 --- a/backend/src/api/routes/assistant/_dispatch.py +++ b/backend/src/api/routes/assistant/_dispatch.py @@ -1,11 +1,11 @@ # #region AssistantDispatch [C:5] [TYPE Module] # @BRIEF Intent dispatch engine, confirmation summary, and clarification text for the assistant API. -# @LAYER API -# @INVARIANT Unsupported operations are rejected via HTTPException(400). +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantSchemas] # @RELATION DEPENDS_ON -> [AssistantResolvers] # @RELATION DEPENDS_ON -> [AssistantLlmPlanner] # @RELATION DEPENDS_ON -> [AssistantDatasetReview] +# @INVARIANT: Unsupported operations are rejected via HTTPException(400). from __future__ import annotations @@ -42,8 +42,8 @@ git_service = GitService() # #region _clarification_text_for_intent [C:2] [TYPE Function] # @BRIEF Convert technical missing-parameter errors into user-facing clarification prompts. -# @PRE state was classified as needs_clarification for current intent/error combination. -# @POST Returned text is human-readable and actionable for target operation. +# @PRE: state was classified as needs_clarification for current intent/error combination. +# @POST: Returned text is human-readable and actionable for target operation. def _clarification_text_for_intent( intent: Optional[Dict[str, Any]], detail_text: str ) -> str: @@ -69,9 +69,9 @@ def _clarification_text_for_intent( # #region _async_confirmation_summary [C:4] [TYPE Function] # @BRIEF Build human-readable confirmation prompt for an intent before execution. -# @PRE actions is a non-empty list of planned review actions. -# @POST Returns a formatted summary string suitable for display to the user. -# @SIDE_EFFECT None - pure formatting function. +# @PRE: actions is a non-empty list of planned review actions. +# @POST: Returns a formatted summary string suitable for display to the user. +# @SIDE_EFFECT: None - pure formatting function. async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: ConfigManager, db: Session) -> str: with belief_scope('_confirmation_summary'): logger.reason('Belief protocol reasoning checkpoint for _confirmation_summary') @@ -139,15 +139,15 @@ async def _async_confirmation_summary(intent: Dict[str, Any], config_manager: Co # #region _dispatch_intent [C:5] [TYPE Function] # @BRIEF Execute parsed assistant intent via existing task/plugin/git services. -# @PRE intent operation is known and actor permissions are validated per operation. -# @POST Returns response text, optional task id, and UI actions for follow-up. -# @SIDE_EFFECT May enqueue tasks, invoke git operations, and query/update external service state. -# @DATA_CONTRACT Input[intent,current_user,task_manager,config_manager,db] -> Output[Tuple[text:str,task_id:Optional[str],actions:List[AssistantAction]]] -# @INVARIANT unsupported operations are rejected via HTTPException(400). +# @DATA_CONTRACT: Input[intent,current_user,task_manager,config_manager,db] -> Output[Tuple[text:str,task_id:Optional[str],actions:List[AssistantAction]]] # @RELATION DEPENDS_ON -> [_check_any_permission] # @RELATION DEPENDS_ON -> [_resolve_dashboard_id_entity] # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [GitService] +# @SIDE_EFFECT: May enqueue tasks, invoke git operations, and query/update external service state. +# @PRE: intent operation is known and actor permissions are validated per operation. +# @POST: Returns response text, optional task id, and UI actions for follow-up. +# @INVARIANT: unsupported operations are rejected via HTTPException(400). async def _dispatch_intent(intent: Dict[str, Any], current_user: User, task_manager: TaskManager, config_manager: ConfigManager, db: Session) -> Tuple[str, Optional[str], List[AssistantAction]]: with belief_scope('_dispatch_intent'): logger.reason('Belief protocol reasoning checkpoint for _dispatch_intent') diff --git a/backend/src/api/routes/assistant/_history.py b/backend/src/api/routes/assistant/_history.py index fcac8923..a6726579 100644 --- a/backend/src/api/routes/assistant/_history.py +++ b/backend/src/api/routes/assistant/_history.py @@ -1,8 +1,8 @@ # #region AssistantHistory [C:2] [TYPE Module] # @BRIEF Conversation history, audit trail, and confirmation persistence helpers for the assistant API. -# @LAYER API -# @INVARIANT Failed persistence attempts always rollback before returning. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantSchemas] +# @INVARIANT: Failed persistence attempts always rollback before returning. from __future__ import annotations @@ -32,12 +32,12 @@ logger = logger # #region _append_history [C:2] [TYPE Function] # @BRIEF Append conversation message to in-memory history buffer. -# @PRE user_id and conversation_id identify target conversation bucket. -# @POST Message entry is appended to CONVERSATIONS key list. -# @SIDE_EFFECT Mutates in-memory CONVERSATIONS store for user conversation history. -# @DATA_CONTRACT Input[user_id,conversation_id,role,text,state?,task_id?,confirmation_id?] -> Output[None] -# @INVARIANT every appended entry includes generated message_id and created_at timestamp. +# @DATA_CONTRACT: Input[user_id,conversation_id,role,text,state?,task_id?,confirmation_id?] -> Output[None] # @RELATION UPDATES -> [CONVERSATIONS] +# @SIDE_EFFECT: Mutates in-memory CONVERSATIONS store for user conversation history. +# @PRE: user_id and conversation_id identify target conversation bucket. +# @POST: Message entry is appended to CONVERSATIONS key list. +# @INVARIANT: every appended entry includes generated message_id and created_at timestamp. def _append_history( user_id: str, conversation_id: str, @@ -69,12 +69,12 @@ def _append_history( # #region _persist_message [C:2] [TYPE Function] # @BRIEF Persist assistant/user message record to database. -# @PRE db session is writable and message payload is serializable. -# @POST Message row is committed or persistence failure is logged. -# @SIDE_EFFECT Writes AssistantMessageRecord rows and commits or rollbacks the DB session. -# @DATA_CONTRACT Input[Session,user_id,conversation_id,role,text,state?,task_id?,confirmation_id?,metadata?] -> Output[None] -# @INVARIANT failed persistence attempts always rollback before returning. +# @DATA_CONTRACT: Input[Session,user_id,conversation_id,role,text,state?,task_id?,confirmation_id?,metadata?] -> Output[None] # @RELATION DEPENDS_ON -> [AssistantMessageRecord] +# @SIDE_EFFECT: Writes AssistantMessageRecord rows and commits or rollbacks the DB session. +# @PRE: db session is writable and message payload is serializable. +# @POST: Message row is committed or persistence failure is logged. +# @INVARIANT: failed persistence attempts always rollback before returning. def _persist_message( db: Session, user_id: str, @@ -110,12 +110,12 @@ def _persist_message( # #region _audit [C:2] [TYPE Function] # @BRIEF Append in-memory audit record for assistant decision trace. -# @PRE payload describes decision/outcome fields. -# @POST ASSISTANT_AUDIT list for user contains new timestamped entry. -# @SIDE_EFFECT Mutates in-memory ASSISTANT_AUDIT store and emits structured log event. -# @DATA_CONTRACT Input[user_id,payload:Dict[str,Any]] -> Output[None] -# @INVARIANT persisted in-memory audit entry always contains created_at in ISO format. +# @DATA_CONTRACT: Input[user_id,payload:Dict[str,Any]] -> Output[None] # @RELATION UPDATES -> [ASSISTANT_AUDIT] +# @SIDE_EFFECT: Mutates in-memory ASSISTANT_AUDIT store and emits structured log event. +# @PRE: payload describes decision/outcome fields. +# @POST: ASSISTANT_AUDIT list for user contains new timestamped entry. +# @INVARIANT: persisted in-memory audit entry always contains created_at in ISO format. def _audit(user_id: str, payload: Dict[str, Any]): if user_id not in ASSISTANT_AUDIT: ASSISTANT_AUDIT[user_id] = [] @@ -130,8 +130,8 @@ def _audit(user_id: str, payload: Dict[str, Any]): # #region _persist_audit [C:2] [TYPE Function] # @BRIEF Persist structured assistant audit payload in database. -# @PRE db session is writable and payload is JSON-serializable. -# @POST Audit row is committed or failure is logged with rollback. +# @PRE: db session is writable and payload is JSON-serializable. +# @POST: Audit row is committed or failure is logged with rollback. def _persist_audit( db: Session, user_id: str, payload: Dict[str, Any], conversation_id: Optional[str] ): @@ -157,8 +157,8 @@ def _persist_audit( # #region _persist_confirmation [C:2] [TYPE Function] # @BRIEF Persist confirmation token record to database. -# @PRE record contains id/user/intent/dispatch/expiry fields. -# @POST Confirmation row exists in persistent storage. +# @PRE: record contains id/user/intent/dispatch/expiry fields. +# @POST: Confirmation row exists in persistent storage. def _persist_confirmation(db: Session, record: ConfirmationRecord): try: row = AssistantConfirmationRecord( @@ -184,8 +184,8 @@ def _persist_confirmation(db: Session, record: ConfirmationRecord): # #region _update_confirmation_state [C:2] [TYPE Function] # @BRIEF Update persistent confirmation token lifecycle state. -# @PRE confirmation_id references existing row. -# @POST State and consumed_at fields are updated when applicable. +# @PRE: confirmation_id references existing row. +# @POST: State and consumed_at fields are updated when applicable. def _update_confirmation_state(db: Session, confirmation_id: str, state: str): try: row = ( @@ -209,8 +209,8 @@ def _update_confirmation_state(db: Session, confirmation_id: str, state: str): # #region _load_confirmation_from_db [C:2] [TYPE Function] # @BRIEF Load confirmation token from database into in-memory model. -# @PRE confirmation_id may or may not exist in storage. -# @POST Returns ConfirmationRecord when found, otherwise None. +# @PRE: confirmation_id may or may not exist in storage. +# @POST: Returns ConfirmationRecord when found, otherwise None. def _load_confirmation_from_db( db: Session, confirmation_id: str ) -> Optional[ConfirmationRecord]: @@ -238,8 +238,8 @@ def _load_confirmation_from_db( # #region _ensure_conversation [C:2] [TYPE Function] # @BRIEF Resolve active conversation id in memory or create a new one. -# @PRE user_id identifies current actor. -# @POST Returns stable conversation id and updates USER_ACTIVE_CONVERSATION. +# @PRE: user_id identifies current actor. +# @POST: Returns stable conversation id and updates USER_ACTIVE_CONVERSATION. def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: if conversation_id: from ._schemas import USER_ACTIVE_CONVERSATION @@ -261,8 +261,8 @@ def _ensure_conversation(user_id: str, conversation_id: Optional[str]) -> str: # #region _resolve_or_create_conversation [C:2] [TYPE Function] # @BRIEF Resolve active conversation using explicit id, memory cache, or persisted history. -# @PRE user_id and db session are available. -# @POST Returns conversation id and updates USER_ACTIVE_CONVERSATION cache. +# @PRE: user_id and db session are available. +# @POST: Returns conversation id and updates USER_ACTIVE_CONVERSATION cache. def _resolve_or_create_conversation( user_id: str, conversation_id: Optional[str], db: Session ) -> str: @@ -298,8 +298,8 @@ def _resolve_or_create_conversation( # #region _cleanup_history_ttl [C:2] [TYPE Function] # @BRIEF Enforce assistant message retention window by deleting expired rows and in-memory records. -# @PRE db session is available and user_id references current actor scope. -# @POST Messages older than ASSISTANT_MESSAGE_TTL_DAYS are removed from persistence and memory mirrors. +# @PRE: db session is available and user_id references current actor scope. +# @POST: Messages older than ASSISTANT_MESSAGE_TTL_DAYS are removed from persistence and memory mirrors. def _cleanup_history_ttl(db: Session, user_id: str): cutoff = datetime.utcnow() - timedelta(days=ASSISTANT_MESSAGE_TTL_DAYS) try: @@ -339,8 +339,8 @@ def _cleanup_history_ttl(db: Session, user_id: str): # #region _is_conversation_archived [C:2] [TYPE Function] # @BRIEF Determine archived state for a conversation based on last update timestamp. -# @PRE updated_at can be null for empty conversations. -# @POST Returns True when conversation inactivity exceeds archive threshold. +# @PRE: updated_at can be null for empty conversations. +# @POST: Returns True when conversation inactivity exceeds archive threshold. def _is_conversation_archived(updated_at: Optional[datetime]) -> bool: if not updated_at: return False @@ -353,8 +353,8 @@ def _is_conversation_archived(updated_at: Optional[datetime]) -> bool: # #region _coerce_query_bool [C:2] [TYPE Function] # @BRIEF Normalize bool-like query values for compatibility in direct handler invocations/tests. -# @PRE value may be bool, string, or FastAPI Query metadata object. -# @POST Returns deterministic boolean flag. +# @PRE: value may be bool, string, or FastAPI Query metadata object. +# @POST: Returns deterministic boolean flag. def _coerce_query_bool(value: Any) -> bool: if isinstance(value, bool): return value diff --git a/backend/src/api/routes/assistant/_llm_planner.py b/backend/src/api/routes/assistant/_llm_planner.py index 4af20aa4..677337cb 100644 --- a/backend/src/api/routes/assistant/_llm_planner.py +++ b/backend/src/api/routes/assistant/_llm_planner.py @@ -1,10 +1,10 @@ # #region AssistantLlmPlanner [C:3] [TYPE Module] # @BRIEF LLM-based intent planning, tool catalog construction, and authorization for the assistant API. -# @LAYER API -# @INVARIANT Tool catalog is filtered by user permissions before being sent to LLM. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantSchemas] # @RELATION DEPENDS_ON -> [AssistantResolvers] # @RELATION DISPATCHES -> [AssistantLlmPlannerIntent] +# @INVARIANT: Tool catalog is filtered by user permissions before being sent to LLM. from __future__ import annotations @@ -30,8 +30,8 @@ from ._resolvers import ( # #region _check_any_permission [C:2] [TYPE Function] # @BRIEF Validate user against alternative permission checks (logical OR). -# @PRE checks list contains resource-action tuples. -# @POST Returns on first successful permission; raises 403-like HTTPException otherwise. +# @PRE: checks list contains resource-action tuples. +# @POST: Returns on first successful permission; raises 403-like HTTPException otherwise. def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): errors: List[HTTPException] = [] for resource, action in checks: @@ -53,8 +53,8 @@ def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): # #region _has_any_permission [C:2] [TYPE Function] # @BRIEF Check whether user has at least one permission tuple from the provided list. -# @PRE current_user and checks list are valid. -# @POST Returns True when at least one permission check passes. +# @PRE: current_user and checks list are valid. +# @POST: Returns True when at least one permission check passes. def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bool: try: _check_any_permission(current_user, checks) @@ -68,9 +68,9 @@ def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bo # #region _build_tool_catalog [C:3] [TYPE Function] # @BRIEF Build current-user tool catalog for LLM planner with operation contracts and defaults. -# @PRE current_user is authenticated; config/db are available. -# @POST Returns list of executable tools filtered by permission and runtime availability. -# @RELATION CALLS -> [LLMProviderService] +# @PRE: current_user is authenticated; config/db are available. +# @POST: Returns list of executable tools filtered by permission and runtime availability. +# @RELATION CALLS -> LLMProviderService def _build_tool_catalog( current_user: User, config_manager: ConfigManager, @@ -270,8 +270,8 @@ def _build_tool_catalog( # #region _coerce_intent_entities [C:2] [TYPE Function] # @BRIEF Normalize intent entity value types from LLM output to route-compatible values. -# @PRE intent contains entities dict or missing entities. -# @POST Returned intent has numeric ids coerced where possible and string values stripped. +# @PRE: intent contains entities dict or missing entities. +# @POST: Returned intent has numeric ids coerced where possible and string values stripped. def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]: entities = intent.get("entities") if not isinstance(entities, dict): diff --git a/backend/src/api/routes/assistant/_llm_planner_intent.py b/backend/src/api/routes/assistant/_llm_planner_intent.py index a59799e0..85a67760 100644 --- a/backend/src/api/routes/assistant/_llm_planner_intent.py +++ b/backend/src/api/routes/assistant/_llm_planner_intent.py @@ -1,9 +1,9 @@ # #region AssistantLlmPlannerIntent [C:3] [TYPE Module] # @BRIEF LLM-based intent planning and authorization for the assistant API — separated from tool catalog. -# @LAYER API -# @INVARIANT Production deployments always require confirmation. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantLlmPlanner] # @RELATION DEPENDS_ON -> [AssistantResolvers] +# @INVARIANT: Production deployments always require confirmation. from __future__ import annotations @@ -35,8 +35,8 @@ from ._llm_planner import ( # #region _plan_intent_with_llm [C:2] [TYPE Function] # @BRIEF Use active LLM provider to select best tool/operation from dynamic catalog. -# @PRE tools list contains allowed operations for current user. -# @POST Returns normalized intent dict when planning succeeds; otherwise None. +# @PRE: tools list contains allowed operations for current user. +# @POST: Returns normalized intent dict when planning succeeds; otherwise None. async def _plan_intent_with_llm( message: str, tools: List[Dict[str, Any]], @@ -156,8 +156,8 @@ async def _plan_intent_with_llm( # #region _authorize_intent [C:2] [TYPE Function] # @BRIEF Validate user permissions for parsed intent before confirmation/dispatch. -# @PRE intent.operation is present for known assistant command domains. -# @POST Returns if authorized; raises HTTPException(403) when denied. +# @PRE: intent.operation is present for known assistant command domains. +# @POST: Returns if authorized; raises HTTPException(403) when denied. def _authorize_intent(intent: Dict[str, Any], current_user: User): operation = intent.get("operation") if operation in INTENT_PERMISSION_CHECKS: diff --git a/backend/src/api/routes/assistant/_resolvers.py b/backend/src/api/routes/assistant/_resolvers.py index a9f24ee9..e0646819 100644 --- a/backend/src/api/routes/assistant/_resolvers.py +++ b/backend/src/api/routes/assistant/_resolvers.py @@ -1,9 +1,9 @@ # #region AssistantResolvers [C:2] [TYPE Module] # @BRIEF Environment, dashboard, provider, and task resolution utilities for the assistant API. -# @LAYER API -# @INVARIANT Resolution functions never raise; they return None on failure. +# @LAYER: API # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [SupersetClient] +# @INVARIANT: Resolution functions never raise; they return None on failure. from __future__ import annotations @@ -24,8 +24,8 @@ logger = cast(Any, logger) # #region _extract_id [C:2] [TYPE Function] # @BRIEF Extract first regex match group from text by ordered pattern list. -# @PRE patterns contain at least one capture group. -# @POST Returns first matched token or None. +# @PRE: patterns contain at least one capture group. +# @POST: Returns first matched token or None. def _extract_id(text: str, patterns: List[str]) -> Optional[str]: for p in patterns: m = re.search(p, text, flags=re.IGNORECASE) @@ -38,8 +38,8 @@ def _extract_id(text: str, patterns: List[str]) -> Optional[str]: # #region _resolve_env_id [C:2] [TYPE Function] # @BRIEF Resolve environment identifier/name token to canonical environment id. -# @PRE config_manager provides environment list. -# @POST Returns matched environment id or None. +# @PRE: config_manager provides environment list. +# @POST: Returns matched environment id or None. def _resolve_env_id( token: Optional[str], config_manager: ConfigManager ) -> Optional[str]: @@ -58,8 +58,8 @@ def _resolve_env_id( # #region _is_production_env [C:2] [TYPE Function] # @BRIEF Determine whether environment token resolves to production-like target. -# @PRE config_manager provides environments or token text is provided. -# @POST Returns True for production/prod synonyms, else False. +# @PRE: config_manager provides environments or token text is provided. +# @POST: Returns True for production/prod synonyms, else False. def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> bool: env_id = _resolve_env_id(token, config_manager) if not env_id: @@ -76,8 +76,8 @@ def _is_production_env(token: Optional[str], config_manager: ConfigManager) -> b # #region _resolve_provider_id [C:2] [TYPE Function] # @BRIEF Resolve provider token to provider id with active/default fallback. -# @PRE db session can load provider list through LLMProviderService. -# @POST Returns provider id or None when no providers configured. +# @PRE: db session can load provider list through LLMProviderService. +# @POST: Returns provider id or None when no providers configured. def _resolve_provider_id( provider_token: Optional[str], db: Session, @@ -112,8 +112,8 @@ def _resolve_provider_id( # #region _get_default_environment_id [C:2] [TYPE Function] # @BRIEF Resolve default environment id from settings or first configured environment. -# @PRE config_manager returns environments list. -# @POST Returns default environment id or None when environment list is empty. +# @PRE: config_manager returns environments list. +# @POST: Returns default environment id or None when environment list is empty. def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: configured = config_manager.get_environments() if not configured: @@ -136,8 +136,8 @@ def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: # #region _resolve_dashboard_id_by_ref [C:2] [TYPE Function] # @BRIEF Resolve dashboard id by title or slug reference in selected environment. -# @PRE dashboard_ref is a non-empty string-like token. -# @POST Returns dashboard id when uniquely matched, otherwise None. +# @PRE: dashboard_ref is a non-empty string-like token. +# @POST: Returns dashboard id when uniquely matched, otherwise None. def _resolve_dashboard_id_by_ref( dashboard_ref: Optional[str], env_id: Optional[str], @@ -188,8 +188,8 @@ def _resolve_dashboard_id_by_ref( # #region _resolve_dashboard_id_entity [C:2] [TYPE Function] # @BRIEF Resolve dashboard id from intent entities using numeric id or dashboard_ref fallback. -# @PRE entities may contain dashboard_id as int/str and optional dashboard_ref. -# @POST Returns resolved dashboard id or None when ambiguous/unresolvable. +# @PRE: entities may contain dashboard_id as int/str and optional dashboard_ref. +# @POST: Returns resolved dashboard id or None when ambiguous/unresolvable. def _resolve_dashboard_id_entity( entities: Dict[str, Any], config_manager: ConfigManager, @@ -229,8 +229,8 @@ def _resolve_dashboard_id_entity( # #region _get_environment_name_by_id [C:2] [TYPE Function] # @BRIEF Resolve human-readable environment name by id. -# @PRE environment id may be None. -# @POST Returns matching environment name or fallback id. +# @PRE: environment id may be None. +# @POST: Returns matching environment name or fallback id. def _get_environment_name_by_id( env_id: Optional[str], config_manager: ConfigManager ) -> str: @@ -246,8 +246,8 @@ def _get_environment_name_by_id( # #region _extract_result_deep_links [C:2] [TYPE Function] # @BRIEF Build deep-link actions to verify task result from assistant chat. -# @PRE task object is available. -# @POST Returns zero or more assistant actions for dashboard open/diff. +# @PRE: task object is available. +# @POST: Returns zero or more assistant actions for dashboard open/diff. def _extract_result_deep_links( task: Any, config_manager: ConfigManager ) -> List: @@ -319,8 +319,8 @@ def _extract_result_deep_links( # #region _build_task_observability_summary [C:2] [TYPE Function] # @BRIEF Build compact textual summary for completed tasks to reduce "black box" effect. -# @PRE task may contain plugin-specific result payload. -# @POST Returns non-empty summary line for known task types or empty string fallback. +# @PRE: task may contain plugin-specific result payload. +# @POST: Returns non-empty summary line for known task types or empty string fallback. def _build_task_observability_summary(task: Any, config_manager: ConfigManager) -> str: plugin_id = getattr(task, "plugin_id", None) status = str(getattr(task, "status", "")).upper() diff --git a/backend/src/api/routes/assistant/_routes.py b/backend/src/api/routes/assistant/_routes.py index a4a7947d..3312a89d 100644 --- a/backend/src/api/routes/assistant/_routes.py +++ b/backend/src/api/routes/assistant/_routes.py @@ -1,7 +1,6 @@ # #region AssistantRoutes [C:5] [TYPE Module] # @BRIEF FastAPI route handlers for the assistant API — message sending, confirmation, conversation management. -# @LAYER API -# @INVARIANT Risky operations are never executed without valid confirmation token. +# @LAYER: API # @RELATION DEPENDS_ON -> [AssistantSchemas] # @RELATION DEPENDS_ON -> [AssistantHistory] # @RELATION DEPENDS_ON -> [AssistantCommandParser] @@ -9,6 +8,7 @@ # @RELATION DEPENDS_ON -> [AssistantDatasetReview] # @RELATION DEPENDS_ON -> [AssistantDispatch] # @RELATION DISPATCHES -> [AssistantAdminRoutes] +# @INVARIANT: Risky operations are never executed without valid confirmation token. from __future__ import annotations @@ -78,18 +78,17 @@ router = APIRouter(tags=["Assistant"]) @router.post("/messages", response_model=AssistantMessageResponse) # #region send_message [C:5] [TYPE Function] # @BRIEF Parse assistant command, enforce safety gates, and dispatch executable intent. -# @PRE Authenticated user is available and message text is non-empty. -# @POST Response state is one of clarification/confirmation/started/success/denied/failed. -# @SIDE_EFFECT Persists chat/audit state, mutates in-memory conversation and confirmation stores, and may create confirmation records. -# @DATA_CONTRACT Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse] -# @INVARIANT non-safe operations are gated with confirmation before execution from this endpoint. -# @RETURN AssistantMessageResponse with operation feedback and optional actions. +# @DATA_CONTRACT: Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse] # @RELATION DEPENDS_ON -> [_plan_intent_with_llm] # @RELATION DEPENDS_ON -> [_parse_command] # @RELATION DEPENDS_ON -> [_dispatch_intent] # @RELATION DEPENDS_ON -> [_append_history] # @RELATION DEPENDS_ON -> [_persist_message] # @RELATION DEPENDS_ON -> [_audit] +# @SIDE_EFFECT: Persists chat/audit state, mutates in-memory conversation and confirmation stores, and may create confirmation records. +# @PRE: Authenticated user is available and message text is non-empty. +# @POST: Response state is one of clarification/confirmation/started/success/denied/failed. +# @INVARIANT: non-safe operations are gated with confirmation before execution from this endpoint. async def send_message(request: AssistantMessageRequest, current_user: User=Depends(get_current_user), task_manager: TaskManager=Depends(get_task_manager), config_manager: ConfigManager=Depends(get_config_manager), db: Session=Depends(get_db)): with belief_scope('send_message'): logger.reason('Belief protocol reasoning checkpoint for send_message') @@ -172,9 +171,8 @@ async def send_message(request: AssistantMessageRequest, current_user: User=Depe ) # #region confirm_operation [C:2] [TYPE Function] # @BRIEF Execute previously requested risky operation after explicit user confirmation. -# @PRE confirmation_id exists, belongs to current user, is pending, and not expired. -# @POST Confirmation state becomes consumed and operation result is persisted in history. -# @RETURN AssistantMessageResponse with task details when async execution starts. +# @PRE: confirmation_id exists, belongs to current user, is pending, and not expired. +# @POST: Confirmation state becomes consumed and operation result is persisted in history. async def confirm_operation( confirmation_id: str, current_user: User = Depends(get_current_user), @@ -260,9 +258,8 @@ async def confirm_operation( ) # #region cancel_operation [C:2] [TYPE Function] # @BRIEF Cancel pending risky operation and mark confirmation token as cancelled. -# @PRE confirmation_id exists, belongs to current user, and is still pending. -# @POST Confirmation becomes cancelled and cannot be executed anymore. -# @RETURN AssistantMessageResponse confirming cancellation. +# @PRE: confirmation_id exists, belongs to current user, and is still pending. +# @POST: Confirmation becomes cancelled and cannot be executed anymore. async def cancel_operation( confirmation_id: str, current_user: User = Depends(get_current_user), diff --git a/backend/src/api/routes/assistant/_schemas.py b/backend/src/api/routes/assistant/_schemas.py index a27b37af..d2767f7f 100644 --- a/backend/src/api/routes/assistant/_schemas.py +++ b/backend/src/api/routes/assistant/_schemas.py @@ -1,9 +1,9 @@ # #region AssistantSchemas [C:2] [TYPE Module] # @BRIEF Pydantic models, in-memory stores, and permission mappings for the assistant API. -# @LAYER API -# @INVARIANT In-memory stores are module-level singletons shared across the assistant package. +# @LAYER: API # @RELATION USED_BY -> [AssistantRoutes] # @RELATION USED_BY -> [AssistantHistory] +# @INVARIANT: In-memory stores are module-level singletons shared across the assistant package. from __future__ import annotations @@ -17,12 +17,12 @@ from src.schemas.auth import User # #region AssistantMessageRequest [C:1] [TYPE Class] # @BRIEF Input payload for assistant message endpoint. -# @PRE message length is within accepted bounds. -# @POST Request object provides message text and optional conversation binding. -# @SIDE_EFFECT None (schema declaration only). -# @DATA_CONTRACT Input[conversation_id?:str, message:str(1..4000)] -> Output[AssistantMessageRequest] -# @INVARIANT message is always non-empty and no longer than 4000 characters. +# @DATA_CONTRACT: Input[conversation_id?:str, message:str(1..4000)] -> Output[AssistantMessageRequest] # @RELATION USED_BY -> [send_message] +# @SIDE_EFFECT: None (schema declaration only). +# @PRE: message length is within accepted bounds. +# @POST: Request object provides message text and optional conversation binding. +# @INVARIANT: message is always non-empty and no longer than 4000 characters. class AssistantMessageRequest(BaseModel): conversation_id: Optional[str] = None message: str = Field(..., min_length=1, max_length=4000) @@ -34,12 +34,12 @@ class AssistantMessageRequest(BaseModel): # #region AssistantAction [C:1] [TYPE Class] # @BRIEF UI action descriptor returned with assistant responses. -# @PRE type and label are provided by orchestration logic. -# @POST Action can be rendered as button on frontend. -# @SIDE_EFFECT None (schema declaration only). -# @DATA_CONTRACT Input[type:str, label:str, target?:str] -> Output[AssistantAction] -# @INVARIANT type and label are required for every UI action. +# @DATA_CONTRACT: Input[type:str, label:str, target?:str] -> Output[AssistantAction] # @RELATION USED_BY -> [AssistantMessageResponse] +# @SIDE_EFFECT: None (schema declaration only). +# @PRE: type and label are provided by orchestration logic. +# @POST: Action can be rendered as button on frontend. +# @INVARIANT: type and label are required for every UI action. class AssistantAction(BaseModel): type: str label: str @@ -51,14 +51,14 @@ class AssistantAction(BaseModel): # #region AssistantMessageResponse [C:1] [TYPE Class] # @BRIEF Output payload contract for assistant interaction endpoints. -# @PRE Response includes deterministic state and text. -# @POST Payload may include task_id/confirmation_id/actions for UI follow-up. -# @SIDE_EFFECT None (schema declaration only). -# @DATA_CONTRACT Input[conversation_id,response_id,state,text,intent?,confirmation_id?,task_id?,actions[],created_at] -> Output[AssistantMessageResponse] -# @INVARIANT created_at and state are always present in endpoint responses. +# @DATA_CONTRACT: Input[conversation_id,response_id,state,text,intent?,confirmation_id?,task_id?,actions[],created_at] -> Output[AssistantMessageResponse] # @RELATION RETURNED_BY -> [send_message] # @RELATION RETURNED_BY -> [confirm_operation] # @RELATION RETURNED_BY -> [cancel_operation] +# @SIDE_EFFECT: None (schema declaration only). +# @PRE: Response includes deterministic state and text. +# @POST: Payload may include task_id/confirmation_id/actions for UI follow-up. +# @INVARIANT: created_at and state are always present in endpoint responses. class AssistantMessageResponse(BaseModel): conversation_id: str response_id: str @@ -76,14 +76,14 @@ class AssistantMessageResponse(BaseModel): # #region ConfirmationRecord [C:1] [TYPE Class] # @BRIEF In-memory confirmation token model for risky operation dispatch. -# @PRE intent/dispatch/user_id are populated at confirmation request time. -# @POST Record tracks lifecycle state and expiry timestamp. -# @SIDE_EFFECT None (schema declaration only). -# @DATA_CONTRACT Input[id,user_id,conversation_id,intent,dispatch,expires_at,state?,created_at] -> Output[ConfirmationRecord] -# @INVARIANT state defaults to "pending" and expires_at bounds confirmation validity. +# @DATA_CONTRACT: Input[id,user_id,conversation_id,intent,dispatch,expires_at,state?,created_at] -> Output[ConfirmationRecord] # @RELATION USED_BY -> [send_message] # @RELATION USED_BY -> [confirm_operation] # @RELATION USED_BY -> [cancel_operation] +# @SIDE_EFFECT: None (schema declaration only). +# @PRE: intent/dispatch/user_id are populated at confirmation request time. +# @POST: Record tracks lifecycle state and expiry timestamp. +# @INVARIANT: state defaults to "pending" and expires_at bounds confirmation validity. class ConfirmationRecord(BaseModel): id: str user_id: str diff --git a/backend/src/api/routes/clean_release.py b/backend/src/api/routes/clean_release.py index 9659abf4..cf5b3db9 100644 --- a/backend/src/api/routes/clean_release.py +++ b/backend/src/api/routes/clean_release.py @@ -1,12 +1,12 @@ # #region CleanReleaseApi [C:4] [TYPE Module] [SEMANTICS api, clean-release, candidate-preparation, compliance] # @BRIEF Expose clean release endpoints for candidate preparation and subsequent compliance flow. -# @LAYER API -# @PRE Clean release repository and preparation service dependencies are configured for the current request scope. -# @POST Candidate preparation, manifest, and compliance routes expose deterministic API payloads without reporting prepared state on failed preparation. -# @SIDE_EFFECT Persists candidate/compliance lifecycle state and triggers clean-release orchestration services. -# @INVARIANT API never reports prepared status if preparation errors are present. +# @LAYER: API # @RELATION DEPENDS_ON -> [get_clean_release_repository] # @RELATION DEPENDS_ON -> [PreparationService] +# @PRE: Clean release repository and preparation service dependencies are configured for the current request scope. +# @POST: Candidate preparation, manifest, and compliance routes expose deterministic API payloads without reporting prepared state on failed preparation. +# @SIDE_EFFECT: Persists candidate/compliance lifecycle state and triggers clean-release orchestration services. +# @INVARIANT: API never reports prepared status if preparation errors are present. from __future__ import annotations @@ -119,8 +119,8 @@ class CreateComplianceRunRequest(BaseModel): # #region register_candidate_v2_endpoint [TYPE Function] # @BRIEF Register a clean-release candidate for headless lifecycle. -# @PRE Candidate identifier is unique. -# @POST Candidate is persisted in DRAFT status. +# @PRE: Candidate identifier is unique. +# @POST: Candidate is persisted in DRAFT status. @router.post( "/candidates", response_model=CandidateDTO, status_code=status.HTTP_201_CREATED ) @@ -160,8 +160,8 @@ async def register_candidate_v2_endpoint( # #region import_candidate_artifacts_v2_endpoint [TYPE Function] # @BRIEF Import candidate artifacts in headless flow. -# @PRE Candidate exists and artifacts array is non-empty. -# @POST Artifacts are persisted and candidate advances to PREPARED if it was DRAFT. +# @PRE: Candidate exists and artifacts array is non-empty. +# @POST: Artifacts are persisted and candidate advances to PREPARED if it was DRAFT. @router.post("/candidates/{candidate_id}/artifacts") async def import_candidate_artifacts_v2_endpoint( candidate_id: str, @@ -218,8 +218,8 @@ async def import_candidate_artifacts_v2_endpoint( # #region build_candidate_manifest_v2_endpoint [TYPE Function] # @BRIEF Build immutable manifest snapshot for prepared candidate. -# @PRE Candidate exists and has imported artifacts. -# @POST Returns created ManifestDTO with incremented version. +# @PRE: Candidate exists and has imported artifacts. +# @POST: Returns created ManifestDTO with incremented version. @router.post( "/candidates/{candidate_id}/manifests", response_model=ManifestDTO, @@ -262,8 +262,8 @@ async def build_candidate_manifest_v2_endpoint( # #region get_candidate_overview_v2_endpoint [TYPE Function] # @BRIEF Return expanded candidate overview DTO for headless lifecycle visibility. -# @PRE Candidate exists. -# @POST Returns CandidateOverviewDTO built from the same repository state used by headless US1 endpoints. +# @PRE: Candidate exists. +# @POST: Returns CandidateOverviewDTO built from the same repository state used by headless US1 endpoints. @router.get("/candidates/{candidate_id}/overview", response_model=CandidateOverviewDTO) async def get_candidate_overview_v2_endpoint( candidate_id: str, @@ -378,8 +378,8 @@ async def get_candidate_overview_v2_endpoint( # #region prepare_candidate_endpoint [TYPE Function] # @BRIEF Prepare candidate with policy evaluation and deterministic manifest generation. -# @PRE Candidate and active policy exist in repository. -# @POST Returns preparation result including manifest reference and violations. +# @PRE: Candidate and active policy exist in repository. +# @POST: Returns preparation result including manifest reference and violations. @router.post("/candidates/prepare") async def prepare_candidate_endpoint( payload: PrepareCandidateRequest, @@ -412,8 +412,8 @@ async def prepare_candidate_endpoint( # #region start_check [TYPE Function] # @BRIEF Start and finalize a clean compliance check run and persist report artifacts. -# @PRE Active policy and candidate exist. -# @POST Returns accepted payload with check_run_id and started_at. +# @PRE: Active policy and candidate exist. +# @POST: Returns accepted payload with check_run_id and started_at. @router.post("/checks", status_code=status.HTTP_202_ACCEPTED) async def start_check( payload: StartCheckRequest, @@ -548,8 +548,8 @@ async def start_check( # #region get_check_status [TYPE Function] # @BRIEF Return terminal/intermediate status payload for a check run. -# @PRE check_run_id references an existing run. -# @POST Deterministic payload shape includes checks and violations arrays. +# @PRE: check_run_id references an existing run. +# @POST: Deterministic payload shape includes checks and violations arrays. @router.get("/checks/{check_run_id}") async def get_check_status( check_run_id: str, @@ -600,8 +600,8 @@ async def get_check_status( # #region get_report [TYPE Function] # @BRIEF Return persisted compliance report by report_id. -# @PRE report_id references an existing report. -# @POST Returns serialized report object. +# @PRE: report_id references an existing report. +# @POST: Returns serialized report object. @router.get("/reports/{report_id}") async def get_report( report_id: str, diff --git a/backend/src/api/routes/clean_release_v2.py b/backend/src/api/routes/clean_release_v2.py index 83dc71ed..f4dd36b7 100644 --- a/backend/src/api/routes/clean_release_v2.py +++ b/backend/src/api/routes/clean_release_v2.py @@ -1,12 +1,12 @@ # #region CleanReleaseV2Api [C:4] [TYPE Module] # @BRIEF Redesigned clean release API for headless candidate lifecycle. -# @LAYER UI (API) -# @PRE Clean release repository dependency is available for candidate lifecycle endpoints. -# @POST Candidate registration, approval, publication, and revocation routes are registered without behavior changes. -# @SIDE_EFFECT Persists candidate lifecycle state through clean release services and repository adapters. +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [CleanReleaseRepository] # @RELATION CALLS -> [approve_candidate] # @RELATION CALLS -> [publish_candidate] +# @PRE: Clean release repository dependency is available for candidate lifecycle endpoints. +# @POST: Candidate registration, approval, publication, and revocation routes are registered without behavior changes. +# @SIDE_EFFECT: Persists candidate lifecycle state through clean release services and repository adapters. from fastapi import APIRouter, Depends, HTTPException, status from typing import List, Dict, Any @@ -61,9 +61,8 @@ class RevokeRequest(dict): # #region register_candidate [C:3] [TYPE Function] # @BRIEF Register a new release candidate. -# @PRE Payload contains required fields (id, version, source_snapshot_ref, created_by). -# @POST Candidate is saved in repository. -# @RETURN CandidateDTO +# @PRE: Payload contains required fields (id, version, source_snapshot_ref, created_by). +# @POST: Candidate is saved in repository. # @RELATION DEPENDS_ON -> [CleanReleaseRepository] # @RELATION DEPENDS_ON -> [clean_release_dto] @router.post( @@ -97,8 +96,8 @@ async def register_candidate( # #region import_artifacts [C:3] [TYPE Function] # @BRIEF Associate artifacts with a release candidate. -# @PRE Candidate exists. -# @POST Artifacts are processed (placeholder). +# @PRE: Candidate exists. +# @POST: Artifacts are processed (placeholder). # @RELATION DEPENDS_ON -> [CleanReleaseRepository] @router.post("/candidates/{candidate_id}/artifacts") async def import_artifacts( @@ -130,9 +129,8 @@ async def import_artifacts( # #region build_manifest [C:3] [TYPE Function] # @BRIEF Generate distribution manifest for a candidate. -# @PRE Candidate exists. -# @POST Manifest is created and saved. -# @RETURN ManifestDTO +# @PRE: Candidate exists. +# @POST: Manifest is created and saved. # @RELATION DEPENDS_ON -> [CleanReleaseRepository] @router.post( "/candidates/{candidate_id}/manifests", diff --git a/backend/src/api/routes/connections.py b/backend/src/api/routes/connections.py index c415be4f..462b2f77 100644 --- a/backend/src/api/routes/connections.py +++ b/backend/src/api/routes/connections.py @@ -1,10 +1,9 @@ # #region ConnectionsRouter [C:3] [TYPE Module] [SEMANTICS api, router, connections, database] # @BRIEF Defines the FastAPI router for managing external database connections. -# @LAYER UI (API) +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [get_db] # @RELATION DEPENDS_ON -> [ConnectionConfig] -# [SECTION: IMPORTS] from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session @@ -13,16 +12,15 @@ from ...models.connection import ConnectionConfig from pydantic import BaseModel from datetime import datetime from ...core.logger import logger, belief_scope -# [/SECTION] router = APIRouter() # #region _ensure_connections_schema [C:3] [TYPE Function] # @BRIEF Ensures the connection_configs table exists before CRUD access. -# @PRE db is an active SQLAlchemy session. -# @POST The current bind can safely query ConnectionConfig. -# @RELATION CALLS -> [ensure_connection_configs_table] +# @PRE: db is an active SQLAlchemy session. +# @POST: The current bind can safely query ConnectionConfig. +# @RELATION CALLS -> ensure_connection_configs_table def _ensure_connections_schema(db: Session): with belief_scope("ConnectionsRouter.ensure_schema"): ensure_connection_configs_table(db.get_bind()) @@ -33,7 +31,7 @@ def _ensure_connections_schema(db: Session): # #region ConnectionSchema [C:3] [TYPE Class] # @BRIEF Pydantic model for connection response. -# @RELATION BINDS_TO -> [ConnectionConfig] +# @RELATION BINDS_TO -> ConnectionConfig class ConnectionSchema(BaseModel): id: str name: str @@ -53,7 +51,7 @@ class ConnectionSchema(BaseModel): # #region ConnectionCreate [C:3] [TYPE Class] # @BRIEF Pydantic model for creating a connection. -# @RELATION BINDS_TO -> [ConnectionConfig] +# @RELATION BINDS_TO -> ConnectionConfig class ConnectionCreate(BaseModel): name: str type: str @@ -69,12 +67,10 @@ class ConnectionCreate(BaseModel): # #region list_connections [C:3] [TYPE Function] # @BRIEF Lists all saved connections. -# @PRE Database session is active. -# @POST Returns list of connection configs. -# @PARAM: db (Session) - Database session. -# @RETURN: List[ConnectionSchema] - List of connections. -# @RELATION: CALLS -> _ensure_connections_schema -# @RELATION: DEPENDS_ON -> ConnectionConfig +# @PRE: Database session is active. +# @POST: Returns list of connection configs. +# @RELATION CALLS -> _ensure_connections_schema +# @RELATION DEPENDS_ON -> ConnectionConfig @router.get("", response_model=List[ConnectionSchema]) async def list_connections(db: Session = Depends(get_db)): with belief_scope("ConnectionsRouter.list_connections"): @@ -88,13 +84,10 @@ async def list_connections(db: Session = Depends(get_db)): # #region create_connection [C:3] [TYPE Function] # @BRIEF Creates a new connection configuration. -# @PRE Connection name is unique. -# @POST Connection is saved to DB. -# @PARAM: connection (ConnectionCreate) - Config data. -# @PARAM: db (Session) - Database session. -# @RETURN: ConnectionSchema - Created connection. -# @RELATION: CALLS -> _ensure_connections_schema -# @RELATION: DEPENDS_ON -> ConnectionConfig +# @PRE: Connection name is unique. +# @POST: Connection is saved to DB. +# @RELATION CALLS -> _ensure_connections_schema +# @RELATION DEPENDS_ON -> ConnectionConfig @router.post("", response_model=ConnectionSchema, status_code=status.HTTP_201_CREATED) async def create_connection( connection: ConnectionCreate, db: Session = Depends(get_db) @@ -116,13 +109,10 @@ async def create_connection( # #region delete_connection [C:3] [TYPE Function] # @BRIEF Deletes a connection configuration. -# @PRE Connection ID exists. -# @POST Connection is removed from DB. -# @PARAM: connection_id (str) - ID to delete. -# @PARAM: db (Session) - Database session. -# @RETURN: None. -# @RELATION: CALLS -> _ensure_connections_schema -# @RELATION: DEPENDS_ON -> ConnectionConfig +# @PRE: Connection ID exists. +# @POST: Connection is removed from DB. +# @RELATION CALLS -> _ensure_connections_schema +# @RELATION DEPENDS_ON -> ConnectionConfig @router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_connection(connection_id: str, db: Session = Depends(get_db)): with belief_scope("ConnectionsRouter.delete_connection", f"id={connection_id}"): diff --git a/backend/src/api/routes/dashboards/__init__.py b/backend/src/api/routes/dashboards/__init__.py index cf2de5fb..71c20119 100644 --- a/backend/src/api/routes/dashboards/__init__.py +++ b/backend/src/api/routes/dashboards/__init__.py @@ -1,17 +1,17 @@ # #region DashboardsApi [C:5] [TYPE Module] [SEMANTICS api, dashboards, resources, hub] +# # @BRIEF API endpoints for the Dashboard Hub - listing dashboards with Git and task status -# @LAYER API -# @PRE Valid environment configurations exist in ConfigManager. -# @POST Dashboard responses are projected into DashboardsResponse DTO. -# @SIDE_EFFECT Performs external calls to Superset API and potentially Git providers. -# @DATA_CONTRACT Input(env_id, filters) -> Output(DashboardsResponse) -# @INVARIANT All dashboard responses include git_status and last_task metadata +# @LAYER: API # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [ResourceService] # @RELATION DEPENDS_ON -> [SupersetClient] # +# @INVARIANT: All dashboard responses include git_status and last_task metadata # -# +# @PRE: Valid environment configurations exist in ConfigManager. +# @POST: Dashboard responses are projected into DashboardsResponse DTO. +# @SIDE_EFFECT: Performs external calls to Superset API and potentially Git providers. +# @DATA_CONTRACT: Input(env_id, filters) -> Output(DashboardsResponse) # # @TEST_CONTRACT: DashboardsAPI -> { # required_fields: {env_id: string, page: integer, page_size: integer}, diff --git a/backend/src/api/routes/dashboards/_action_routes.py b/backend/src/api/routes/dashboards/_action_routes.py index 277679f6..16be1362 100644 --- a/backend/src/api/routes/dashboards/_action_routes.py +++ b/backend/src/api/routes/dashboards/_action_routes.py @@ -1,10 +1,9 @@ # #region DashboardActionRoutes [C:2] [TYPE Module] [SEMANTICS api, dashboards, actions, routes] # @BRIEF Dashboard action route handlers — migrate, backup. -# @LAYER API +# @LAYER: API # @RELATION DEPENDS_ON -> [DashboardRouter] # @RELATION DEPENDS_ON -> [DashboardSchemas] -# [SECTION: IMPORTS] from fastapi import Depends, HTTPException from src.dependencies import ( @@ -19,20 +18,17 @@ from ._schemas import ( TaskResponse, ) from ._router import router -# [/SECTION] # #region migrate_dashboards [C:2] [TYPE Function] # @BRIEF Trigger bulk migration of dashboards from source to target environment -# @PRE User has permission plugin:migration:execute -# @PRE source_env_id and target_env_id are valid environment IDs -# @PRE dashboard_ids is a non-empty list -# @POST Returns task_id for tracking migration progress -# @POST Task is created and queued for execution -# @PARAM: request (MigrateRequest) - Migration request with source, target, and dashboard IDs -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[MigrationPlugin:execute] -# @RELATION: CALLS ->[TaskManager] +# @PRE: User has permission plugin:migration:execute +# @PRE: source_env_id and target_env_id are valid environment IDs +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking migration progress +# @POST: Task is created and queued for execution +# @RELATION DISPATCHES -> [MigrationPlugin:execute] +# @RELATION CALLS -> [TaskManager] @router.post("/migrate", response_model=TaskResponse) async def migrate_dashboards( request: MigrateRequest, @@ -104,16 +100,14 @@ async def migrate_dashboards( # #region backup_dashboards [C:2] [TYPE Function] # @BRIEF Trigger bulk backup of dashboards with optional cron schedule -# @PRE User has permission plugin:backup:execute -# @PRE env_id is a valid environment ID -# @PRE dashboard_ids is a non-empty list -# @POST Returns task_id for tracking backup progress -# @POST Task is created and queued for execution -# @POST If schedule is provided, a scheduled task is created -# @PARAM: request (BackupRequest) - Backup request with environment and dashboard IDs -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[BackupPlugin:execute] -# @RELATION: CALLS ->[TaskManager] +# @PRE: User has permission plugin:backup:execute +# @PRE: env_id is a valid environment ID +# @PRE: dashboard_ids is a non-empty list +# @POST: Returns task_id for tracking backup progress +# @POST: Task is created and queued for execution +# @POST: If schedule is provided, a scheduled task is created +# @RELATION DISPATCHES -> [BackupPlugin:execute] +# @RELATION CALLS -> [TaskManager] @router.post("/backup", response_model=TaskResponse) async def backup_dashboards( request: BackupRequest, diff --git a/backend/src/api/routes/dashboards/_detail_routes.py b/backend/src/api/routes/dashboards/_detail_routes.py index e4e0d6ec..e1b37e51 100644 --- a/backend/src/api/routes/dashboards/_detail_routes.py +++ b/backend/src/api/routes/dashboards/_detail_routes.py @@ -1,12 +1,11 @@ # #region DashboardDetailRoutes [C:3] [TYPE Module] [SEMANTICS api, dashboards, detail, routes] # @BRIEF Dashboard detail, db-mappings, task history, thumbnail route handlers. -# @LAYER API +# @LAYER: API # @RELATION DEPENDS_ON -> [DashboardRouter] # @RELATION DEPENDS_ON -> [DashboardSchemas] # @RELATION DEPENDS_ON -> [DashboardHelpers] # @RELATION DEPENDS_ON -> [DashboardProjection] -# [SECTION: IMPORTS] from typing import Optional, List, Dict, Any import re from urllib.parse import urlparse @@ -39,18 +38,14 @@ from ._schemas import ( DatabaseMappingsResponse, ) from ._router import router -# [/SECTION] # #region get_database_mappings [C:2] [TYPE Function] # @BRIEF Get database mapping suggestions between source and target environments -# @PRE User has permission plugin:migration:read -# @PRE source_env_id and target_env_id are valid environment IDs -# @POST Returns list of suggested database mappings with confidence scores -# @PARAM: source_env_id (str) - Source environment ID -# @PARAM: target_env_id (str) - Target environment ID -# @RETURN: DatabaseMappingsResponse - List of suggested mappings -# @RELATION: CALLS ->[MappingService:get_suggestions] +# @PRE: User has permission plugin:migration:read +# @PRE: source_env_id and target_env_id are valid environment IDs +# @POST: Returns list of suggested database mappings with confidence scores +# @RELATION CALLS -> [MappingService:get_suggestions] @router.get("/db-mappings", response_model=DatabaseMappingsResponse) async def get_database_mappings( source_env_id: str, @@ -113,8 +108,8 @@ async def get_database_mappings( # #region get_dashboard_detail [C:2] [TYPE Function] # @BRIEF Fetch detailed dashboard info with related charts and datasets -# @PRE env_id must be valid and dashboard ref (slug or id) must exist -# @POST Returns dashboard detail payload for overview page +# @PRE: env_id must be valid and dashboard ref (slug or id) must exist +# @POST: Returns dashboard detail payload for overview page # @RELATION CALLS -> [AsyncSupersetClient] @router.get("/{dashboard_ref}", response_model=DashboardDetailResponse) async def get_dashboard_detail( @@ -158,8 +153,8 @@ async def get_dashboard_detail( # #region get_dashboard_tasks_history [C:2] [TYPE Function] # @BRIEF Returns history of backup and LLM validation tasks for a dashboard. -# @PRE dashboard ref (slug or id) is valid. -# @POST Response contains sorted task history (newest first). +# @PRE: dashboard ref (slug or id) is valid. +# @POST: Response contains sorted task history (newest first). @router.get("/{dashboard_ref}/tasks", response_model=DashboardTaskHistoryResponse) async def get_dashboard_tasks_history( dashboard_ref: str, @@ -260,9 +255,9 @@ async def get_dashboard_tasks_history( # #region get_dashboard_thumbnail [C:3] [TYPE Function] # @BRIEF Proxies Superset dashboard thumbnail with cache support. -# @PRE env_id must exist. -# @POST Returns image bytes or 202 when thumbnail is being prepared by Superset. # @RELATION CALLS -> [AsyncSupersetClient] +# @PRE: env_id must exist. +# @POST: Returns image bytes or 202 when thumbnail is being prepared by Superset. @router.get("/{dashboard_ref}/thumbnail") async def get_dashboard_thumbnail( dashboard_ref: str, diff --git a/backend/src/api/routes/dashboards/_helpers.py b/backend/src/api/routes/dashboards/_helpers.py index e2429592..d1b72d7c 100644 --- a/backend/src/api/routes/dashboards/_helpers.py +++ b/backend/src/api/routes/dashboards/_helpers.py @@ -1,6 +1,6 @@ # #region DashboardHelpers [C:2] [TYPE Module] [SEMANTICS api, dashboards, helpers, resolution] # @BRIEF Basic helper functions for dashboard route handlers — slug resolution, filter normalization. -# @LAYER Infra +# @LAYER: Infra # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [AsyncSupersetClient] @@ -13,8 +13,8 @@ from src.core.logger import logger # #region _find_dashboard_id_by_slug [C:2] [TYPE Function] # @BRIEF Resolve dashboard numeric ID by slug using Superset list endpoint. -# @PRE `dashboard_slug` is non-empty. -# @POST Returns dashboard ID when found, otherwise None. +# @PRE: `dashboard_slug` is non-empty. +# @POST: Returns dashboard ID when found, otherwise None. def _find_dashboard_id_by_slug( client: SupersetClient, dashboard_slug: str, @@ -50,8 +50,8 @@ def _find_dashboard_id_by_slug( # #region _resolve_dashboard_id_from_ref [C:2] [TYPE Function] # @BRIEF Resolve dashboard ID from slug-first reference with numeric fallback. -# @PRE `dashboard_ref` is provided in route path. -# @POST Returns a valid dashboard ID or raises HTTPException(404). +# @PRE: `dashboard_ref` is provided in route path. +# @POST: Returns a valid dashboard ID or raises HTTPException(404). def _resolve_dashboard_id_from_ref( dashboard_ref: str, client: SupersetClient, @@ -76,8 +76,8 @@ def _resolve_dashboard_id_from_ref( # #region _find_dashboard_id_by_slug_async [C:2] [TYPE Function] # @BRIEF Resolve dashboard numeric ID by slug using async Superset list endpoint. -# @PRE dashboard_slug is non-empty. -# @POST Returns dashboard ID when found, otherwise None. +# @PRE: dashboard_slug is non-empty. +# @POST: Returns dashboard ID when found, otherwise None. async def _find_dashboard_id_by_slug_async( client: AsyncSupersetClient, dashboard_slug: str, @@ -113,8 +113,8 @@ async def _find_dashboard_id_by_slug_async( # #region _resolve_dashboard_id_from_ref_async [C:2] [TYPE Function] # @BRIEF Resolve dashboard ID from slug-first reference using async Superset client. -# @PRE dashboard_ref is provided in route path. -# @POST Returns valid dashboard ID or raises HTTPException(404). +# @PRE: dashboard_ref is provided in route path. +# @POST: Returns valid dashboard ID or raises HTTPException(404). async def _resolve_dashboard_id_from_ref_async( dashboard_ref: str, client: AsyncSupersetClient, @@ -138,8 +138,8 @@ async def _resolve_dashboard_id_from_ref_async( # #region _normalize_filter_values [C:2] [TYPE Function] # @BRIEF Normalize query filter values to lower-cased non-empty tokens. -# @PRE values may be None or list of strings. -# @POST Returns trimmed normalized list preserving input order. +# @PRE: values may be None or list of strings. +# @POST: Returns trimmed normalized list preserving input order. def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: if not values: return [] @@ -156,8 +156,8 @@ def _normalize_filter_values(values: Optional[List[str]]) -> List[str]: # #region _dashboard_git_filter_value [C:2] [TYPE Function] # @BRIEF Build comparable git status token for dashboards filtering. -# @PRE dashboard payload may contain git_status or None. -# @POST Returns one of ok|diff|no_repo|error|pending. +# @PRE: dashboard payload may contain git_status or None. +# @POST: Returns one of ok|diff|no_repo|error|pending. def _dashboard_git_filter_value(dashboard: Dict[str, Any]) -> str: git_status = dashboard.get("git_status") or {} sync_status = str(git_status.get("sync_status") or "").strip().upper() diff --git a/backend/src/api/routes/dashboards/_listing_routes.py b/backend/src/api/routes/dashboards/_listing_routes.py index 4f3ac8db..92380b1a 100644 --- a/backend/src/api/routes/dashboards/_listing_routes.py +++ b/backend/src/api/routes/dashboards/_listing_routes.py @@ -1,12 +1,11 @@ # #region DashboardListingRoutes [C:4] [TYPE Module] [SEMANTICS api, dashboards, listing, routes] # @BRIEF Dashboard listing route handler for Dashboard Hub. -# @LAYER API +# @LAYER: API # @RELATION DEPENDS_ON -> [DashboardRouter] # @RELATION DEPENDS_ON -> [DashboardSchemas] # @RELATION DEPENDS_ON -> [DashboardHelpers] # @RELATION DEPENDS_ON -> [DashboardProjection] -# [SECTION: IMPORTS] import os from typing import List, Optional, Dict, Any, Literal from fastapi import Depends, HTTPException, Query @@ -16,10 +15,7 @@ from src.dependencies import ( get_current_user, has_permission, ) from src.core.database import get_db -from src.core.cot_logger import MarkerLogger from src.core.logger import logger, belief_scope - -log = MarkerLogger("DashboardListing") from src.models.auth import User from src.services.profile_service import ProfileService from ._helpers import _normalize_filter_values, _dashboard_git_filter_value @@ -29,23 +25,17 @@ from ._projection import ( ) from ._schemas import EffectiveProfileFilter, DashboardsResponse from ._router import router -# [/SECTION] # #region get_dashboards [C:3] [TYPE Function] # @BRIEF Fetch list of dashboards from a specific environment with Git status and last task status -# @PRE env_id must be a valid environment ID -# @PRE page must be >= 1 if provided -# @PRE page_size must be between 1 and 100 if provided -# @POST Returns a list of dashboards with enhanced metadata and pagination info -# @POST Response includes pagination metadata (page, page_size, total, total_pages) -# @POST Response includes effective profile filter metadata for main dashboards page context -# @PARAM: env_id (str) - The environment ID to fetch dashboards from -# @PARAM: search (Optional[str]) - Filter by title/slug -# @PARAM: page (Optional[int]) - Page number (default: 1) -# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) -# @RETURN: DashboardsResponse - List of dashboards with status metadata -# @RELATION: CALLS ->[get_dashboards_with_status] +# @PRE: env_id must be a valid environment ID +# @PRE: page must be >= 1 if provided +# @PRE: page_size must be between 1 and 100 if provided +# @POST: Returns a list of dashboards with enhanced metadata and pagination info +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) +# @POST: Response includes effective profile filter metadata for main dashboards page context +# @RELATION CALLS -> [get_dashboards_with_status] @router.get("", response_model=DashboardsResponse) async def get_dashboards( env_id: str, @@ -73,16 +63,16 @@ async def get_dashboards( f"page_context={page_context}, applyp={apply_profile_default}, overhide={override_show_all}", ): if page < 1: - log.explore(f"Invalid page: {page}", error="Page must be >= 1") + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page: {page}") raise HTTPException(status_code=400, detail="Page must be >= 1") if page_size < 1 or page_size > 100: - log.explore(f"Invalid page_size: {page_size}", error="Page size must be between 1 and 100") + logger.error(f"[get_dashboards][Coherence:Failed] Invalid page_size: {page_size}") raise HTTPException(status_code=400, detail="Page size must be between 1 and 100") environments = config_manager.get_environments() env = next((e for e in environments if e.id == env_id), None) if not env: - log.explore(f"Environment not found: {env_id}", error="Environment not found") + logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}") raise HTTPException(status_code=404, detail="Environment not found") bound_username: Optional[str] = None @@ -151,8 +141,7 @@ async def get_dashboards( ) except Exception as profile_error: logger.explore( - f"Profile preference unavailable; continuing without profile-default filter", - error=str(profile_error), + f"[EXPLORE] Profile preference unavailable; continuing without profile-default filter: {profile_error}" ) try: @@ -194,7 +183,10 @@ async def get_dashboards( total = page_payload["total"] total_pages = page_payload["total_pages"] except Exception as page_error: - log.explore(f"Page-based fetch failed; using compatibility fallback", error=str(page_error)) + logger.warning( + "[get_dashboards][Action] Page-based fetch failed; using compatibility fallback: %s", + page_error, + ) if can_apply_slug_filter: dashboards = await resource_service.get_dashboards_with_status( env, @@ -249,7 +241,7 @@ async def get_dashboards( if not actor_aliases: actor_aliases = [bound_username] logger.reason( - f"Applying profile actor filter " + "[REASON] Applying profile actor filter " f"(env={env_id}, bound_username={bound_username}, actor_aliases={actor_aliases!r}, " f"dashboards_before={len(dashboards)})" ) @@ -267,7 +259,7 @@ async def get_dashboards( ) if index < max_actor_samples: logger.reflect( - f"Profile actor filter sample " + "[REFLECT] Profile actor filter sample " f"(env={env_id}, dashboard_id={dashboard.get('id')}, " f"bound_username={bound_username!r}, actor_aliases={actor_aliases!r}, " f"owners={owners_value!r}, created_by={created_by_value!r}, " @@ -277,7 +269,7 @@ async def get_dashboards( filtered_dashboards.append(dashboard) logger.reflect( - f"Profile actor filter summary " + "[REFLECT] Profile actor filter summary " f"(env={env_id}, bound_username={bound_username!r}, " f"dashboards_before={len(dashboards)}, dashboards_after={len(filtered_dashboards)})" ) @@ -363,7 +355,10 @@ async def get_dashboards( end_idx = start_idx + page_size paginated_dashboards = dashboards[start_idx:end_idx] - log.reflect(f"Returning {len(paginated_dashboards)} dashboards (page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})") + logger.info( + f"[get_dashboards][Coherence:OK] Returning {len(paginated_dashboards)} dashboards " + f"(page {page}/{total_pages}, total: {total}, profile_filter_applied={effective_profile_filter.applied})" + ) response_dashboards = _project_dashboard_response_items( paginated_dashboards @@ -381,7 +376,9 @@ async def get_dashboards( except HTTPException: raise except Exception as e: - log.explore("Failed to fetch dashboards", error=str(e)) + logger.error( + f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}" + ) raise HTTPException( status_code=503, detail=f"Failed to fetch dashboards: {str(e)}" ) diff --git a/backend/src/api/routes/dashboards/_projection.py b/backend/src/api/routes/dashboards/_projection.py index a8e06d2e..6753e310 100644 --- a/backend/src/api/routes/dashboards/_projection.py +++ b/backend/src/api/routes/dashboards/_projection.py @@ -1,6 +1,6 @@ # #region DashboardProjection [C:2] [TYPE Module] [SEMANTICS api, dashboards, projection, profile, filtering] # @BRIEF Dashboard response projection and profile-filter helpers for Dashboard Hub routes. -# @LAYER Infra +# @LAYER: Infra # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [ProfileService] @@ -142,7 +142,7 @@ def _get_profile_filter_binding( # #region _resolve_profile_actor_aliases [C:2] [TYPE Function] # @BRIEF Resolve stable actor aliases for profile filtering without per-dashboard detail fan-out. -# @SIDE_EFFECT Performs at most one Superset users-lookup request. +# @SIDE_EFFECT: Performs at most one Superset users-lookup request. def _resolve_profile_actor_aliases(env: Any, bound_username: str) -> List[str]: normalized_bound = _normalize_actor_alias_token(bound_username) if not normalized_bound: diff --git a/backend/src/api/routes/dashboards/_schemas.py b/backend/src/api/routes/dashboards/_schemas.py index c91a8cf6..537614db 100644 --- a/backend/src/api/routes/dashboards/_schemas.py +++ b/backend/src/api/routes/dashboards/_schemas.py @@ -1,6 +1,6 @@ # #region DashboardSchemas [C:1] [TYPE Module] [SEMANTICS api, dashboards, schemas, dto] # @BRIEF DTO classes for the Dashboard Hub API. -# @LAYER Infra +# @LAYER: Infra # @RELATION DEPENDS_ON -> [Pydantic] from typing import List, Optional, Dict, Any, Literal diff --git a/backend/src/api/routes/dataset_review.py b/backend/src/api/routes/dataset_review.py index 284ae7f9..e09f82a0 100644 --- a/backend/src/api/routes/dataset_review.py +++ b/backend/src/api/routes/dataset_review.py @@ -1,8 +1,8 @@ # #region DatasetReviewApi [C:3] [TYPE Module] [SEMANTICS dataset_review, api, session_lifecycle, exports, rbac, feature_flags] # @BRIEF Thin facade re-exporting router and public symbols from decomposed dataset review API sub-modules. -# @LAYER API -# @RATIONALE Original 2484-line monolith violated INV_7 (400-line module limit) by 6x. Decomposed into _dependencies (DTOs/guards/serializers) and _routes (handlers). -# @REJECTED Keeping all routes in one file because it exceeded the fractal limit by 6x and accumulated severe structural erosion risk. +# @LAYER: API +# @RATIONALE: Original 2484-line monolith violated INV_7 (400-line module limit) by 6x. Decomposed into _dependencies (DTOs/guards/serializers) and _routes (handlers). +# @REJECTED: Keeping all routes in one file because it exceeded the fractal limit by 6x and accumulated severe structural erosion risk. from src.api.routes.dataset_review_pkg._dependencies import ( # noqa: F401 StartSessionRequest, diff --git a/backend/src/api/routes/dataset_review_pkg/_dependencies.py b/backend/src/api/routes/dataset_review_pkg/_dependencies.py index 957f4f34..7e08e2b4 100644 --- a/backend/src/api/routes/dataset_review_pkg/_dependencies.py +++ b/backend/src/api/routes/dataset_review_pkg/_dependencies.py @@ -1,6 +1,6 @@ # #region DatasetReviewDependencies [C:2] [TYPE Module] # @BRIEF Dependency injection, serialization helpers, and feature-flag guards for dataset review endpoints. -# @LAYER API +# @LAYER: API from __future__ import annotations @@ -12,9 +12,8 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Response, from pydantic import BaseModel, Field from sqlalchemy.orm import Session -from src.core.cot_logger import MarkerLogger from src.core.database import get_db -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.dependencies import ( get_config_manager, get_current_user, @@ -69,8 +68,6 @@ from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionVersionConflictError, ) -log = MarkerLogger("DatasetReviewDeps") - # #region StartSessionRequest [C:1] [TYPE Class] # @BRIEF Request DTO for starting one dataset review session. @@ -417,7 +414,7 @@ def _enforce_session_version(repository, session, expected_version): try: repository.require_session_version(session, expected_version) except DatasetReviewSessionVersionConflictError as exc: - log.explore("Dataset review optimistic-lock conflict detected", payload={"session_id": exc.session_id, "expected_version": exc.expected_version, "actual_version": exc.actual_version}, error="Optimistic lock conflict") + logger.explore("Dataset review optimistic-lock conflict detected", extra={"session_id": exc.session_id, "expected_version": exc.expected_version, "actual_version": exc.actual_version}) raise _build_session_version_conflict_http_exception(exc) from exc return session @@ -653,7 +650,7 @@ def _resolve_candidate_source_version(field, source_id): # #region _update_semantic_field_state [C:3] [TYPE Function] # @BRIEF Apply field-level semantic manual override or candidate acceptance. -# @POST Manual overrides always set manual provenance plus lock. +# @POST: Manual overrides always set manual provenance plus lock. def _update_semantic_field_state(field, request, changed_by): has_manual_override = any(v is not None for v in [request.verbose_name, request.description, request.display_format]) selected_candidate = None diff --git a/backend/src/api/routes/dataset_review_pkg/_routes.py b/backend/src/api/routes/dataset_review_pkg/_routes.py index 03cc00cd..4ee25f82 100644 --- a/backend/src/api/routes/dataset_review_pkg/_routes.py +++ b/backend/src/api/routes/dataset_review_pkg/_routes.py @@ -9,8 +9,7 @@ from typing import Any, List, Optional, Union, cast from fastapi import APIRouter, Depends, HTTPException, Query, Response, status -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.dataset_review import ( @@ -89,8 +88,6 @@ from src.api.routes.dataset_review_pkg._dependencies import ( _build_session_version_conflict_http_exception, ) -log = MarkerLogger("DatasetReviewRoutes") - router = APIRouter(prefix="/api/dataset-orchestration", tags=["Dataset Orchestration"]) @@ -111,17 +108,17 @@ async def list_sessions( current_user: User = Depends(get_current_user), ): with belief_scope("dataset_review.list_sessions"): - log.reason( + logger.reason( "Listing dataset review sessions", - payload={"user_id": current_user.id, "page": page, "page_size": page_size}, + extra={"user_id": current_user.id, "page": page, "page_size": page_size}, ) sessions = repository.list_sessions_for_user(current_user.id) start = (page - 1) * page_size end = start + page_size items = [_serialize_session_summary(s) for s in sessions[start:end]] - log.reflect( + logger.reflect( "Session page assembled", - payload={ + extra={ "user_id": current_user.id, "returned": len(items), "total": len(sessions), @@ -156,9 +153,9 @@ async def start_session( current_user: User = Depends(get_current_user), ): with belief_scope("start_session"): - log.reason( + logger.reason( "Starting dataset review session", - payload={ + extra={ "user_id": current_user.id, "environment_id": request.environment_id, }, @@ -173,7 +170,10 @@ async def start_session( ) ) except ValueError as exc: - log.explore("Session start rejected", error="Session start rejected by orchestrator", payload={"user_id": current_user.id, "error": str(exc)}) + logger.explore( + "Session start rejected", + extra={"user_id": current_user.id, "error": str(exc)}, + ) detail = str(exc) sc = ( status.HTTP_404_NOT_FOUND @@ -181,8 +181,8 @@ async def start_session( else status.HTTP_400_BAD_REQUEST ) raise HTTPException(status_code=sc, detail=detail) from exc - log.reflect( - "Session started", payload={"session_id": result.session.session_id} + logger.reflect( + "Session started", extra={"session_id": result.session.session_id} ) return _serialize_session_summary(result.session) diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 63a5f2ed..6a5e43d4 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -1,21 +1,19 @@ # #region DatasetsApi [C:3] [TYPE Module] [SEMANTICS api, datasets, resources, hub] +# # @BRIEF API endpoints for the Dataset Hub - listing datasets with mapping progress -# @LAYER API -# @INVARIANT All dataset responses include last_task metadata +# @LAYER: API # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [ResourceService] # @RELATION DEPENDS_ON -> [SupersetClient] # -# +# @INVARIANT: All dataset responses include last_task metadata -# [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException, Query from typing import List, Optional from pydantic import BaseModel, Field from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission from ...core.logger import logger, belief_scope from ...core.superset_client import SupersetClient -# [/SECTION] router = APIRouter(prefix="/api/datasets", tags=["Datasets"]) @@ -105,12 +103,9 @@ class TaskResponse(BaseModel): # #region get_dataset_ids [C:3] [TYPE Function] # @BRIEF Fetch list of all dataset IDs from a specific environment (without pagination) -# @PRE env_id must be a valid environment ID -# @POST Returns a list of all dataset IDs -# @PARAM: env_id (str) - The environment ID to fetch datasets from -# @PARAM: search (Optional[str]) - Filter by table name -# @RETURN: List[int] - List of dataset IDs -# @RELATION: CALLS ->[get_datasets_with_status] +# @PRE: env_id must be a valid environment ID +# @POST: Returns a list of all dataset IDs +# @RELATION CALLS -> [get_datasets_with_status] @router.get("/ids") async def get_dataset_ids( env_id: str, @@ -156,17 +151,12 @@ async def get_dataset_ids( # #region get_datasets [C:3] [TYPE Function] # @BRIEF Fetch list of datasets from a specific environment with mapping progress -# @PRE env_id must be a valid environment ID -# @PRE page must be >= 1 if provided -# @PRE page_size must be between 1 and 100 if provided -# @POST Returns a list of datasets with enhanced metadata and pagination info -# @POST Response includes pagination metadata (page, page_size, total, total_pages) -# @PARAM: env_id (str) - The environment ID to fetch datasets from -# @PARAM: search (Optional[str]) - Filter by table name -# @PARAM: page (Optional[int]) - Page number (default: 1) -# @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) -# @RETURN: DatasetsResponse - List of datasets with status metadata -# @RELATION: CALLS ->[get_datasets_with_status] +# @PRE: env_id must be a valid environment ID +# @PRE: page must be >= 1 if provided +# @PRE: page_size must be between 1 and 100 if provided +# @POST: Returns a list of datasets with enhanced metadata and pagination info +# @POST: Response includes pagination metadata (page, page_size, total, total_pages) +# @RELATION CALLS -> [get_datasets_with_status] @router.get("", response_model=DatasetsResponse) async def get_datasets( env_id: str, @@ -245,15 +235,13 @@ class MapColumnsRequest(BaseModel): # #region map_columns [C:3] [TYPE Function] # @BRIEF Trigger bulk column mapping for datasets -# @PRE User has permission plugin:mapper:execute -# @PRE env_id is a valid environment ID -# @PRE dataset_ids is a non-empty list -# @POST Returns task_id for tracking mapping progress -# @POST Task is created and queued for execution -# @PARAM: request (MapColumnsRequest) - Mapping request with environment and dataset IDs -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[MapperPlugin] -# @RELATION: CALLS ->[create_task] +# @PRE: User has permission plugin:mapper:execute +# @PRE: env_id is a valid environment ID +# @PRE: dataset_ids is a non-empty list +# @POST: Returns task_id for tracking mapping progress +# @POST: Task is created and queued for execution +# @RELATION DISPATCHES -> [MapperPlugin] +# @RELATION CALLS -> [create_task] @router.post("/map-columns", response_model=TaskResponse) async def map_columns( request: MapColumnsRequest, @@ -315,15 +303,13 @@ class GenerateDocsRequest(BaseModel): # #region generate_docs [C:3] [TYPE Function] # @BRIEF Trigger bulk documentation generation for datasets -# @PRE User has permission plugin:llm_analysis:execute -# @PRE env_id is a valid environment ID -# @PRE dataset_ids is a non-empty list -# @POST Returns task_id for tracking documentation generation progress -# @POST Task is created and queued for execution -# @PARAM: request (GenerateDocsRequest) - Documentation generation request -# @RETURN: TaskResponse - Task ID for tracking -# @RELATION: DISPATCHES ->[DocumentationPlugin] -# @RELATION: CALLS ->[create_task] +# @PRE: User has permission plugin:llm_analysis:execute +# @PRE: env_id is a valid environment ID +# @PRE: dataset_ids is a non-empty list +# @POST: Returns task_id for tracking documentation generation progress +# @POST: Task is created and queued for execution +# @RELATION DISPATCHES -> [DocumentationPlugin] +# @RELATION CALLS -> [create_task] @router.post("/generate-docs", response_model=TaskResponse) async def generate_docs( request: GenerateDocsRequest, @@ -370,13 +356,10 @@ async def generate_docs( # #region get_dataset_detail [C:3] [TYPE Function] # @BRIEF Get detailed dataset information including columns and linked dashboards -# @PRE env_id is a valid environment ID -# @PRE dataset_id is a valid dataset ID -# @POST Returns detailed dataset info with columns and linked dashboards -# @PARAM: env_id (str) - The environment ID -# @PARAM: dataset_id (int) - The dataset ID -# @RETURN: DatasetDetailResponse - Detailed dataset information -# @RELATION: CALLS ->[SupersetClientGetDatasetDetail] +# @PRE: env_id is a valid environment ID +# @PRE: dataset_id is a valid dataset ID +# @POST: Returns detailed dataset info with columns and linked dashboards +# @RELATION CALLS -> [SupersetClientGetDatasetDetail] @router.get("/{dataset_id}", response_model=DatasetDetailResponse) async def get_dataset_detail( env_id: str, diff --git a/backend/src/api/routes/environments.py b/backend/src/api/routes/environments.py index 129233d6..006aed04 100644 --- a/backend/src/api/routes/environments.py +++ b/backend/src/api/routes/environments.py @@ -1,28 +1,26 @@ # #region EnvironmentsApi [C:3] [TYPE Module] [SEMANTICS api, environments, superset, databases] +# # @BRIEF API endpoints for listing environments and their databases. -# @LAYER API -# @INVARIANT Environment IDs must exist in the configuration. +# @LAYER: API # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [SupersetClient] # -# +# @INVARIANT: Environment IDs must exist in the configuration. -# [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException from typing import List, Optional from ...dependencies import get_config_manager, get_scheduler_service, has_permission from ...core.superset_client import SupersetClient from pydantic import BaseModel, Field from ...core.logger import belief_scope -# [/SECTION] router = APIRouter(prefix="/api/environments", tags=["Environments"]) # #region _normalize_superset_env_url [TYPE Function] # @BRIEF Canonicalize Superset environment URL to base host/path without trailing /api/v1. -# @PRE raw_url can be empty. -# @POST Returns normalized base URL. +# @PRE: raw_url can be empty. +# @POST: Returns normalized base URL. def _normalize_superset_env_url(raw_url: str) -> str: normalized = str(raw_url or "").strip().rstrip("/") if normalized.lower().endswith("/api/v1"): @@ -55,10 +53,9 @@ class DatabaseResponse(BaseModel): # #region get_environments [TYPE Function] [SEMANTICS list, environments, config] # @BRIEF List all configured environments. -# @LAYER API -# @PRE config_manager is injected via Depends. -# @POST Returns a list of EnvironmentResponse objects. -# @RETURN List[EnvironmentResponse] +# @LAYER: API +# @PRE: config_manager is injected via Depends. +# @POST: Returns a list of EnvironmentResponse objects. @router.get("", response_model=List[EnvironmentResponse]) async def get_environments( config_manager=Depends(get_config_manager), @@ -93,11 +90,9 @@ async def get_environments( # #region update_environment_schedule [TYPE Function] [SEMANTICS update, schedule, backup, environment] # @BRIEF Update backup schedule for an environment. -# @LAYER API -# @PRE Environment id exists, schedule is valid ScheduleSchema. -# @POST Backup schedule updated and scheduler reloaded. -# @PARAM: id (str) - The environment ID. -# @PARAM: schedule (ScheduleSchema) - The new schedule. +# @LAYER: API +# @PRE: Environment id exists, schedule is valid ScheduleSchema. +# @POST: Backup schedule updated and scheduler reloaded. @router.put("/{id}/schedule") async def update_environment_schedule( id: str, @@ -126,11 +121,9 @@ async def update_environment_schedule( # #region get_environment_databases [TYPE Function] [SEMANTICS fetch, databases, superset, environment] # @BRIEF Fetch the list of databases from a specific environment. -# @LAYER API -# @PRE Environment id exists. -# @POST Returns a list of database summaries from the environment. -# @PARAM: id (str) - The environment ID. -# @RETURN: List[Dict] - List of databases. +# @LAYER: API +# @PRE: Environment id exists. +# @POST: Returns a list of database summaries from the environment. @router.get("/{id}/databases") async def get_environment_databases( id: str, diff --git a/backend/src/api/routes/git/__init__.py b/backend/src/api/routes/git/__init__.py index 8d38e78b..d7fc2d09 100644 --- a/backend/src/api/routes/git/__init__.py +++ b/backend/src/api/routes/git/__init__.py @@ -1,6 +1,6 @@ # #region GitPackage [C:3] [TYPE Module] [SEMANTICS git, package, re-exports] # @BRIEF Package root for decomposed git routes. Re-exports all public symbols from submodules. -# @LAYER API +# @LAYER: API # @RELATION USES -> [GitRouter, GitDeps, GitHelpers, GitConfigRoutes, GitGiteaRoutes, # GitRepoRoutes, GitRepoOperationsRoutes, GitRepoLifecycleRoutes, # GitMergeRoutes, GitEnvironmentRoutes] diff --git a/backend/src/api/routes/git/_config_routes.py b/backend/src/api/routes/git/_config_routes.py index 57e81866..15969a0e 100644 --- a/backend/src/api/routes/git/_config_routes.py +++ b/backend/src/api/routes/git/_config_routes.py @@ -1,6 +1,6 @@ # #region GitConfigRoutes [C:2] [TYPE Module] [SEMANTICS git, config, routes, crud] # @BRIEF FastAPI endpoints for Git server configuration CRUD and connection testing. -# @LAYER API +# @LAYER: API from typing import List diff --git a/backend/src/api/routes/git/_deps.py b/backend/src/api/routes/git/_deps.py index 5d3ca60d..c4e97571 100644 --- a/backend/src/api/routes/git/_deps.py +++ b/backend/src/api/routes/git/_deps.py @@ -1,7 +1,7 @@ # #region GitDeps [C:1] [TYPE Module] [SEMANTICS git, deps, service-locator] # @BRIEF Shared dependency wiring for monkeypatch-safe git_service access and constants. -# @LAYER API -# @INVARIANT get_git_service() resolves from sys.modules at call time so test monkeypatching +# @LAYER: API +# @INVARIANT: get_git_service() resolves from sys.modules at call time so test monkeypatching # of git_routes.git_service (the __init__ attribute) takes effect across all submodules. import sys diff --git a/backend/src/api/routes/git/_environment_routes.py b/backend/src/api/routes/git/_environment_routes.py index 5e639215..8c5c5435 100644 --- a/backend/src/api/routes/git/_environment_routes.py +++ b/backend/src/api/routes/git/_environment_routes.py @@ -1,6 +1,6 @@ # #region GitEnvironmentRoutes [C:2] [TYPE Module] [SEMANTICS git, environments, routes] # @BRIEF FastAPI endpoint for listing deployment environments. -# @LAYER API +# @LAYER: API from typing import List diff --git a/backend/src/api/routes/git/_gitea_routes.py b/backend/src/api/routes/git/_gitea_routes.py index 90b939db..6577bd60 100644 --- a/backend/src/api/routes/git/_gitea_routes.py +++ b/backend/src/api/routes/git/_gitea_routes.py @@ -1,6 +1,6 @@ # #region GitGiteaRoutes [C:2] [TYPE Module] [SEMANTICS git, gitea, routes, repositories] # @BRIEF FastAPI endpoints for Gitea-specific repository operations. -# @LAYER API +# @LAYER: API from typing import List diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py index 639226a7..d70c9029 100644 --- a/backend/src/api/routes/git/_helpers.py +++ b/backend/src/api/routes/git/_helpers.py @@ -1,6 +1,6 @@ # #region GitHelpers [C:3] [TYPE Module] [SEMANTICS git, helpers, resolution, identity] # @BRIEF Shared helper functions for Git route modules. -# @LAYER API +# @LAYER: API # @RELATION USES -> [GitDeps] # @RELATION USES -> [SupersetClient] # @RELATION USES -> [UserDashboardPreference] @@ -11,9 +11,7 @@ from typing import Optional from fastapi import HTTPException from sqlalchemy.orm import Session -from src.core.cot_logger import MarkerLogger - -log = MarkerLogger("GitHelpers") +from src.core.logger import logger from src.core.superset_client import SupersetClient from src.dependencies import get_config_manager from src.models.auth import User @@ -25,7 +23,7 @@ from ._deps import get_git_service, MAX_REPOSITORY_STATUS_BATCH # #region _build_no_repo_status_payload [C:1] [TYPE Function] # @BRIEF Build a consistent status payload for dashboards without initialized repositories. -# @POST Returns a stable payload compatible with frontend repository status parsing. +# @POST: Returns a stable payload compatible with frontend repository status parsing. def _build_no_repo_status_payload() -> dict: return { "is_dirty": False, @@ -47,24 +45,24 @@ def _build_no_repo_status_payload() -> dict: # #region _handle_unexpected_git_route_error [C:1] [TYPE Function] # @BRIEF Convert unexpected route-level exceptions to stable 500 API responses. -# @PRE `error` is a non-HTTPException instance. -# @POST Raises HTTPException(500) with route-specific context. +# @PRE: `error` is a non-HTTPException instance. +# @POST: Raises HTTPException(500) with route-specific context. def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: - log.explore(f"{route_name} failed: {error}", error=str(error)) + logger.error(f"[{route_name}][Coherence:Failed] {error}") raise HTTPException(status_code=500, detail=f"{route_name} failed: {str(error)}") # #endregion _handle_unexpected_git_route_error # #region _resolve_repository_status [C:2] [TYPE Function] # @BRIEF Resolve repository status for one dashboard with graceful NO_REPO semantics. -# @PRE `dashboard_id` is a valid integer. -# @POST Returns standard status payload or `NO_REPO` payload when repository path is absent. +# @PRE: `dashboard_id` is a valid integer. +# @POST: Returns standard status payload or `NO_REPO` payload when repository path is absent. def _resolve_repository_status(dashboard_id: int) -> dict: git_service = get_git_service() repo_path = git_service._get_repo_path(dashboard_id) if not os.path.exists(repo_path): - log.reason( - f"Repository is not initialized for dashboard {dashboard_id}" + logger.debug( + f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" ) return _build_no_repo_status_payload() @@ -72,8 +70,8 @@ def _resolve_repository_status(dashboard_id: int) -> dict: return git_service.get_status(dashboard_id) except HTTPException as e: if e.status_code == 404: - log.reason( - f"Repository is not initialized for dashboard {dashboard_id}" + logger.debug( + f"[get_repository_status][Action] Repository is not initialized for dashboard {dashboard_id}" ) return _build_no_repo_status_payload() raise @@ -289,7 +287,11 @@ def _resolve_current_user_git_identity( .first() ) except Exception as resolve_error: - log.explore(f"Failed to load profile preference for user {user_id}: {resolve_error}", error=str(resolve_error)) + logger.warning( + "[_resolve_current_user_git_identity][Action] Failed to load profile preference for user %s: %s", + user_id, + resolve_error, + ) return None if not preference: diff --git a/backend/src/api/routes/git/_merge_routes.py b/backend/src/api/routes/git/_merge_routes.py index ca73e1bc..ee3a39fb 100644 --- a/backend/src/api/routes/git/_merge_routes.py +++ b/backend/src/api/routes/git/_merge_routes.py @@ -1,6 +1,6 @@ # #region GitMergeRoutes [C:3] [TYPE Module] [SEMANTICS git, merge, routes, conflicts] # @BRIEF FastAPI endpoints for merge operations (status, conflicts, resolve, abort, continue). -# @LAYER API +# @LAYER: API from typing import List, Optional diff --git a/backend/src/api/routes/git/_repo_lifecycle_routes.py b/backend/src/api/routes/git/_repo_lifecycle_routes.py index 547542d5..02f64898 100644 --- a/backend/src/api/routes/git/_repo_lifecycle_routes.py +++ b/backend/src/api/routes/git/_repo_lifecycle_routes.py @@ -1,6 +1,6 @@ # #region GitRepoLifecycleRoutes [C:3] [TYPE Module] [SEMANTICS git, lifecycle, sync, promote, deploy] # @BRIEF FastAPI endpoints for Git lifecycle operations (sync, promote, deploy). -# @LAYER API +# @LAYER: API import typing from typing import Optional diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py index 173005b7..e9a4904f 100644 --- a/backend/src/api/routes/git/_repo_operations_routes.py +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -1,6 +1,6 @@ # #region GitRepoOperationsRoutes [C:3] [TYPE Module] [SEMANTICS git, repository, operations, commit, push, pull, status] # @BRIEF FastAPI endpoints for Git repository operations (commit, push, pull, status, diff, history). -# @LAYER API +# @LAYER: API from typing import List, Optional @@ -8,10 +8,7 @@ from fastapi import Depends, HTTPException from sqlalchemy.orm import Session from src.core.database import get_db -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope - -log = MarkerLogger("GitRepoOps") +from src.core.logger import belief_scope, logger from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.git import GitRepository, GitServerConfig @@ -131,13 +128,23 @@ async def pull_changes( config_url = config_row.url config_provider = config_row.provider except Exception as diagnostics_error: - log.explore(f"Failed to load repository binding diagnostics for dashboard {dashboard_id}: {diagnostics_error}", error=str(diagnostics_error)) - log.reason( - f"Route diagnostics dashboard_ref={dashboard_ref} env_id={env_id} resolved_dashboard_id={dashboard_id} " - f"binding_exists={bool(db_repo)} binding_local_path={(db_repo.local_path if db_repo else None)} " - f"binding_remote_url={(db_repo.remote_url if db_repo else None)} " - f"binding_config_id={(db_repo.config_id if db_repo else None)} " - f"config_provider={config_provider} config_url={config_url}" + logger.warning( + "[pull_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", + dashboard_id, + diagnostics_error, + ) + logger.info( + "[pull_changes][Action] Route diagnostics dashboard_ref=%s env_id=%s resolved_dashboard_id=%s " + "binding_exists=%s binding_local_path=%s binding_remote_url=%s binding_config_id=%s config_provider=%s config_url=%s", + dashboard_ref, + env_id, + dashboard_id, + bool(db_repo), + (db_repo.local_path if db_repo else None), + (db_repo.remote_url if db_repo else None), + (db_repo.config_id if db_repo else None), + config_provider, + config_url, ) _apply_git_identity_from_profile(dashboard_id, db, current_user) _gs.pull_changes(dashboard_id) @@ -183,7 +190,11 @@ async def get_repository_status_batch( with belief_scope("get_repository_status_batch"): dashboard_ids = list(dict.fromkeys(request.dashboard_ids)) if len(dashboard_ids) > MAX_REPOSITORY_STATUS_BATCH: - log.explore(f"Batch size {len(dashboard_ids)} exceeds limit {MAX_REPOSITORY_STATUS_BATCH}. Truncating request.", error="Batch size exceeds limit") + logger.warning( + "[get_repository_status_batch][Action] Batch size %s exceeds limit %s. Truncating request.", + len(dashboard_ids), + MAX_REPOSITORY_STATUS_BATCH, + ) dashboard_ids = dashboard_ids[:MAX_REPOSITORY_STATUS_BATCH] statuses = {} @@ -197,7 +208,9 @@ async def get_repository_status_batch( "sync_status": "ERROR", } except Exception as e: - log.explore(f"Failed for dashboard {dashboard_id}: {e}", error=str(e)) + logger.error( + f"[get_repository_status_batch][Coherence:Failed] Failed for dashboard {dashboard_id}: {e}" + ) statuses[str(dashboard_id)] = { **_build_no_repo_status_payload(), "sync_state": "ERROR", diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py index 5ad37ef0..315dc9af 100644 --- a/backend/src/api/routes/git/_repo_routes.py +++ b/backend/src/api/routes/git/_repo_routes.py @@ -1,6 +1,6 @@ # #region GitRepoRoutes [C:3] [TYPE Module] [SEMANTICS git, repository, routes, init, branches] # @BRIEF FastAPI endpoints for core Git repository operations (init, binding, branches, checkout). -# @LAYER API +# @LAYER: API from typing import List, Optional @@ -8,10 +8,7 @@ from fastapi import Depends, HTTPException from sqlalchemy.orm import Session from src.core.database import get_db -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope - -log = MarkerLogger("GitRepoRoutes") +from src.core.logger import belief_scope, logger from src.dependencies import get_config_manager, get_current_user, has_permission from src.models.auth import User from src.models.git import GitRepository, GitServerConfig @@ -64,7 +61,9 @@ async def init_repository( raise HTTPException(status_code=404, detail="Git configuration not found") try: - log.reason(f"Initializing repo for dashboard {dashboard_id}") + logger.info( + f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}" + ) _gs.init_repo( dashboard_id, init_data.remote_url, @@ -95,10 +94,15 @@ async def init_repository( db_repo.current_branch = "dev" db.commit() + logger.info( + f"[init_repository][Coherence:OK] Repository initialized for dashboard {dashboard_id}" + ) return {"status": "success", "message": "Repository initialized"} except Exception as e: db.rollback() - log.explore("Failed to init repository", error=str(e)) + logger.error( + f"[init_repository][Coherence:Failed] Failed to init repository: {e}" + ) if isinstance(e, HTTPException): raise _handle_unexpected_git_route_error("init_repository", e) diff --git a/backend/src/api/routes/git/_router.py b/backend/src/api/routes/git/_router.py index 83e4e7e2..36fa0267 100644 --- a/backend/src/api/routes/git/_router.py +++ b/backend/src/api/routes/git/_router.py @@ -1,6 +1,6 @@ # #region GitRouter [C:1] [TYPE Module] [SEMANTICS git, routes, fastapi, router] # @BRIEF Shared APIRouter for all Git route modules. -# @LAYER API +# @LAYER: API from fastapi import APIRouter diff --git a/backend/src/api/routes/git_schemas.py b/backend/src/api/routes/git_schemas.py index e0eb1c67..1d3e7c0f 100644 --- a/backend/src/api/routes/git_schemas.py +++ b/backend/src/api/routes/git_schemas.py @@ -1,10 +1,10 @@ # #region GitSchemas [C:1] [TYPE Module] [SEMANTICS git, schemas, pydantic, api, contracts] +# # @BRIEF Defines Pydantic models for the Git integration API layer. -# @LAYER API -# @INVARIANT All schemas must be compatible with the FastAPI router. -# @RELATION DEPENDS_ON -> [backend.src.models.git] -# +# @LAYER: API +# @RELATION DEPENDS_ON -> backend.src.models.git # +# @INVARIANT: All schemas must be compatible with the FastAPI router. from pydantic import BaseModel, Field from typing import Any, Dict, List, Optional diff --git a/backend/src/api/routes/health.py b/backend/src/api/routes/health.py index 6aaab40a..d536bfe9 100644 --- a/backend/src/api/routes/health.py +++ b/backend/src/api/routes/health.py @@ -1,7 +1,7 @@ # #region health_router [C:3] [TYPE Module] [SEMANTICS health, monitoring, dashboards] # @BRIEF API endpoints for dashboard health monitoring and status aggregation. -# @LAYER UI/API -# @RELATION DEPENDS_ON -> [health_service] +# @LAYER: UI/API +# @RELATION DEPENDS_ON -> health_service from fastapi import APIRouter, Depends, Query, HTTPException, status from typing import List, Optional @@ -15,9 +15,9 @@ router = APIRouter(prefix="/api/health", tags=["Health"]) # #region get_health_summary [TYPE Function] # @BRIEF Get aggregated health status for all dashboards. -# @PRE Caller has read permission for dashboard health view. -# @POST Returns HealthSummaryResponse. -# @RELATION CALLS -> [backend.src.services.health_service.HealthService] +# @PRE: Caller has read permission for dashboard health view. +# @POST: Returns HealthSummaryResponse. +# @RELATION CALLS -> backend.src.services.health_service.HealthService @router.get("/summary", response_model=HealthSummaryResponse) async def get_health_summary( environment_id: Optional[str] = Query(None), @@ -38,9 +38,9 @@ async def get_health_summary( # #region delete_health_report [TYPE Function] # @BRIEF Delete one persisted dashboard validation report from health summary. -# @PRE Caller has write permission for tasks/report maintenance. -# @POST Validation record is removed; linked task/logs are cleaned when available. -# @RELATION CALLS -> [backend.src.services.health_service.HealthService] +# @PRE: Caller has write permission for tasks/report maintenance. +# @POST: Validation record is removed; linked task/logs are cleaned when available. +# @RELATION CALLS -> backend.src.services.health_service.HealthService @router.delete("/summary/{record_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_health_report( record_id: str, diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index bd41066a..d35f37a8 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -1,6 +1,6 @@ # #region LlmRoutes [C:3] [TYPE Module] [SEMANTICS api, routes, llm] # @BRIEF API routes for LLM provider configuration and management. -# @LAYER UI (API) +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderConfig] # @RELATION DEPENDS_ON -> [get_current_user] @@ -8,18 +8,10 @@ from fastapi import APIRouter, Depends, HTTPException, status from typing import List, Optional -import httpx -from ...core.cot_logger import MarkerLogger - -log = MarkerLogger("LlmRoutes") +from ...core.logger import logger from ...schemas.auth import User from ...dependencies import get_current_user as get_current_active_user -from ...plugins.llm_analysis.models import ( - LLMProviderConfig, - LLMProviderType, - FetchModelsRequest, - FetchModelsResponse, -) +from ...plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType from ...services.llm_provider import LLMProviderService from ...core.database import get_db from sqlalchemy.orm import Session @@ -32,8 +24,8 @@ router = APIRouter(tags=["LLM"]) # #region _is_valid_runtime_api_key [TYPE Function] # @BRIEF Validate decrypted runtime API key presence/shape. -# @PRE value can be None. -# @POST Returns True only for non-placeholder key. +# @PRE: value can be None. +# @POST: Returns True only for non-placeholder key. # @RELATION BINDS_TO -> [LlmRoutes] def _is_valid_runtime_api_key(value: Optional[str]) -> bool: key = (value or "").strip() @@ -47,30 +39,10 @@ def _is_valid_runtime_api_key(value: Optional[str]) -> bool: # #endregion _is_valid_runtime_api_key -# #region _build_provider_auth_headers [TYPE Function] -# @BRIEF Build auth headers for provider model listing based on provider type. -# @PRE provider_type is a valid LLMProviderType. -# @POST Returns dict of HTTP headers for the provider's model list API. -def _build_provider_auth_headers(api_key: Optional[str], provider_type: LLMProviderType) -> dict: - headers = {} - if not api_key: - return headers - if provider_type == LLMProviderType.KILO: - headers["Authentication"] = f"Bearer {api_key}" - headers["X-API-Key"] = api_key - else: - # OpenAI, OpenRouter — standard Bearer token - headers["Authorization"] = f"Bearer {api_key}" - return headers - - -# #endregion _build_provider_auth_headers - - # #region get_providers [TYPE Function] # @BRIEF Retrieve all LLM provider configurations. -# @PRE User is authenticated. -# @POST Returns list of LLMProviderConfig. +# @PRE: User is authenticated. +# @POST: Returns list of LLMProviderConfig. # @RELATION CALLS -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderConfig] @router.get("/providers", response_model=List[LLMProviderConfig]) @@ -80,8 +52,8 @@ async def get_providers( """ Get all LLM provider configurations. """ - log.reason( - f"Fetching providers for user: {current_user.username}" + logger.info( + f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}" ) service = LLMProviderService(db) providers = service.get_all_providers() @@ -104,8 +76,8 @@ async def get_providers( # #region get_llm_status [TYPE Function] # @BRIEF Returns whether LLM runtime is configured for dashboard validation. -# @PRE User is authenticated. -# @POST configured=true only when an active provider with valid decrypted key exists. +# @PRE: User is authenticated. +# @POST: configured=true only when an active provider with valid decrypted key exists. # @RELATION CALLS -> [LLMProviderService] # @RELATION CALLS -> [_is_valid_runtime_api_key] @router.get("/status") @@ -175,8 +147,8 @@ async def get_llm_status( # #region create_provider [TYPE Function] # @BRIEF Create a new LLM provider configuration. -# @PRE User is authenticated and has admin permissions. -# @POST Returns the created LLMProviderConfig. +# @PRE: User is authenticated and has admin permissions. +# @POST: Returns the created LLMProviderConfig. # @RELATION CALLS -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderConfig] @router.post( @@ -208,8 +180,8 @@ async def create_provider( # #region update_provider [TYPE Function] # @BRIEF Update an existing LLM provider configuration. -# @PRE User is authenticated and has admin permissions. -# @POST Returns the updated LLMProviderConfig. +# @PRE: User is authenticated and has admin permissions. +# @POST: Returns the updated LLMProviderConfig. # @RELATION CALLS -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMProviderConfig] @router.put("/providers/{provider_id}", response_model=LLMProviderConfig) @@ -243,8 +215,8 @@ async def update_provider( # #region delete_provider [TYPE Function] # @BRIEF Delete an LLM provider configuration. -# @PRE User is authenticated and has admin permissions. -# @POST Returns success status. +# @PRE: User is authenticated and has admin permissions. +# @POST: Returns success status. # @RELATION CALLS -> [LLMProviderService] @router.delete("/providers/{provider_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_provider( @@ -264,129 +236,10 @@ async def delete_provider( # #endregion delete_provider -# #region fetch_models_for_provider [TYPE Function] -# @BRIEF Fetch available models from a provider's API endpoint. -# @PRE valid base_url and optional api_key/provider_id. -# @POST Returns list of model IDs sorted alphabetically. -# @SIDE_EFFECT Makes HTTP GET to multiple possible model endpoints. -@router.post("/providers/fetch-models", response_model=FetchModelsResponse) -async def fetch_models_for_provider( - request: FetchModelsRequest, - current_user: User = Depends(get_current_active_user), - db: Session = Depends(get_db), -): - """ - Fetch available models from a provider's models endpoint. - - Resolves API key in priority order: - 1. Direct api_key from request body - 2. Decrypted api_key from DB via provider_id (for existing providers) - 3. No auth (for local providers without auth) - - Tries multiple paths in order: - 1. {base_url}/models (OpenAI-compatible, stripped of /v1) - 2. {base_url}/v1/models (OpenAI-compatible, with /v1) - 3. {base_url}/api/tags (Ollama format) - Returns empty list if none respond — user can enter model name manually. - """ - log.reason( - f"Fetching models for {request.base_url}" - ) - - # Resolve API key: prefer direct, fall back to stored key via provider_id - api_key = request.api_key - if not api_key and request.provider_id: - try: - svc = LLMProviderService(db) - api_key = svc.get_decrypted_api_key(request.provider_id) - except Exception: - log.explore(f"Could not resolve API key for provider {request.provider_id}", error="API key resolution failed") - - base = request.base_url.rstrip("/") - - # Collect possible model list endpoints - candidates = [] - - # If base ends with /v1, try both /models (after stripping /v1) and /v1/models - if base.endswith("/v1"): - candidates.append(base[:-3] + "/models") # base without /v1 + /models - candidates.append(base + "/models") # base with /v1 + /models (e.g. /v1/models) - else: - candidates.append(base + "/models") # /models - candidates.append(base + "/v1/models") # /v1/models - - candidates.append(base + "/api/tags") # Ollama format - - headers = _build_provider_auth_headers(api_key, request.provider_type) - last_error = None - - try: - async with httpx.AsyncClient(timeout=10.0) as client: - for url in candidates: - try: - response = await client.get(url, headers=headers) - - if response.status_code != 200: - if response.status_code != 404: - last_error = f"HTTP {response.status_code} from {url}: {response.text[:200]}" - continue - - body = response.json() - raw = body.get("data") if isinstance(body, dict) else body - - # Try OpenAI format: [{"id": "model-name", ...}, ...] or list of strings - if isinstance(raw, list): - models = sorted( - m.get("id", str(m)) if isinstance(m, dict) else str(m) - for m in raw - ) - if models: - log.reason( - f"Found {len(models)} models at {url}" - ) - return FetchModelsResponse( - models=models, total=len(models) - ) - - # Try Ollama format: {"models": [{"name": "model-name", ...}, ...]} - if isinstance(body, dict): - models_list = body.get("models", []) - if isinstance(models_list, list): - models = sorted( - m.get("name", str(m)) if isinstance(m, dict) else str(m) - for m in models_list - ) - if models: - log.reason( - f"Found {len(models)} Ollama models at {url}" - ) - return FetchModelsResponse( - models=models, total=len(models) - ) - - except httpx.TimeoutException: - last_error = f"Timeout at {url}" - except httpx.ConnectError: - last_error = f"Cannot connect at {url}" - except Exception as e: - last_error = f"Error at {url}: {str(e)[:100]}" - - # All endpoints failed — return empty list with no error (user can type manually) - log.explore(f"No model endpoint responded for {request.base_url}. Last error: {last_error}", error=f"No model endpoint responded for {request.base_url}") - return FetchModelsResponse(models=[], total=0) - - except Exception as e: - log.explore(f"Unexpected error for {request.base_url}: {e}", error=str(e)) - return FetchModelsResponse(models=[], total=0) - - -# #endregion fetch_models_for_provider - - # #region test_connection [TYPE Function] # @BRIEF Test connection to an LLM provider. -# @PRE User is authenticated. -# @POST Returns success status and message. +# @PRE: User is authenticated. +# @POST: Returns success status and message. # @RELATION CALLS -> [LLMProviderService] # @RELATION DEPENDS_ON -> [LLMClient] @router.post("/providers/{provider_id}/test") @@ -395,8 +248,8 @@ async def test_connection( current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db), ): - log.reason( - f"Testing connection for provider_id: {provider_id}" + logger.info( + f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}" ) """ Test connection to an LLM provider. @@ -412,7 +265,9 @@ async def test_connection( # Check if API key was successfully decrypted if not api_key: - log.explore(f"Failed to decrypt API key for provider {provider_id}", error="API key decryption failed") + logger.error( + f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}" + ) raise HTTPException( status_code=500, detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.", @@ -437,8 +292,8 @@ async def test_connection( # #region test_provider_config [TYPE Function] # @BRIEF Test connection with a provided configuration (not yet saved). -# @PRE User is authenticated. -# @POST Returns success status and message. +# @PRE: User is authenticated. +# @POST: Returns success status and message. # @RELATION DEPENDS_ON -> [LLMClient] # @RELATION DEPENDS_ON -> [LLMProviderConfig] @router.post("/providers/test") @@ -450,8 +305,8 @@ async def test_provider_config( """ from ...plugins.llm_analysis.service import LLMClient - log.reason( - f"Testing config for {config.name}" + logger.info( + f"[llm_routes][test_provider_config][Action] Testing config for {config.name}" ) # Check if API key is provided diff --git a/backend/src/api/routes/mappings.py b/backend/src/api/routes/mappings.py index 0d3ba51f..215eff76 100644 --- a/backend/src/api/routes/mappings.py +++ b/backend/src/api/routes/mappings.py @@ -1,14 +1,13 @@ # #region MappingsApi [C:3] [TYPE Module] [SEMANTICS api, mappings, database, fuzzy-matching] +# # @BRIEF API endpoints for managing database mappings and getting suggestions. -# @LAYER API -# @INVARIANT Mappings are persisted in the SQLite database. +# @LAYER: API # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [DatabaseModule] # @RELATION DEPENDS_ON -> [mapping_service] # -# +# @INVARIANT: Mappings are persisted in the SQLite database. -# [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException from sqlalchemy.orm import Session from typing import List, Optional @@ -17,7 +16,6 @@ from ...dependencies import get_config_manager, has_permission from ...core.database import get_db from ...models.mapping import DatabaseMapping from pydantic import BaseModel -# [/SECTION] router = APIRouter(tags=["mappings"]) @@ -55,8 +53,8 @@ class SuggestRequest(BaseModel): # #region get_mappings [TYPE Function] # @BRIEF List all saved database mappings. -# @PRE db session is injected. -# @POST Returns filtered list of DatabaseMapping records. +# @PRE: db session is injected. +# @POST: Returns filtered list of DatabaseMapping records. @router.get("", response_model=List[MappingResponse]) async def get_mappings( source_env_id: Optional[str] = None, @@ -75,8 +73,8 @@ async def get_mappings( # #region create_mapping [TYPE Function] # @BRIEF Create or update a database mapping. -# @PRE mapping is valid MappingCreate, db session is injected. -# @POST DatabaseMapping created or updated in database. +# @PRE: mapping is valid MappingCreate, db session is injected. +# @POST: DatabaseMapping created or updated in database. @router.post("", response_model=MappingResponse) async def create_mapping( mapping: MappingCreate, @@ -108,8 +106,8 @@ async def create_mapping( # #region suggest_mappings_api [TYPE Function] # @BRIEF Get suggested mappings based on fuzzy matching. -# @PRE request is valid SuggestRequest, config_manager is injected. -# @POST Returns mapping suggestions. +# @PRE: request is valid SuggestRequest, config_manager is injected. +# @POST: Returns mapping suggestions. @router.post("/suggest") async def suggest_mappings_api( request: SuggestRequest, diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index 815c2539..8f204c9e 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -1,11 +1,6 @@ # #region MigrationApi [C:5] [TYPE Module] [SEMANTICS api, migration, dashboards, sync, dry-run] # @BRIEF HTTP contract layer for migration orchestration, settings, dry-run, and mapping sync endpoints. -# @LAYER Infra -# @PRE Backend core services initialized and Database session available. -# @POST Migration tasks are enqueued or dry-run results are computed and returned. -# @SIDE_EFFECT Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls. -# @DATA_CONTRACT [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary] -# @INVARIANT Migration endpoints never execute with invalid environment references and always return explicit HTTP errors on guard failures. +# @LAYER: Infra # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [DatabaseModule] # @RELATION DEPENDS_ON -> [DashboardSelection] @@ -13,6 +8,11 @@ # @RELATION DEPENDS_ON -> [MigrationDryRunService] # @RELATION DEPENDS_ON -> [IdMappingService] # @RELATION DEPENDS_ON -> [ResourceMapping] +# @INVARIANT: Migration endpoints never execute with invalid environment references and always return explicit HTTP errors on guard failures. +# @PRE: Backend core services initialized and Database session available. +# @POST: Migration tasks are enqueued or dry-run results are computed and returned. +# @SIDE_EFFECT: Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls. +# @DATA_CONTRACT: [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary] # @TEST_CONTRACT: [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary] # @TEST_SCENARIO: [invalid_environment] -> [HTTP_400_or_404] # @TEST_SCENARIO: [valid_execution] -> [success_payload_with_required_fields] @@ -40,10 +40,10 @@ router = APIRouter(prefix="/api", tags=["migration"]) # #region get_dashboards [C:3] [TYPE Function] # @BRIEF Fetch dashboard metadata from a requested environment for migration selection UI. -# @PRE env_id is provided and exists in configured environments. -# @POST Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent. -# @SIDE_EFFECT Reads environment configuration and performs remote Superset metadata retrieval over network. -# @DATA_CONTRACT Input[str env_id] -> Output[List[DashboardMetadata]] +# @PRE: env_id is provided and exists in configured environments. +# @POST: Returns List[DashboardMetadata] for the resolved environment; emits HTTP_404 when environment is absent. +# @SIDE_EFFECT: Reads environment configuration and performs remote Superset metadata retrieval over network. +# @DATA_CONTRACT: Input[str env_id] -> Output[List[DashboardMetadata]] # @RELATION CALLS -> [SupersetClient.get_dashboards_summary] @router.get("/environments/{env_id}/dashboards", response_model=List[DashboardMetadata]) async def get_dashboards( @@ -71,13 +71,13 @@ async def get_dashboards( # #region execute_migration [C:5] [TYPE Function] # @BRIEF Validate migration selection and enqueue asynchronous migration task execution. -# @PRE DashboardSelection payload is valid and both source/target environments exist. -# @POST Returns {"task_id": str, "message": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure. -# @SIDE_EFFECT Reads configuration, writes task record through task manager, and writes operational logs. -# @DATA_CONTRACT Input[DashboardSelection] -> Output[Dict[str, str]] -# @INVARIANT Migration task dispatch never occurs before source and target environment ids pass guard validation. +# @PRE: DashboardSelection payload is valid and both source/target environments exist. +# @POST: Returns {"task_id": str, "message": str} when task creation succeeds; emits HTTP_400/HTTP_500 on failure. +# @SIDE_EFFECT: Reads configuration, writes task record through task manager, and writes operational logs. +# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, str]] # @RELATION CALLS -> [create_task] # @RELATION DEPENDS_ON -> [DashboardSelection] +# @INVARIANT: Migration task dispatch never occurs before source and target environment ids pass guard validation. @router.post("/migration/execute") async def execute_migration( selection: DashboardSelection, @@ -134,13 +134,13 @@ async def execute_migration( # #region dry_run_migration [C:5] [TYPE Function] # @BRIEF Build pre-flight migration diff and risk summary without mutating target systems. -# @PRE DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty. -# @POST Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors. -# @SIDE_EFFECT Reads local mappings from DB and fetches source/target metadata via Superset API. -# @DATA_CONTRACT Input[DashboardSelection] -> Output[Dict[str, Any]] -# @INVARIANT Dry-run flow remains read-only and rejects identical source/target environments before service execution. +# @PRE: DashboardSelection is valid, source and target environments exist, differ, and selected_ids is non-empty. +# @POST: Returns deterministic dry-run payload; emits HTTP_400 for guard violations and HTTP_500 for orchestrator value errors. +# @SIDE_EFFECT: Reads local mappings from DB and fetches source/target metadata via Superset API. +# @DATA_CONTRACT: Input[DashboardSelection] -> Output[Dict[str, Any]] # @RELATION DEPENDS_ON -> [DashboardSelection] # @RELATION DEPENDS_ON -> [MigrationDryRunService] +# @INVARIANT: Dry-run flow remains read-only and rejects identical source/target environments before service execution. @router.post("/migration/dry-run", response_model=Dict[str, Any]) async def dry_run_migration( selection: DashboardSelection, @@ -200,10 +200,10 @@ async def dry_run_migration( # #region get_migration_settings [C:3] [TYPE Function] # @BRIEF Read and return configured migration synchronization cron expression. -# @PRE Configuration store is available and requester has READ permission. -# @POST Returns {"cron": str} reflecting current persisted settings value. -# @SIDE_EFFECT Reads configuration from config manager. -# @DATA_CONTRACT Input[None] -> Output[Dict[str, str]] +# @PRE: Configuration store is available and requester has READ permission. +# @POST: Returns {"cron": str} reflecting current persisted settings value. +# @SIDE_EFFECT: Reads configuration from config manager. +# @DATA_CONTRACT: Input[None] -> Output[Dict[str, str]] # @RELATION DEPENDS_ON -> [AppDependencies] @router.get("/migration/settings", response_model=Dict[str, str]) async def get_migration_settings( @@ -221,10 +221,10 @@ async def get_migration_settings( # #region update_migration_settings [C:3] [TYPE Function] # @BRIEF Validate and persist migration synchronization cron expression update. -# @PRE Payload includes "cron" key and requester has WRITE permission. -# @POST Returns {"cron": str, "status": "updated"} and persists updated cron value. -# @SIDE_EFFECT Mutates configuration and writes persisted config through config manager. -# @DATA_CONTRACT Input[Dict[str, str]] -> Output[Dict[str, str]] +# @PRE: Payload includes "cron" key and requester has WRITE permission. +# @POST: Returns {"cron": str, "status": "updated"} and persists updated cron value. +# @SIDE_EFFECT: Mutates configuration and writes persisted config through config manager. +# @DATA_CONTRACT: Input[Dict[str, str]] -> Output[Dict[str, str]] # @RELATION DEPENDS_ON -> [AppDependencies] @router.put("/migration/settings", response_model=Dict[str, str]) async def update_migration_settings( @@ -252,10 +252,10 @@ async def update_migration_settings( # #region get_resource_mappings [C:3] [TYPE Function] # @BRIEF Fetch synchronized resource mappings with optional filters and pagination for migration mappings view. -# @PRE skip>=0, 1<=limit<=500, DB session is active, requester has READ permission. -# @POST Returns {"items": [...], "total": int} where items reflect applied filters and pagination. -# @SIDE_EFFECT Executes database read queries against ResourceMapping table. -# @DATA_CONTRACT Input[QueryParams] -> Output[Dict[str, Any]] +# @PRE: skip>=0, 1<=limit<=500, DB session is active, requester has READ permission. +# @POST: Returns {"items": [...], "total": int} where items reflect applied filters and pagination. +# @SIDE_EFFECT: Executes database read queries against ResourceMapping table. +# @DATA_CONTRACT: Input[QueryParams] -> Output[Dict[str, Any]] # @RELATION DEPENDS_ON -> [ResourceMapping] @router.get("/migration/mappings-data", response_model=Dict[str, Any]) async def get_resource_mappings( @@ -324,10 +324,10 @@ async def get_resource_mappings( # #region trigger_sync_now [C:3] [TYPE Function] # @BRIEF Trigger immediate ID synchronization for every configured environment. -# @PRE At least one environment is configured and requester has EXECUTE permission. -# @POST Returns sync summary with synced/failed counts after attempting all environments. -# @SIDE_EFFECT Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs. -# @DATA_CONTRACT Input[None] -> Output[Dict[str, Any]] +# @PRE: At least one environment is configured and requester has EXECUTE permission. +# @POST: Returns sync summary with synced/failed counts after attempting all environments. +# @SIDE_EFFECT: Upserts Environment rows, commits DB transaction, performs network sync calls, and writes logs. +# @DATA_CONTRACT: Input[None] -> Output[Dict[str, Any]] # @RELATION DEPENDS_ON -> [IdMappingService] # @RELATION CALLS -> [sync_environment] @router.post("/migration/sync-now", response_model=Dict[str, Any]) @@ -337,6 +337,7 @@ async def trigger_sync_now( _=Depends(has_permission("plugin:migration", "EXECUTE")), ): with belief_scope("trigger_sync_now"): + from ...core.logger import logger from ...models.mapping import Environment as EnvironmentModel config = config_manager.get_config() @@ -356,8 +357,8 @@ async def trigger_sync_now( credentials_id=env.id, # Use env.id as credentials reference ) db.add(db_env) - logger.reason( - f"Created environment row for {env.id}" + logger.info( + f"[trigger_sync_now][Action] Created environment row for {env.id}" ) else: existing.name = env.name @@ -372,10 +373,10 @@ async def trigger_sync_now( client = SupersetClient(env) service.sync_environment(env.id, client) results["synced"].append(env.id) - logger.reason(f"Synced environment {env.id}") + logger.info(f"[trigger_sync_now][Action] Synced environment {env.id}") except Exception as e: results["failed"].append({"env_id": env.id, "error": str(e)}) - logger.explore(f"Failed to sync {env.id}", error=str(e)) + logger.error(f"[trigger_sync_now][Error] Failed to sync {env.id}: {e}") return { "status": "completed", diff --git a/backend/src/api/routes/plugins.py b/backend/src/api/routes/plugins.py index 847c24b2..fe2b3e72 100755 --- a/backend/src/api/routes/plugins.py +++ b/backend/src/api/routes/plugins.py @@ -1,6 +1,6 @@ # #region PluginsRouter [C:3] [TYPE Module] [SEMANTICS api, router, plugins, list] # @BRIEF Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins. -# @LAYER UI (API) +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [PluginConfig] # @RELATION DEPENDS_ON -> [get_plugin_loader] # @RELATION BINDS_TO -> [API_Routes] @@ -16,9 +16,8 @@ router = APIRouter() # #region list_plugins [TYPE Function] # @BRIEF Retrieve a list of all available plugins. -# @PRE plugin_loader is injected via Depends. -# @POST Returns a list of PluginConfig objects. -# @RETURN List[PluginConfig] - List of registered plugins. +# @PRE: plugin_loader is injected via Depends. +# @POST: Returns a list of PluginConfig objects. # @RELATION CALLS -> [get_plugin_loader] # @RELATION DEPENDS_ON -> [PluginConfig] @router.get("", response_model=List[PluginConfig]) diff --git a/backend/src/api/routes/profile.py b/backend/src/api/routes/profile.py index 2c701c1e..b5e1fb12 100644 --- a/backend/src/api/routes/profile.py +++ b/backend/src/api/routes/profile.py @@ -1,27 +1,25 @@ # #region ProfileApiModule [C:3] [TYPE Module] [SEMANTICS api, profile, preferences, self-service, account-lookup] +# # @BRIEF Exposes self-scoped profile preference endpoints and environment-based Superset account lookup. -# @LAYER API -# @INVARIANT Endpoints are self-scoped and never mutate another user preference. +# @LAYER: API # @RELATION DEPENDS_ON -> [ProfileService] # @RELATION DEPENDS_ON -> [get_current_user] # @RELATION DEPENDS_ON -> [get_db] # -# +# @INVARIANT: Endpoints are self-scoped and never mutate another user preference. # @UX_STATE: ProfileLoad -> Returns stable ProfilePreferenceResponse for authenticated user. # @UX_STATE: Saving -> Validation errors map to actionable 422 details. # @UX_STATE: LookupLoading -> Returns success/degraded Superset lookup payload. # @UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback. # @UX_RECOVERY: Lookup degradation keeps manual username save path available. -# [SECTION: IMPORTS] from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from ...core.database import get_db -from ...core.logger import belief_scope -from ...core.cot_logger import MarkerLogger +from ...core.logger import logger, belief_scope from ...dependencies import ( get_config_manager, get_current_user, @@ -40,18 +38,15 @@ from ...services.profile_service import ( ProfileService, ProfileValidationError, ) -# [/SECTION] - -log = MarkerLogger("Profile") router = APIRouter(prefix="/api/profile", tags=["profile"]) # #region _get_profile_service [TYPE Function] +# @RELATION CALLS -> ProfileService # @BRIEF Build profile service for current request scope. -# @PRE db session and config manager are available. -# @POST Returns a ready ProfileService instance. -# @RELATION CALLS -> [ProfileService] +# @PRE: db session and config manager are available. +# @POST: Returns a ready ProfileService instance. def _get_profile_service(db: Session, config_manager, plugin_loader=None) -> ProfileService: return ProfileService( db=db, @@ -62,10 +57,10 @@ def _get_profile_service(db: Session, config_manager, plugin_loader=None) -> Pro # #region get_preferences [TYPE Function] +# @RELATION CALLS -> ProfileService # @BRIEF Get authenticated user's dashboard filter preference. -# @PRE Valid JWT and authenticated user context. -# @POST Returns preference payload for current user only. -# @RELATION CALLS -> [ProfileService] +# @PRE: Valid JWT and authenticated user context. +# @POST: Returns preference payload for current user only. @router.get("/preferences", response_model=ProfilePreferenceResponse) async def get_preferences( current_user: User = Depends(get_current_user), @@ -74,17 +69,17 @@ async def get_preferences( plugin_loader=Depends(get_plugin_loader), ): with belief_scope("profile.get_preferences", f"user_id={current_user.id}"): - log.reason("Resolving current user preference") + logger.reason("[REASON] Resolving current user preference") service = _get_profile_service(db, config_manager, plugin_loader) return service.get_my_preference(current_user) # #endregion get_preferences # #region update_preferences [TYPE Function] +# @RELATION CALLS -> ProfileService # @BRIEF Update authenticated user's dashboard filter preference. -# @PRE Valid JWT and valid request payload. -# @POST Persists normalized preference for current user or raises validation/authorization errors. -# @RELATION CALLS -> [ProfileService] +# @PRE: Valid JWT and valid request payload. +# @POST: Persists normalized preference for current user or raises validation/authorization errors. @router.patch("/preferences", response_model=ProfilePreferenceResponse) async def update_preferences( payload: ProfilePreferenceUpdateRequest, @@ -96,22 +91,22 @@ async def update_preferences( with belief_scope("profile.update_preferences", f"user_id={current_user.id}"): service = _get_profile_service(db, config_manager, plugin_loader) try: - log.reason("Attempting preference save") + logger.reason("[REASON] Attempting preference save") return service.update_my_preference(current_user=current_user, payload=payload) except ProfileValidationError as exc: - log.reflect("Preference validation failed") + logger.reflect("[REFLECT] Preference validation failed") raise HTTPException(status_code=422, detail=exc.errors) from exc except ProfileAuthorizationError as exc: - log.explore("Cross-user mutation guard blocked request", error="User attempted to mutate another user's resource") + logger.explore("[EXPLORE] Cross-user mutation guard blocked request") raise HTTPException(status_code=403, detail=str(exc)) from exc # #endregion update_preferences # #region lookup_superset_accounts [TYPE Function] +# @RELATION CALLS -> ProfileService # @BRIEF Lookup Superset account candidates in selected environment. -# @PRE Valid JWT, authenticated context, and environment_id query parameter. -# @POST Returns success or degraded lookup payload with stable shape. -# @RELATION CALLS -> [ProfileService] +# @PRE: Valid JWT, authenticated context, and environment_id query parameter. +# @POST: Returns success or degraded lookup payload with stable shape. @router.get("/superset-accounts", response_model=SupersetAccountLookupResponse) async def lookup_superset_accounts( environment_id: str = Query(...), @@ -139,14 +134,14 @@ async def lookup_superset_accounts( sort_order=sort_order, ) try: - log.reason("Executing Superset account lookup") + logger.reason("[REASON] Executing Superset account lookup") return service.lookup_superset_accounts( current_user=current_user, request=lookup_request, ) except EnvironmentNotFoundError as exc: - log.explore("Lookup request references unknown environment", error="EnvironmentNotFoundError raised during Superset account lookup") + logger.explore("[EXPLORE] Lookup request references unknown environment") raise HTTPException(status_code=404, detail=str(exc)) from exc # #endregion lookup_superset_accounts -# #endregion ProfileApiModule +# #endregion ProfileApiModule \ No newline at end of file diff --git a/backend/src/api/routes/reports.py b/backend/src/api/routes/reports.py index dfb7d12f..8087fad5 100644 --- a/backend/src/api/routes/reports.py +++ b/backend/src/api/routes/reports.py @@ -1,17 +1,16 @@ # #region ReportsRouter [C:5] [TYPE Module] [SEMANTICS api, reports, list, detail, pagination, filters] # @BRIEF FastAPI router for unified task report list and detail retrieval endpoints. -# @LAYER UI (API) -# @PRE Reports service and dependencies are initialized. -# @POST Router is configured and endpoints are ready for registration. -# @SIDE_EFFECT None -# @DATA_CONTRACT [ReportQuery] -> [ReportCollection | ReportDetailView] -# @INVARIANT Endpoints are read-only and do not trigger long-running tasks. +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [ReportsService:Class] # @RELATION DEPENDS_ON -> [get_task_manager:Function] # @RELATION DEPENDS_ON -> [get_clean_release_repository:Function] # @RELATION DEPENDS_ON -> [has_permission:Function] +# @INVARIANT: Endpoints are read-only and do not trigger long-running tasks. +# @PRE: Reports service and dependencies are initialized. +# @POST: Router is configured and endpoints are ready for registration. +# @SIDE_EFFECT: None +# @DATA_CONTRACT: [ReportQuery] -> [ReportCollection | ReportDetailView] -# [SECTION: IMPORTS] from datetime import datetime from typing import List, Optional @@ -33,20 +32,15 @@ from ...models.report import ( ) from ...services.clean_release.repository import CleanReleaseRepository from ...services.reports.report_service import ReportsService -# [/SECTION] router = APIRouter(prefix="/api/reports", tags=["Reports"]) # #region _parse_csv_enum_list [C:1] [TYPE Function] # @BRIEF Parse comma-separated query value into enum list. -# @PRE raw may be None/empty or comma-separated values. -# @POST Returns enum list or raises HTTP 400 with deterministic machine-readable payload. -# @PARAM: raw (Optional[str]) - Comma-separated enum values. -# @PARAM: enum_cls (type) - Enum class for validation. -# @PARAM: field_name (str) - Query field name for diagnostics. -# @RETURN: List - Parsed enum values. -# @RELATION: BINDS_TO -> [ReportsRouter] +# @PRE: raw may be None/empty or comma-separated values. +# @POST: Returns enum list or raises HTTP 400 with deterministic machine-readable payload. +# @RELATION BINDS_TO -> [ReportsRouter] def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: with belief_scope("_parse_csv_enum_list"): if raw is None or not raw.strip(): @@ -77,9 +71,9 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List: # #region list_reports [C:2] [TYPE Function] # @BRIEF Return paginated unified reports list. -# @PRE authenticated/authorized request and validated query params. -# @POST returns {items,total,page,page_size,has_next,applied_filters}. -# @POST deterministic error payload for invalid filters. +# @PRE: authenticated/authorized request and validated query params. +# @POST: returns {items,total,page,page_size,has_next,applied_filters}. +# @POST: deterministic error payload for invalid filters. # @RELATION CALLS -> [_parse_csv_enum_list:Function] # @RELATION DEPENDS_ON -> [ReportQuery:Class] # @RELATION DEPENDS_ON -> [ReportsService:Class] @@ -152,8 +146,8 @@ async def list_reports( # #region get_report_detail [C:2] [TYPE Function] # @BRIEF Return one normalized report detail with diagnostics and next actions. -# @PRE authenticated/authorized request and existing report_id. -# @POST returns normalized detail envelope or 404 when report is not found. +# @PRE: authenticated/authorized request and existing report_id. +# @POST: returns normalized detail envelope or 404 when report is not found. # @RELATION CALLS -> [ReportsService:Class] @router.get("/{report_id}", response_model=ReportDetailView) async def get_report_detail( diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index fd2cc688..bcbe04d2 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -1,15 +1,14 @@ # #region SettingsRouter [C:3] [TYPE Module] [SEMANTICS settings, api, router, fastapi] +# # @BRIEF Provides API endpoints for managing application settings and Superset environments. -# @LAYER API -# @INVARIANT All settings changes must be persisted via ConfigManager. +# @LAYER: API # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [get_config_manager] # @RELATION DEPENDS_ON -> [has_permission] # -# +# @INVARIANT: All settings changes must be persisted via ConfigManager. # @PUBLIC_API: router -# [SECTION: IMPORTS] from fastapi import APIRouter, Depends, HTTPException from typing import List from pydantic import BaseModel @@ -17,10 +16,7 @@ from ...core.config_models import AppConfig, Environment, GlobalSettings, Loggin from ...models.storage import StorageConfig from ...dependencies import get_config_manager, has_permission from ...core.config_manager import ConfigManager -from ...core.logger import belief_scope -from ...core.cot_logger import MarkerLogger - -log = MarkerLogger("Settings") +from ...core.logger import logger, belief_scope from ...core.superset_client import SupersetClient from ...services.llm_prompt_templates import normalize_llm_settings from ...models.llm import ValidationPolicy @@ -32,7 +28,6 @@ from ...schemas.settings import ( ) from ...core.database import get_db from sqlalchemy.orm import Session -# [/SECTION] # #region LoggingConfigResponse [C:1] [TYPE Class] [SEMANTICS logging, config, response] @@ -50,8 +45,8 @@ router = APIRouter() # #region _normalize_superset_env_url [C:1] [TYPE Function] # @BRIEF Canonicalize Superset environment URL to base host/path without trailing /api/v1. -# @PRE raw_url can be empty. -# @POST Returns normalized base URL. +# @PRE: raw_url can be empty. +# @POST: Returns normalized base URL. def _normalize_superset_env_url(raw_url: str) -> str: normalized = str(raw_url or "").strip().rstrip("/") if normalized.lower().endswith("/api/v1"): @@ -64,8 +59,8 @@ def _normalize_superset_env_url(raw_url: str) -> str: # #region _validate_superset_connection_fast [C:2] [TYPE Function] # @BRIEF Run lightweight Superset connectivity validation without full pagination scan. -# @PRE env contains valid URL and credentials. -# @POST Raises on auth/API failures; returns None on success. +# @PRE: env contains valid URL and credentials. +# @POST: Raises on auth/API failures; returns None on success. def _validate_superset_connection_fast(env: Environment) -> None: client = SupersetClient(env) # 1) Explicit auth check @@ -85,16 +80,15 @@ def _validate_superset_connection_fast(env: Environment) -> None: # #region get_settings [C:2] [TYPE Function] # @BRIEF Retrieves all application settings. -# @PRE Config manager is available. -# @POST Returns masked AppConfig. -# @RETURN AppConfig - The current configuration. +# @PRE: Config manager is available. +# @POST: Returns masked AppConfig. @router.get("", response_model=AppConfig) async def get_settings( config_manager: ConfigManager = Depends(get_config_manager), _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_settings"): - log.reason("Fetching all settings") + logger.info("[get_settings][Entry] Fetching all settings") config = config_manager.get_config().copy(deep=True) config.settings.llm = normalize_llm_settings(config.settings.llm) # Mask passwords @@ -109,9 +103,9 @@ async def get_settings( # #region get_features [C:1] [TYPE Function] # @BRIEF Public endpoint returning feature flags for frontend sidebar filtering. -# @PRE Config manager is available. -# @POST Returns dict with dataset_review and health_monitor booleans. -# @RATIONALE Unauthenticated because sidebar filtering must work for all users, not just admins. +# @RATIONALE: Unauthenticated because sidebar filtering must work for all users, not just admins. +# @PRE: Config manager is available. +# @POST: Returns dict with dataset_review and health_monitor booleans. @router.get("/features") async def get_features( config_manager: ConfigManager = Depends(get_config_manager), @@ -124,10 +118,8 @@ async def get_features( # #region update_global_settings [C:2] [TYPE Function] # @BRIEF Updates global application settings. -# @PRE New settings are provided. -# @POST Global settings are updated. -# @PARAM: settings (GlobalSettings) - The new global settings. -# @RETURN: GlobalSettings - The updated settings. +# @PRE: New settings are provided. +# @POST: Global settings are updated. @router.patch("/global", response_model=GlobalSettings) async def update_global_settings( settings: GlobalSettings, @@ -135,7 +127,7 @@ async def update_global_settings( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_global_settings"): - log.reason("Updating global settings") + logger.info("[update_global_settings][Entry] Updating global settings") config_manager.update_global_settings(settings) return settings @@ -146,7 +138,6 @@ async def update_global_settings( # #region get_storage_settings [C:2] [TYPE Function] # @BRIEF Retrieves storage-specific settings. -# @RETURN StorageConfig - The storage configuration. @router.get("/storage", response_model=StorageConfig) async def get_storage_settings( config_manager: ConfigManager = Depends(get_config_manager), @@ -161,9 +152,7 @@ async def get_storage_settings( # #region update_storage_settings [C:2] [TYPE Function] # @BRIEF Updates storage-specific settings. -# @PARAM: storage (StorageConfig) - The new storage settings. # @POST: Storage settings are updated and saved. -# @RETURN: StorageConfig - The updated storage settings. @router.put("/storage", response_model=StorageConfig) async def update_storage_settings( storage: StorageConfig, @@ -186,16 +175,15 @@ async def update_storage_settings( # #region get_environments [C:2] [TYPE Function] # @BRIEF Lists all configured Superset environments. -# @PRE Config manager is available. -# @POST Returns list of environments. -# @RETURN List[Environment] - List of environments. +# @PRE: Config manager is available. +# @POST: Returns list of environments. @router.get("/environments", response_model=List[Environment]) async def get_environments( config_manager: ConfigManager = Depends(get_config_manager), _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_environments"): - log.reason("Fetching environments") + logger.info("[get_environments][Entry] Fetching environments") environments = config_manager.get_environments() return [ env.copy(update={"url": _normalize_superset_env_url(env.url)}) @@ -208,10 +196,8 @@ async def get_environments( # #region add_environment [C:2] [TYPE Function] # @BRIEF Adds a new Superset environment. -# @PRE Environment data is valid and reachable. -# @POST Environment is added to config. -# @PARAM: env (Environment) - The environment to add. -# @RETURN: Environment - The added environment. +# @PRE: Environment data is valid and reachable. +# @POST: Environment is added to config. @router.post("/environments", response_model=Environment) async def add_environment( env: Environment, @@ -219,14 +205,16 @@ async def add_environment( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("add_environment"): - log.reason("Adding environment", payload={"env_id": env.id}) + logger.info(f"[add_environment][Entry] Adding environment {env.id}") env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) # Validate connection before adding (fast path) try: _validate_superset_connection_fast(env) except Exception as e: - log.explore("Connection validation failed", payload={"env_id": env.id}, error=str(e)) + logger.error( + f"[add_environment][Coherence:Failed] Connection validation failed: {e}" + ) raise HTTPException( status_code=400, detail=f"Connection validation failed: {e}" ) @@ -240,11 +228,8 @@ async def add_environment( # #region update_environment [C:2] [TYPE Function] # @BRIEF Updates an existing Superset environment. -# @PRE ID and valid environment data are provided. -# @POST Environment is updated in config. -# @PARAM: id (str) - The ID of the environment to update. -# @PARAM: env (Environment) - The updated environment data. -# @RETURN: Environment - The updated environment. +# @PRE: ID and valid environment data are provided. +# @POST: Environment is updated in config. @router.put("/environments/{id}", response_model=Environment) async def update_environment( id: str, @@ -252,7 +237,7 @@ async def update_environment( config_manager: ConfigManager = Depends(get_config_manager), ): with belief_scope("update_environment"): - log.reason("Updating environment", payload={"id": id}) + logger.info(f"[update_environment][Entry] Updating environment {id}") env = env.copy(update={"url": _normalize_superset_env_url(env.url)}) @@ -269,7 +254,9 @@ async def update_environment( try: _validate_superset_connection_fast(env_to_validate) except Exception as e: - log.explore("Connection validation failed", payload={"id": id}, error=str(e)) + logger.error( + f"[update_environment][Coherence:Failed] Connection validation failed: {e}" + ) raise HTTPException( status_code=400, detail=f"Connection validation failed: {e}" ) @@ -284,15 +271,14 @@ async def update_environment( # #region delete_environment [C:2] [TYPE Function] # @BRIEF Deletes a Superset environment. -# @PRE ID is provided. -# @POST Environment is removed from config. -# @PARAM: id (str) - The ID of the environment to delete. +# @PRE: ID is provided. +# @POST: Environment is removed from config. @router.delete("/environments/{id}") async def delete_environment( id: str, config_manager: ConfigManager = Depends(get_config_manager) ): with belief_scope("delete_environment"): - log.reason("Deleting environment", payload={"id": id}) + logger.info(f"[delete_environment][Entry] Deleting environment {id}") config_manager.delete_environment(id) return {"message": f"Environment {id} deleted"} @@ -302,16 +288,14 @@ async def delete_environment( # #region test_environment_connection [C:2] [TYPE Function] # @BRIEF Tests the connection to a Superset environment. -# @PRE ID is provided. -# @POST Returns success or error status. -# @PARAM: id (str) - The ID of the environment to test. -# @RETURN: dict - Success message or error. +# @PRE: ID is provided. +# @POST: Returns success or error status. @router.post("/environments/{id}/test") async def test_environment_connection( id: str, config_manager: ConfigManager = Depends(get_config_manager) ): with belief_scope("test_environment_connection"): - log.reason("Testing environment connection", payload={"id": id}) + logger.info(f"[test_environment_connection][Entry] Testing environment {id}") # Find environment env = next((e for e in config_manager.get_environments() if e.id == id), None) @@ -321,10 +305,14 @@ async def test_environment_connection( try: _validate_superset_connection_fast(env) - log.reflect("Connection successful", payload={"id": id}) + logger.info( + f"[test_environment_connection][Coherence:OK] Connection successful for {id}" + ) return {"status": "success", "message": "Connection successful"} except Exception as e: - log.explore("Connection failed", payload={"id": id}, error=str(e)) + logger.error( + f"[test_environment_connection][Coherence:Failed] Connection failed for {id}: {e}" + ) return {"status": "error", "message": str(e)} @@ -333,9 +321,8 @@ async def test_environment_connection( # #region get_logging_config [C:2] [TYPE Function] # @BRIEF Retrieves current logging configuration. -# @PRE Config manager is available. -# @POST Returns logging configuration. -# @RETURN LoggingConfigResponse - The current logging config. +# @PRE: Config manager is available. +# @POST: Returns logging configuration. @router.get("/logging", response_model=LoggingConfigResponse) async def get_logging_config( config_manager: ConfigManager = Depends(get_config_manager), @@ -355,10 +342,8 @@ async def get_logging_config( # #region update_logging_config [C:2] [TYPE Function] # @BRIEF Updates logging configuration. -# @PRE New logging config is provided. -# @POST Logging configuration is updated and saved. -# @PARAM: config (LoggingConfig) - The new logging configuration. -# @RETURN: LoggingConfigResponse - The updated logging config. +# @PRE: New logging config is provided. +# @POST: Logging configuration is updated and saved. @router.patch("/logging", response_model=LoggingConfigResponse) async def update_logging_config( config: LoggingConfig, @@ -366,7 +351,9 @@ async def update_logging_config( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_logging_config"): - log.reason("Updating logging config", payload={"level": config.level, "task_log_level": config.task_log_level}) + logger.info( + f"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}" + ) # Get current settings and update logging config settings = config_manager.get_config().settings @@ -401,10 +388,10 @@ class ConsolidatedSettingsResponse(BaseModel): # #region get_consolidated_settings [C:4] [TYPE Function] # @BRIEF Retrieves all settings categories in a single call. -# @PRE Config manager is available and the caller holds admin settings read permission. -# @POST Returns consolidated settings, provider metadata, and persisted notification payload in one stable response. -# @SIDE_EFFECT Opens one database session to read LLM providers and config-backed notification payload, then closes it. -# @DATA_CONTRACT Input[ConfigManager] -> Output[ConsolidatedSettingsResponse] +# @PRE: Config manager is available and the caller holds admin settings read permission. +# @POST: Returns consolidated settings, provider metadata, and persisted notification payload in one stable response. +# @SIDE_EFFECT: Opens one database session to read LLM providers and config-backed notification payload, then closes it. +# @DATA_CONTRACT: Input[ConfigManager] -> Output[ConsolidatedSettingsResponse] # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [AppConfigRecord] @@ -417,7 +404,7 @@ async def get_consolidated_settings( _=Depends(has_permission("admin:settings", "READ")), ): with belief_scope("get_consolidated_settings"): - log.reason("Fetching consolidated settings payload") + logger.reason("Fetching consolidated settings payload") config = config_manager.get_config() @@ -464,9 +451,9 @@ async def get_consolidated_settings( notifications=notifications_payload, features=config.settings.features.model_dump(), ) - log.reflect( + logger.reflect( "Consolidated settings payload assembled", - payload={ + extra={ "environment_count": len(response_payload.environments), "provider_count": len(response_payload.llm_providers), }, @@ -479,8 +466,8 @@ async def get_consolidated_settings( # #region update_consolidated_settings [C:2] [TYPE Function] # @BRIEF Bulk update application settings from the consolidated view. -# @PRE User has admin permissions, config is valid. -# @POST Settings are updated and saved via ConfigManager. +# @PRE: User has admin permissions, config is valid. +# @POST: Settings are updated and saved via ConfigManager. @router.patch("/consolidated") async def update_consolidated_settings( settings_patch: dict, @@ -488,7 +475,9 @@ async def update_consolidated_settings( _=Depends(has_permission("admin:settings", "WRITE")), ): with belief_scope("update_consolidated_settings"): - log.reason("Applying consolidated settings patch") + logger.info( + "[update_consolidated_settings][Entry] Applying consolidated settings patch" + ) current_config = config_manager.get_config() current_settings = current_config.settings @@ -533,7 +522,6 @@ async def update_consolidated_settings( # #region get_validation_policies [C:2] [TYPE Function] # @BRIEF Lists all validation policies. -# @RETURN List[ValidationPolicyResponse] - List of policies. @router.get("/automation/policies", response_model=List[ValidationPolicyResponse]) async def get_validation_policies( db: Session = Depends(get_db), _=Depends(has_permission("admin:settings", "READ")) @@ -547,8 +535,6 @@ async def get_validation_policies( # #region create_validation_policy [C:2] [TYPE Function] # @BRIEF Creates a new validation policy. -# @PARAM: policy (ValidationPolicyCreate) - The policy data. -# @RETURN: ValidationPolicyResponse - The created policy. @router.post("/automation/policies", response_model=ValidationPolicyResponse) async def create_validation_policy( policy: ValidationPolicyCreate, @@ -568,9 +554,6 @@ async def create_validation_policy( # #region update_validation_policy [C:2] [TYPE Function] # @BRIEF Updates an existing validation policy. -# @PARAM: id (str) - The ID of the policy to update. -# @PARAM: policy (ValidationPolicyUpdate) - The updated policy data. -# @RETURN: ValidationPolicyResponse - The updated policy. @router.patch("/automation/policies/{id}", response_model=ValidationPolicyResponse) async def update_validation_policy( id: str, @@ -597,7 +580,6 @@ async def update_validation_policy( # #region delete_validation_policy [C:2] [TYPE Function] # @BRIEF Deletes a validation policy. -# @PARAM: id (str) - The ID of the policy to delete. @router.delete("/automation/policies/{id}") async def delete_validation_policy( id: str, diff --git a/backend/src/api/routes/storage.py b/backend/src/api/routes/storage.py index 72e9efed..1d710f82 100644 --- a/backend/src/api/routes/storage.py +++ b/backend/src/api/routes/storage.py @@ -1,13 +1,12 @@ # #region storage_routes [C:3] [TYPE Module] [SEMANTICS storage, files, upload, download, backup, repository] +# # @BRIEF API endpoints for file storage management (backups and repositories). -# @LAYER API -# @INVARIANT All paths must be validated against path traversal. +# @LAYER: API # @RELATION DEPENDS_ON -> [StorageModels] # @RELATION DEPENDS_ON -> [StoragePlugin] # -# +# @INVARIANT: All paths must be validated against path traversal. -# [SECTION: IMPORTS] from pathlib import Path from fastapi import APIRouter, Depends, UploadFile, File, Form, HTTPException from fastapi.responses import FileResponse @@ -16,20 +15,16 @@ from ...models.storage import StoredFile, FileCategory from ...dependencies import get_plugin_loader, has_permission from ...plugins.storage.plugin import StoragePlugin from ...core.logger import belief_scope -# [/SECTION] router = APIRouter(tags=["storage"]) # #region list_files [C:3] [TYPE Function] # @BRIEF List all files and directories in the storage system. -# @PRE None. -# @POST Returns a list of StoredFile objects. # +# @PRE: None. +# @POST: Returns a list of StoredFile objects. # -# @PARAM: category (Optional[FileCategory]) - Filter by category. -# @PARAM: path (Optional[str]) - Subpath within the category. -# @RETURN: List[StoredFile] - List of files/directories. -# @RELATION: DEPENDS_ON -> [StoragePlugin] +# @RELATION DEPENDS_ON -> [StoragePlugin] @router.get("/files", response_model=List[StoredFile]) async def list_files( category: Optional[FileCategory] = None, @@ -47,19 +42,15 @@ async def list_files( # #region upload_file [C:3] [TYPE Function] # @BRIEF Upload a file to the storage system. -# @PRE category must be a valid FileCategory. -# @PRE file must be a valid UploadFile. -# @POST Returns the StoredFile object of the uploaded file. # +# @PRE: category must be a valid FileCategory. +# @PRE: file must be a valid UploadFile. +# @POST: Returns the StoredFile object of the uploaded file. # -# @PARAM: category (FileCategory) - Target category. -# @PARAM: path (Optional[str]) - Target subpath. -# @PARAM: file (UploadFile) - The file content. -# @RETURN: StoredFile - Metadata of the uploaded file. # # @SIDE_EFFECT: Writes file to the filesystem. # -# @RELATION: DEPENDS_ON -> [StoragePlugin] +# @RELATION DEPENDS_ON -> [StoragePlugin] @router.post("/upload", response_model=StoredFile, status_code=201) async def upload_file( category: FileCategory = Form(...), @@ -80,17 +71,14 @@ async def upload_file( # #region delete_file [C:3] [TYPE Function] # @BRIEF Delete a specific file or directory. -# @PRE category must be a valid FileCategory. -# @POST Item is removed from storage. # +# @PRE: category must be a valid FileCategory. +# @POST: Item is removed from storage. # -# @PARAM: category (FileCategory) - File category. -# @PARAM: path (str) - Relative path of the item. -# @RETURN: None # # @SIDE_EFFECT: Deletes item from the filesystem. # -# @RELATION: DEPENDS_ON -> [StoragePlugin] +# @RELATION DEPENDS_ON -> [StoragePlugin] @router.delete("/files/{category}/{path:path}", status_code=204) async def delete_file( category: FileCategory, @@ -112,15 +100,12 @@ async def delete_file( # #region download_file [C:3] [TYPE Function] # @BRIEF Retrieve a file for download. -# @PRE category must be a valid FileCategory. -# @POST Returns a FileResponse. +# +# @PRE: category must be a valid FileCategory. +# @POST: Returns a FileResponse. # # -# @PARAM: category (FileCategory) - File category. -# @PARAM: path (str) - Relative path of the file. -# @RETURN: FileResponse - The file content. -# -# @RELATION: DEPENDS_ON -> [StoragePlugin] +# @RELATION DEPENDS_ON -> [StoragePlugin] @router.get("/download/{category}/{path:path}") async def download_file( category: FileCategory, @@ -144,14 +129,12 @@ async def download_file( # #region get_file_by_path [C:3] [TYPE Function] # @BRIEF Retrieve a file by validated absolute/relative path under storage root. -# @PRE path must resolve under configured storage root. -# @POST Returns a FileResponse for existing files. +# +# @PRE: path must resolve under configured storage root. +# @POST: Returns a FileResponse for existing files. # # -# @PARAM: path (str) - Absolute or storage-root-relative file path. -# @RETURN: FileResponse - The file content. -# -# @RELATION: DEPENDS_ON -> [StoragePlugin] +# @RELATION DEPENDS_ON -> [StoragePlugin] @router.get("/file") async def get_file_by_path( path: str, diff --git a/backend/src/api/routes/tasks.py b/backend/src/api/routes/tasks.py index 5d686453..15c328e6 100755 --- a/backend/src/api/routes/tasks.py +++ b/backend/src/api/routes/tasks.py @@ -1,11 +1,10 @@ # #region TasksRouter [C:3] [TYPE Module] [SEMANTICS api, router, tasks, create, list, get, logs] # @BRIEF Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks. -# @LAYER UI (API) +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [LLMProviderService] -# [SECTION: IMPORTS] from typing import List, Dict, Any, Optional from fastapi import APIRouter, Depends, HTTPException, status, Query from pydantic import BaseModel @@ -25,7 +24,6 @@ from ...services.llm_prompt_templates import ( normalize_llm_settings, resolve_bound_provider_id, ) -# [/SECTION] router = APIRouter() @@ -51,14 +49,11 @@ class ResumeTaskRequest(BaseModel): # #region create_task [C:3] [TYPE Function] # @BRIEF Create and start a new task for a given plugin. -# @PARAM: request (CreateTaskRequest) - The request body containing plugin_id and params. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: plugin_id must exist and params must be valid for that plugin. # @POST: A new task is created and started. -# @RETURN: Task - The created task instance. -# @RELATION: CALLS -> [TaskManager] -# @RELATION: DEPENDS_ON -> [ConfigManager] -# @RELATION: DEPENDS_ON -> [LLMProviderService] +# @RELATION CALLS -> [TaskManager] +# @RELATION DEPENDS_ON -> [ConfigManager] +# @RELATION DEPENDS_ON -> [LLMProviderService] @router.post("", response_model=Task, status_code=status.HTTP_201_CREATED) async def create_task( request: CreateTaskRequest, @@ -133,15 +128,10 @@ async def create_task( # #region list_tasks [C:2] [TYPE Function] # @BRIEF Retrieve a list of tasks with pagination and optional status filter. -# @PARAM: limit (int) - Maximum number of tasks to return. -# @PARAM: offset (int) - Number of tasks to skip. -# @PARAM: status (Optional[TaskStatus]) - Filter by task status. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_manager must be available. # @POST: Returns a list of tasks. -# @RETURN: List[Task] - List of tasks. -# @RELATION: CALLS -> [TaskManager] -# @RELATION: BINDS_TO -> [TASK_TYPE_PLUGIN_MAP] +# @RELATION CALLS -> [TaskManager] +# @RELATION BINDS_TO -> [TASK_TYPE_PLUGIN_MAP] @router.get("", response_model=List[Task]) async def list_tasks( limit: int = 10, @@ -183,12 +173,9 @@ async def list_tasks( # #region get_task [C:2] [TYPE Function] # @BRIEF Retrieve the details of a specific task. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_id must exist. # @POST: Returns task details or raises 404. -# @RETURN: Task - The task details. -# @RELATION: CALLS -> [TaskManager] +# @RELATION CALLS -> [TaskManager] @router.get("/{task_id}", response_model=Task) async def get_task( task_id: str, @@ -209,18 +196,10 @@ async def get_task( # #region get_task_logs [C:5] [TYPE Function] # @BRIEF Retrieve logs for a specific task with optional filtering. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: level (Optional[str]) - Filter by log level (DEBUG, INFO, WARNING, ERROR). -# @PARAM: source (Optional[str]) - Filter by source component. -# @PARAM: search (Optional[str]) - Text search in message. -# @PARAM: offset (int) - Number of logs to skip. -# @PARAM: limit (int) - Maximum number of logs to return. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_id must exist. # @POST: Returns a list of log entries or raises 404. -# @RETURN: List[LogEntry] - List of log entries. -# @RELATION: CALLS -> [TaskManager] -# @RELATION: DEPENDS_ON -> [LogFilter] +# @RELATION CALLS -> [TaskManager] +# @RELATION DEPENDS_ON -> [LogFilter] # @TEST_CONTRACT: TaskLogQueryInput -> List[LogEntry] # @TEST_SCENARIO: existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination. # @TEST_FIXTURE: valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json @@ -264,13 +243,10 @@ async def get_task_logs( # #region get_task_log_stats [C:2] [TYPE Function] # @BRIEF Get statistics about logs for a task (counts by level and source). -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_id must exist. # @POST: Returns log statistics or raises 404. -# @RETURN: LogStats - Statistics about task logs. -# @RELATION: CALLS -> [TaskManager] -# @RELATION: DEPENDS_ON -> [LogStats] +# @RELATION CALLS -> [TaskManager] +# @RELATION DEPENDS_ON -> [LogStats] @router.get("/{task_id}/logs/stats", response_model=LogStats) async def get_task_log_stats( task_id: str, @@ -312,12 +288,9 @@ async def get_task_log_stats( # #region get_task_log_sources [C:2] [TYPE Function] # @BRIEF Get unique sources for a task's logs. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_id must exist. # @POST: Returns list of unique source names or raises 404. -# @RETURN: List[str] - Unique source names. -# @RELATION: CALLS -> [TaskManager] +# @RELATION CALLS -> [TaskManager] @router.get("/{task_id}/logs/sources", response_model=List[str]) async def get_task_log_sources( task_id: str, @@ -337,13 +310,9 @@ async def get_task_log_sources( # #region resolve_task [C:2] [TYPE Function] # @BRIEF Resolve a task that is awaiting mapping. -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: request (ResolveTaskRequest) - The resolution parameters. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task must be in AWAITING_MAPPING status. # @POST: Task is resolved and resumes execution. -# @RETURN: Task - The updated task object. -# @RELATION: CALLS -> [TaskManager] +# @RELATION CALLS -> [TaskManager] @router.post("/{task_id}/resolve", response_model=Task) async def resolve_task( task_id: str, @@ -364,13 +333,9 @@ async def resolve_task( # #region resume_task [C:2] [TYPE Function] # @BRIEF Resume a task that is awaiting input (e.g., passwords). -# @PARAM: task_id (str) - The unique identifier of the task. -# @PARAM: request (ResumeTaskRequest) - The input (passwords). -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task must be in AWAITING_INPUT status. # @POST: Task resumes execution with provided input. -# @RETURN: Task - The updated task object. -# @RELATION: CALLS -> [TaskManager] +# @RELATION CALLS -> [TaskManager] @router.post("/{task_id}/resume", response_model=Task) async def resume_task( task_id: str, @@ -391,11 +356,9 @@ async def resume_task( # #region clear_tasks [C:2] [TYPE Function] # @BRIEF Clear tasks matching the status filter. -# @PARAM: status (Optional[TaskStatus]) - Filter by task status. -# @PARAM: task_manager (TaskManager) - The task manager instance. # @PRE: task_manager is available. # @POST: Tasks are removed from memory/persistence. -# @RELATION: CALLS -> [TaskManager] +# @RELATION CALLS -> [TaskManager] @router.delete("", status_code=status.HTTP_204_NO_CONTENT) async def clear_tasks( status: Optional[TaskStatus] = None, diff --git a/backend/src/api/routes/translate/__init__.py b/backend/src/api/routes/translate/__init__.py index f5275bdb..ac606f65 100644 --- a/backend/src/api/routes/translate/__init__.py +++ b/backend/src/api/routes/translate/__init__.py @@ -1,16 +1,15 @@ -# #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS api,routes,translate] +# #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS api, routes, translate] # @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import. -# @LAYER UI +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [TranslateJobResponse] -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @RELATION DEPENDS_ON -> [DictionaryManager:Class] # @RELATION DEPENDS_ON -> [get_current_user] # @RELATION DEPENDS_ON -> [get_db] # @RELATION DEPENDS_ON -> [has_permission] # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [SupersetClient] -# @PRE All sub-modules importable. Router instance available from ._router. -# @POST Package imports all route modules and re-exports router. Sub-routes registered on shared router. -# @SIDE_EFFECT Registers route handlers on shared APIRouter at import time. +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. """ Translate routes package. diff --git a/backend/src/api/routes/translate/_correction_routes.py b/backend/src/api/routes/translate/_correction_routes.py index 6dff482e..3ec8d274 100644 --- a/backend/src/api/routes/translate/_correction_routes.py +++ b/backend/src/api/routes/translate/_correction_routes.py @@ -1,19 +1,13 @@ -# #region TranslateCorrectionRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,corrections] -# @BRIEF Term correction submission endpoints for applying user feedback to dictionary entries. -# @LAYER UI -# @RELATION DEPENDS_ON -> [DictionaryManager] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE DB session initialized. User authenticated with translate.dictionary.edit permissions. -# @POST Term corrections submitted and applied to dictionary with conflict detection. -# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB based on correction submissions. +# #region TranslateCorrectionRoutesModule [C:2] [TYPE Module] [SEMANTICS api, routes, translate, corrections] +# @BRIEF Term correction submission endpoints. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager @@ -25,18 +19,15 @@ from ....schemas.translate import ( from ._router import router -log = MarkerLogger("TranslateCorrectionRoutes") # ============================================================ # Corrections # ============================================================ -# #region submit_correction [C:4] [TYPE Function] [SEMANTICS api,translate,corrections] -# @BRIEF Submit a single term correction from a run result review. -# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided. -# @POST Correction applied with conflict detection. Result returned. -# @SIDE_EFFECT Creates/updates DictionaryEntry record in DB; logs correction origin. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# #region submit_correction [TYPE Function] +# @BRIEF Submit a term correction from a run result review. +# @PRE: User has translate.dictionary.edit permission. +# @POST: Correction applied with conflict detection. @router.post("/corrections") async def submit_correction( payload: TermCorrectionSubmit, @@ -45,7 +36,7 @@ async def submit_correction( db: Session = Depends(get_db), ): """Submit a term correction from a run result review.""" - log.reason("Correction request", payload={"user": current_user.username}) + logger.info(f"[translate_routes][submit_correction] User: {current_user.username}") if not payload.dictionary_id: raise HTTPException(status_code=422, detail="dictionary_id is required") try: @@ -65,12 +56,10 @@ async def submit_correction( # #endregion submit_correction -# #region submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS api,translate,corrections] -# @BRIEF Submit multiple term corrections atomically with full conflict reporting. -# @PRE User has translate.dictionary.edit permission. Dictionary ID is provided. -# @POST All corrections applied or none with conflict list returned. -# @SIDE_EFFECT Creates/updates multiple DictionaryEntry records in DB atomically. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# #region submit_bulk_corrections [TYPE Function] +# @BRIEF Submit multiple term corrections atomically. +# @PRE: User has translate.dictionary.edit permission. +# @POST: All corrections applied or none with conflict list. @router.post("/corrections/bulk") async def submit_bulk_corrections( payload: TermCorrectionBulkSubmit, @@ -79,7 +68,7 @@ async def submit_bulk_corrections( db: Session = Depends(get_db), ): """Submit multiple term corrections atomically.""" - log.reason("Bulk correction request", payload={"user": current_user.username}) + logger.info(f"[translate_routes][submit_bulk_corrections] User: {current_user.username}") try: corr_list = [] for c in payload.corrections: diff --git a/backend/src/api/routes/translate/_dictionary_routes.py b/backend/src/api/routes/translate/_dictionary_routes.py index 2c5e54a2..5718fed6 100644 --- a/backend/src/api/routes/translate/_dictionary_routes.py +++ b/backend/src/api/routes/translate/_dictionary_routes.py @@ -1,23 +1,14 @@ -# #region TranslateDictionaryRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,dictionaries] +# #region TranslateDictionaryRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, dictionaries] # @BRIEF Terminology Dictionary CRUD, entries management, and import routes. -# @LAYER UI -# @RELATION DEPENDS_ON -> [DictionaryManager] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE DB session initialized. User authenticated with translate.dictionary permissions. -# @POST Dictionaries and entries CRUD completed. Import executed with conflict resolution. -# @SIDE_EFFECT Reads/writes TerminologyDictionary and DictionaryEntry records to DB. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import belief_scope -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslateDictRoutes") from ....dependencies import get_current_user, has_permission from ....plugins.translate.dictionary import DictionaryManager from ....schemas.translate import ( @@ -36,12 +27,10 @@ from ._helpers import _dict_to_response, _get_dictionary_entry_counts # Terminology Dictionaries # ============================================================ -# #region list_dictionaries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] -# @BRIEF List all terminology dictionaries with pagination and entry counts. -# @PRE User has translate.dictionary.view permission. -# @POST Returns paginated list of dictionaries with total count. -# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# #region list_dictionaries [TYPE Function] +# @BRIEF List all terminology dictionaries. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns list of dictionaries with total count. @router.get("/dictionaries") async def list_dictionaries( page: int = Query(1, ge=1), @@ -52,22 +41,20 @@ async def list_dictionaries( ): """List all terminology dictionaries.""" with belief_scope("list_dictionaries"): - log.reason("List dictionaries", payload={"user": current_user.username}) + logger.reason(f"User: {current_user.username}") dicts, total = DictionaryManager.list_dictionaries(db, page=page, page_size=page_size) dict_ids = [d.id for d in dicts] counts = _get_dictionary_entry_counts(db, dict_ids) if dict_ids else {} items = [_dict_to_response(d, counts.get(d.id, 0)) for d in dicts] - log.reflect("Dictionaries listed", payload={"total": total, "returned": len(items)}) + logger.reflect("Dictionaries listed", {"total": total, "returned": len(items)}) return {"items": items, "total": total, "page": page, "page_size": page_size} # #endregion list_dictionaries -# #region get_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] -# @BRIEF Get a single terminology dictionary by ID with entry count. -# @PRE User has translate.dictionary.view permission. -# @POST Returns the dictionary with entry_count or 404. -# @SIDE_EFFECT Reads TerminologyDictionary and DictionaryEntry records from DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# #region get_dictionary [TYPE Function] +# @BRIEF Get a single terminology dictionary by ID. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns the dictionary with entry_count. @router.get("/dictionaries/{dictionary_id}") async def get_dictionary( dictionary_id: str, @@ -77,24 +64,22 @@ async def get_dictionary( ): """Get a terminology dictionary by ID.""" with belief_scope("get_dictionary"): - log.reason("Get dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: d = DictionaryManager.get_dictionary(db, dictionary_id) - from ....models.translate import DictionaryEntry + from ...models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() - log.reflect("Dictionary found", payload={"id": dictionary_id}) + logger.reflect("Dictionary found", {"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) # #endregion get_dictionary -# #region create_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# #region create_dictionary [TYPE Function] # @BRIEF Create a new terminology dictionary. -# @PRE User has translate.dictionary.create permission. Payload validated. -# @POST Dictionary created and returned with zero entries. -# @SIDE_EFFECT Creates TerminologyDictionary record in DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.create permission. +# @POST: Returns the created dictionary. @router.post("/dictionaries", status_code=status.HTTP_201_CREATED) async def create_dictionary( payload: DictionaryCreate, @@ -104,7 +89,7 @@ async def create_dictionary( ): """Create a new terminology dictionary.""" with belief_scope("create_dictionary"): - log.reason("Create dictionary", payload={"user": current_user.username}) + logger.reason(f"User: {current_user.username}") d = DictionaryManager.create_dictionary( db, name=payload.name, @@ -114,17 +99,15 @@ async def create_dictionary( description=payload.description, is_active=payload.is_active, ) - log.reflect("Dictionary created", payload={"id": d.id}) + logger.reflect("Dictionary created", {"id": d.id}) return _dict_to_response(d, 0) # #endregion create_dictionary -# #region update_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# #region update_dictionary [TYPE Function] # @BRIEF Update an existing terminology dictionary. -# @PRE User has translate.dictionary.edit permission. Dictionary exists. -# @POST Dictionary updated and returned with current entry count. -# @SIDE_EFFECT Updates TerminologyDictionary record in DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.edit permission. +# @POST: Returns the updated dictionary. @router.put("/dictionaries/{dictionary_id}") async def update_dictionary( dictionary_id: str, @@ -135,7 +118,7 @@ async def update_dictionary( ): """Update a terminology dictionary.""" with belief_scope("update_dictionary"): - log.reason("Update dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: d = DictionaryManager.update_dictionary( db, @@ -146,21 +129,19 @@ async def update_dictionary( target_dialect=payload.target_dialect, is_active=payload.is_active, ) - from ....models.translate import DictionaryEntry + from ...models.translate import DictionaryEntry entry_count = db.query(DictionaryEntry.id).filter(DictionaryEntry.dictionary_id == dictionary_id).count() - log.reflect("Dictionary updated", payload={"id": dictionary_id}) + logger.reflect("Dictionary updated", {"id": dictionary_id}) return _dict_to_response(d, entry_count) except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) # #endregion update_dictionary -# #region delete_dictionary [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries] +# #region delete_dictionary [TYPE Function] # @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs. -# @PRE User has translate.dictionary.delete permission. Dictionary exists and is not in use. -# @POST Dictionary is deleted or 409 if attached to active jobs. -# @SIDE_EFFECT Deletes TerminologyDictionary record from DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.delete permission. +# @POST: Dictionary is deleted. @router.delete("/dictionaries/{dictionary_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_dictionary( dictionary_id: str, @@ -170,10 +151,10 @@ async def delete_dictionary( ): """Delete a terminology dictionary.""" with belief_scope("delete_dictionary"): - log.reason("Delete dictionary", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: DictionaryManager.delete_dictionary(db, dictionary_id) - log.reflect("Dictionary deleted", payload={"id": dictionary_id}) + logger.reflect("Dictionary deleted", {"id": dictionary_id}) return None except ValueError as e: if "active/scheduled" in str(e): @@ -186,12 +167,10 @@ async def delete_dictionary( # Dictionary Entries CRUD # ============================================================ -# #region list_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] -# @BRIEF List entries for a dictionary with pagination. -# @PRE User has translate.dictionary.view permission. Dictionary exists. -# @POST Returns paginated list of entries. -# @SIDE_EFFECT Reads DictionaryEntry records from DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# #region list_dictionary_entries [TYPE Function] +# @BRIEF List entries for a dictionary. +# @PRE: User has translate.dictionary.view permission. +# @POST: Returns paginated list of entries. @router.get("/dictionaries/{dictionary_id}/entries") async def list_dictionary_entries( dictionary_id: str, @@ -203,7 +182,7 @@ async def list_dictionary_entries( ): """List entries for a dictionary.""" with belief_scope("list_dictionary_entries"): - log.reason("List dictionary entries", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: entries, total = DictionaryManager.list_entries(db, dictionary_id, page=page, page_size=page_size) items = [ @@ -219,19 +198,17 @@ async def list_dictionary_entries( } for e in entries ] - log.reflect("Entries listed", payload={"dict_id": dictionary_id, "total": total}) + logger.reflect("Entries listed", {"dict_id": dictionary_id, "total": total}) return {"items": items, "total": total, "page": page, "page_size": page_size} except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) # #endregion list_dictionary_entries -# #region add_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# #region add_dictionary_entry [TYPE Function] # @BRIEF Add a new entry to a dictionary. -# @PRE User has translate.dictionary.edit permission. Dictionary exists. -# @POST Entry created and returned. -# @SIDE_EFFECT Creates DictionaryEntry record in DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is created. @router.post("/dictionaries/{dictionary_id}/entries", status_code=status.HTTP_201_CREATED) async def add_dictionary_entry( dictionary_id: str, @@ -242,7 +219,7 @@ async def add_dictionary_entry( ): """Add a new entry to a dictionary.""" with belief_scope("add_dictionary_entry"): - log.reason("Add dictionary entry", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") try: entry = DictionaryManager.add_entry( db, dictionary_id, @@ -250,7 +227,7 @@ async def add_dictionary_entry( target_term=payload.target_term, context_notes=payload.context_notes, ) - log.reflect("Entry added", payload={"entry_id": entry.id}) + logger.reflect("Entry added", {"entry_id": entry.id}) return { "id": entry.id, "dictionary_id": entry.dictionary_id, @@ -268,12 +245,10 @@ async def add_dictionary_entry( # #endregion add_dictionary_entry -# #region edit_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# #region edit_dictionary_entry [TYPE Function] # @BRIEF Update an existing dictionary entry. -# @PRE User has translate.dictionary.edit permission. Entry exists. -# @POST Entry updated and returned. -# @SIDE_EFFECT Updates DictionaryEntry record in DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is updated. @router.put("/dictionaries/{dictionary_id}/entries/{entry_id}") async def edit_dictionary_entry( dictionary_id: str, @@ -285,7 +260,7 @@ async def edit_dictionary_entry( ): """Update an existing dictionary entry.""" with belief_scope("edit_dictionary_entry"): - log.reason("Edit dictionary entry", payload={"entry_id": entry_id, "user": current_user.username}) + logger.reason(f"Entry: {entry_id}, User: {current_user.username}") try: entry = DictionaryManager.edit_entry( db, entry_id, @@ -293,7 +268,7 @@ async def edit_dictionary_entry( target_term=payload.target_term, context_notes=payload.context_notes, ) - log.reflect("Entry updated", payload={"entry_id": entry_id}) + logger.reflect("Entry updated", {"entry_id": entry_id}) return { "id": entry.id, "dictionary_id": entry.dictionary_id, @@ -311,12 +286,10 @@ async def edit_dictionary_entry( # #endregion edit_dictionary_entry -# #region delete_dictionary_entry [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,entries] +# #region delete_dictionary_entry [TYPE Function] # @BRIEF Delete a dictionary entry. -# @PRE User has translate.dictionary.edit permission. Entry exists. -# @POST Entry is deleted. -# @SIDE_EFFECT Deletes DictionaryEntry record from DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entry is deleted. @router.delete("/dictionaries/{dictionary_id}/entries/{entry_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_dictionary_entry( dictionary_id: str, @@ -327,22 +300,20 @@ async def delete_dictionary_entry( ): """Delete a dictionary entry.""" with belief_scope("delete_dictionary_entry"): - log.reason("Delete dictionary entry", payload={"entry_id": entry_id, "user": current_user.username}) + logger.reason(f"Entry: {entry_id}, User: {current_user.username}") try: DictionaryManager.delete_entry(db, entry_id) - log.reflect("Entry deleted", payload={"entry_id": entry_id}) + logger.reflect("Entry deleted", {"entry_id": entry_id}) return None except ValueError as e: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(e)) # #endregion delete_dictionary_entry -# #region import_dictionary_entries [C:4] [TYPE Function] [SEMANTICS api,translate,dictionaries,import] +# #region import_dictionary_entries [TYPE Function] # @BRIEF Import entries into a terminology dictionary from CSV/TSV content. -# @PRE User has translate.dictionary.edit permission. Dictionary exists. Content is valid CSV/TSV. -# @POST Entries imported per conflict mode (overwrite/keep_existing/cancel). Import result returned. -# @SIDE_EFFECT Creates/updates DictionaryEntry records in DB. -# @RELATION DEPENDS_ON -> [DictionaryManager] +# @PRE: User has translate.dictionary.edit permission. +# @POST: Entries are imported per conflict mode. @router.post("/dictionaries/{dictionary_id}/import") async def import_dictionary_entries( dictionary_id: str, @@ -353,7 +324,7 @@ async def import_dictionary_entries( ): """Import entries into a terminology dictionary from CSV/TSV content.""" with belief_scope("import_dictionary_entries"): - log.reason("Import dictionary entries", payload={"dict_id": dictionary_id, "user": current_user.username}) + logger.reason(f"Dict: {dictionary_id}, User: {current_user.username}") if payload.on_conflict not in ("overwrite", "keep_existing", "cancel"): raise HTTPException( @@ -370,7 +341,7 @@ async def import_dictionary_entries( on_conflict=payload.on_conflict, preview_only=payload.preview_only, ) - log.reflect("Import completed", payload={ + logger.reflect("Import completed", { "dict_id": dictionary_id, "total": result["total"], "errors": len(result["errors"]), diff --git a/backend/src/api/routes/translate/_helpers.py b/backend/src/api/routes/translate/_helpers.py index e9a6380e..44a2029e 100644 --- a/backend/src/api/routes/translate/_helpers.py +++ b/backend/src/api/routes/translate/_helpers.py @@ -1,12 +1,13 @@ -# #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS api,routes,translate,helpers] +# #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS api, routes, translate, helpers] # @BRIEF Shared helper functions for translate route handlers. -# @LAYER UI +# @LAYER: API +# @RELATION DEPENDS_ON -> [models.translate] from typing import Any, Dict, List from sqlalchemy.orm import Session -# #region _run_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# #region _run_to_response [C:2] [TYPE Function] # @BRIEF Convert TranslationRun ORM to response dict. def _run_to_response(run: Any) -> dict: return { @@ -33,7 +34,7 @@ def _run_to_response(run: Any) -> dict: # #endregion _run_to_response -# #region _dict_to_response [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# #region _dict_to_response [C:2] [TYPE Function] # @BRIEF Convert a TerminologyDictionary ORM model to a response dict with computed entry_count. @staticmethod def _dict_to_response(d: Any, entry_count: int = 0) -> dict: @@ -52,11 +53,11 @@ def _dict_to_response(d: Any, entry_count: int = 0) -> dict: # #endregion _dict_to_response -# #region _get_dictionary_entry_counts [C:2] [TYPE Function] [SEMANTICS helpers,translate] +# #region _get_dictionary_entry_counts [C:2] [TYPE Function] # @BRIEF Get entry counts for a list of dictionary IDs. def _get_dictionary_entry_counts(db: Session, dict_ids: List[str]) -> Dict[str, int]: from sqlalchemy import func - from ....models.translate import DictionaryEntry + from ...models.translate import DictionaryEntry counts = ( db.query(DictionaryEntry.dictionary_id, func.count(DictionaryEntry.id)) .filter(DictionaryEntry.dictionary_id.in_(dict_ids)) diff --git a/backend/src/api/routes/translate/_job_routes.py b/backend/src/api/routes/translate/_job_routes.py index 10617a6d..c1afd7d4 100644 --- a/backend/src/api/routes/translate/_job_routes.py +++ b/backend/src/api/routes/translate/_job_routes.py @@ -1,25 +1,14 @@ -# #region TranslateJobRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,jobs] +# #region TranslateJobRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, jobs] # @BRIEF Translation Job CRUD and datasource column routes. -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslateJobService] -# @RELATION DEPENDS_ON -> [ConfigManager] -# @RELATION DEPENDS_ON -> [SupersetClient] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE ConfigManager and DB session initialized. User authenticated with translate.job permissions. -# @POST Translation job CRUD completed or datasource columns fetched. -# @SIDE_EFFECT Reads/writes TranslationJob records to DB; fetches datasource metadata from Superset API. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.logger import belief_scope -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslateJobRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.service import ( @@ -42,9 +31,10 @@ from ._router import router # Translation Job CRUD # ============================================================ -# #region list_jobs [C:3] [TYPE Function] [SEMANTICS api,translate,jobs] -# @BRIEF List all translation jobs with pagination and optional status filter. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# #region list_jobs [TYPE Function] +# @BRIEF List all translation jobs. +# @PRE: User has translate.job.view permission. +# @POST: Returns list of translation jobs. @router.get("/jobs", response_model=List[TranslateJobResponse]) async def list_jobs( page: int = Query(1, ge=1), @@ -56,7 +46,7 @@ async def list_jobs( config_manager: ConfigManager = Depends(get_config_manager), ): """List all translation jobs.""" - log.reason("Listing jobs", payload={"user": current_user.username}) + logger.info(f"[translate_routes][list_jobs] User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) total, jobs = service.list_jobs( @@ -74,9 +64,10 @@ async def list_jobs( # #endregion list_jobs -# #region get_job [C:3] [TYPE Function] [SEMANTICS api,translate,jobs] +# #region get_job [TYPE Function] # @BRIEF Get a single translation job by ID. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# @PRE: User has translate.job.view permission. +# @POST: Returns the translation job. @router.get("/jobs/{job_id}", response_model=TranslateJobResponse) async def get_job( job_id: str, @@ -86,7 +77,7 @@ async def get_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Get a translation job by ID.""" - log.reason("Getting job", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_job] Job: {job_id}, User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) job = service.get_job(job_id) @@ -97,12 +88,11 @@ async def get_job( # #endregion get_job -# #region create_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# #region create_job [TYPE Function] # @BRIEF Create a new translation job. -# @PRE User has translate.job.create permission. Payload validated. -# @POST Translation job created and returned. -# @SIDE_EFFECT Creates TranslationJob record in DB; validates columns via SupersetClient; caches database_dialect. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# @PRE: User has translate.job.create permission. +# @POST: Returns the created translation job. +# @SIDE_EFFECT: Validates columns via SupersetClient; caches database_dialect. @router.post("/jobs", response_model=TranslateJobResponse, status_code=status.HTTP_201_CREATED) async def create_job( payload: TranslateJobCreate, @@ -112,7 +102,7 @@ async def create_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Create a new translation job.""" - log.reason("Creating job", payload={"user": current_user.username}) + logger.info(f"[translate_routes][create_job] User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) job = service.create_job(payload) @@ -123,12 +113,11 @@ async def create_job( # #endregion create_job -# #region update_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# #region update_job [TYPE Function] # @BRIEF Update an existing translation job. -# @PRE User has translate.job.edit permission. Job exists. -# @POST Translation job updated and returned. -# @SIDE_EFFECT Updates TranslationJob record in DB; re-detects database_dialect if datasource changed. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# @PRE: User has translate.job.edit permission. +# @POST: Returns the updated translation job. +# @SIDE_EFFECT: Re-detects database_dialect if datasource changed. @router.put("/jobs/{job_id}", response_model=TranslateJobResponse) async def update_job( job_id: str, @@ -139,7 +128,7 @@ async def update_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Update a translation job.""" - log.reason("Updating job", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][update_job] Job: {job_id}, User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) job = service.update_job(job_id, payload) @@ -150,12 +139,10 @@ async def update_job( # #endregion update_job -# #region delete_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# #region delete_job [TYPE Function] # @BRIEF Delete a translation job. -# @PRE User has translate.job.delete permission. Job exists. -# @POST Translation job is deleted from DB. -# @SIDE_EFFECT Deletes TranslationJob and associated records from DB. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# @PRE: User has translate.job.delete permission. +# @POST: Job is deleted. @router.delete("/jobs/{job_id}", status_code=status.HTTP_204_NO_CONTENT) async def delete_job( job_id: str, @@ -165,7 +152,7 @@ async def delete_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Delete a translation job.""" - log.reason("Deleting job", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][delete_job] Job: {job_id}, User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) service.delete_job(job_id) @@ -174,12 +161,10 @@ async def delete_job( # #endregion delete_job -# #region duplicate_job [C:4] [TYPE Function] [SEMANTICS api,translate,jobs] +# #region duplicate_job [TYPE Function] # @BRIEF Duplicate a translation job. -# @PRE User has translate.job.create permission. Source job exists. -# @POST Duplicated job created with status DRAFT and returned. -# @SIDE_EFFECT Creates a new TranslationJob record in DB as a copy of the source. -# @RELATION DEPENDS_ON -> [TranslateJobService] +# @PRE: User has translate.job.create permission. +# @POST: Returns the duplicated job with status DRAFT. @router.post("/jobs/{job_id}/duplicate", response_model=DuplicateJobResponse, status_code=status.HTTP_201_CREATED) async def duplicate_job( job_id: str, @@ -189,7 +174,7 @@ async def duplicate_job( config_manager: ConfigManager = Depends(get_config_manager), ): """Duplicate a translation job.""" - log.reason("Duplicating job", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][duplicate_job] Job: {job_id}, User: {current_user.username}") try: service = TranslateJobService(db, config_manager, current_user.username) new_job = service.duplicate_job(job_id) @@ -203,13 +188,14 @@ async def duplicate_job( # #endregion duplicate_job -# #region list_datasources [C:4] [TYPE Function] [SEMANTICS api,translate,datasources] -# @BRIEF List all Superset datasets available for translation jobs. -# @PRE User has translate.job.view permission. -# @POST Returns list of datasets with metadata. -# @SIDE_EFFECT Queries Superset API for available datasets. -# @RELATION DEPENDS_ON -> [TranslateJobService] -# @RELATION DEPENDS_ON -> [SupersetClient] +# #region get_datasource_columns [TYPE Function] +# @BRIEF Get column metadata and database dialect for a Superset datasource. +# @PRE: User has translate.job.view permission. +# @POST: Returns column list with metadata and database_dialect. +# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. +# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time. +# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config. + @router.get("/datasources") async def list_datasources( env_id: str = Query(..., description="Superset environment ID"), @@ -226,18 +212,9 @@ async def list_datasources( datasets = service.fetch_available_datasources(env_id, search) return datasets except Exception as e: - log.explore("List datasources failed", error=str(e)) + logger.error(f"[translate_routes][list_datasources] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=str(e)) -# #endregion list_datasources - -# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS api,translate,datasources] -# @BRIEF Get column metadata and database dialect for a Superset datasource. -# @PRE User has translate.job.view permission. Datasource exists in Superset. -# @POST Returns column list with metadata and database_dialect. -# @SIDE_EFFECT Queries Superset API for dataset detail and database info. -# @RELATION DEPENDS_ON -> [SupersetClient] -# @RELATION DEPENDS_ON -> [ConfigManager] @router.get("/datasources/{datasource_id}/columns", response_model=DatasourceColumnsResponse) async def get_datasource_columns( datasource_id: int, @@ -247,11 +224,10 @@ async def get_datasource_columns( config_manager: ConfigManager = Depends(get_config_manager), ): """Get column metadata and database dialect for a Superset datasource.""" - log.reason("Getting datasource columns", payload={ - "datasource_id": datasource_id, - "env_id": env_id, - "user": current_user.username, - }) + logger.info( + f"[translate_routes][get_datasource_columns] " + f"Datasource: {datasource_id}, Env: {env_id}, User: {current_user.username}" + ) try: result = fetch_datasource_columns_service( datasource_id=datasource_id, @@ -262,7 +238,7 @@ async def get_datasource_columns( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - log.explore("Get datasource columns failed", error=str(e)) + logger.error(f"[translate_routes][get_datasource_columns] Error: {e}") raise HTTPException( status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Failed to fetch datasource columns from Superset: {e}", diff --git a/backend/src/api/routes/translate/_metrics_routes.py b/backend/src/api/routes/translate/_metrics_routes.py index 87ad8ab7..0ccbaf61 100644 --- a/backend/src/api/routes/translate/_metrics_routes.py +++ b/backend/src/api/routes/translate/_metrics_routes.py @@ -1,31 +1,28 @@ -# #region TranslateMetricsRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,metrics] -# @BRIEF Translation Metrics endpoints providing aggregated stats on runs and corrections. -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslationMetrics] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] +# #region TranslateMetricsRoutesModule [C:2] [TYPE Module] [SEMANTICS api, routes, translate, metrics] +# @BRIEF Translation Metrics endpoints. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User from ....dependencies import get_current_user, has_permission from ....plugins.translate.metrics import TranslationMetrics from ._router import router -log = MarkerLogger("TranslateMetricsRoutes") # ============================================================ # Metrics # ============================================================ -# #region get_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics] -# @BRIEF Get translation metrics across all jobs, optionally filtered by job. -# @RELATION DEPENDS_ON -> [TranslationMetrics] +# #region get_metrics [TYPE Function] +# @BRIEF Get translation metrics, optionally filtered by job. +# @PRE: User has translate.metrics.view permission. +# @POST: Returns metrics data. @router.get("/metrics") async def get_metrics( job_id: Optional[str] = Query(None), @@ -34,7 +31,7 @@ async def get_metrics( db: Session = Depends(get_db), ): """Get translation metrics.""" - log.reason("Metrics request", payload={"user": current_user.username}) + logger.info(f"[translate_routes][get_metrics] User: {current_user.username}") try: metrics_svc = TranslationMetrics(db) if job_id: @@ -45,9 +42,10 @@ async def get_metrics( # #endregion get_metrics -# #region get_job_metrics [C:3] [TYPE Function] [SEMANTICS api,translate,metrics] -# @BRIEF Get aggregated metrics for a specific translation job. -# @RELATION DEPENDS_ON -> [TranslationMetrics] +# #region get_job_metrics [TYPE Function] +# @BRIEF Get aggregated metrics for a specific job. +# @PRE: User has translate.metrics.view permission. +# @POST: Returns metrics dict. @router.get("/jobs/{job_id}/metrics") async def get_job_metrics( job_id: str, @@ -56,7 +54,7 @@ async def get_job_metrics( db: Session = Depends(get_db), ): """Get aggregated metrics for a specific job.""" - log.reason("Job metrics request", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_job_metrics] Job: {job_id}, User: {current_user.username}") try: metrics_svc = TranslationMetrics(db) return metrics_svc.get_job_metrics(job_id) diff --git a/backend/src/api/routes/translate/_preview_routes.py b/backend/src/api/routes/translate/_preview_routes.py index 4134d796..ea31caaf 100644 --- a/backend/src/api/routes/translate/_preview_routes.py +++ b/backend/src/api/routes/translate/_preview_routes.py @@ -1,23 +1,14 @@ -# #region TranslatePreviewRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,preview] +# #region TranslatePreviewRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, preview] # @BRIEF Translation Preview session management routes. -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslationPreview] -# @RELATION DEPENDS_ON -> [ConfigManager] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions. -# @POST Preview sessions created, rows reviewed/accepted, records queried. -# @SIDE_EFFECT Creates/updates preview sessions and rows in DB; fetches sample data from Superset; calls LLM provider. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslatePreviewRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.preview import TranslationPreview @@ -35,13 +26,11 @@ from ._router import router # Preview # ============================================================ -# #region preview_translation [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# #region preview_translation [TYPE Function] # @BRIEF Preview a translation before applying it. -# @PRE User has translate.job.execute permission. Job exists. -# @POST Preview session created with sample rows and cost estimation. -# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates PreviewSession and PreviewRecord rows in DB. -# @RELATION DEPENDS_ON -> [TranslationPreview] -# @RELATION DEPENDS_ON -> [SupersetClient] +# @PRE: User has translate.job.execute permission. +# @POST: Returns a preview session with records and cost estimation. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. @router.post("/jobs/{job_id}/preview", status_code=status.HTTP_201_CREATED) async def preview_translation( job_id: str, @@ -52,7 +41,7 @@ async def preview_translation( config_manager: ConfigManager = Depends(get_config_manager), ): """Preview a translation before applying it.""" - log.reason("Preview translation", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][preview_translation] Job: {job_id}, User: {current_user.username}") if payload is None: payload = PreviewRequest() try: @@ -67,17 +56,15 @@ async def preview_translation( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - log.explore("Preview translation error", error=str(e)) + logger.error(f"[translate_routes][preview_translation] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Preview failed: {e}") # #endregion preview_translation -# #region update_preview_row [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# #region update_preview_row [TYPE Function] # @BRIEF Approve, edit, or reject a preview row. -# @PRE User has translate.job.execute permission. Preview row exists. -# @POST Preview row status updated to APPROVED/REJECTED/EDITED. -# @SIDE_EFFECT Updates PreviewRecord status in DB. -# @RELATION DEPENDS_ON -> [TranslationPreview] +# @PRE: User has translate.job.execute permission. +# @POST: Preview row status is updated. @router.put("/jobs/{job_id}/preview/rows/{row_key}") async def update_preview_row( job_id: str, @@ -89,7 +76,7 @@ async def update_preview_row( config_manager: ConfigManager = Depends(get_config_manager), ): """Approve, edit, or reject a preview row.""" - log.reason("Update preview row", payload={"job_id": job_id, "row": row_key, "action": payload.action}) + logger.info(f"[translate_routes][update_preview_row] Job: {job_id}, Row: {row_key}, Action: {payload.action}") try: preview_service = TranslationPreview(db, config_manager, current_user.username) result = preview_service.update_preview_row( @@ -105,12 +92,10 @@ async def update_preview_row( # #endregion update_preview_row -# #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# #region accept_preview_session [TYPE Function] # @BRIEF Accept a preview session, marking it as the quality gate for full execution. -# @PRE User has translate.job.execute permission. Job has an ACTIVE preview session. -# @POST Preview session status set to APPLIED. Full execution can proceed. -# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB. -# @RELATION DEPENDS_ON -> [TranslationPreview] +# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session. +# @POST: Preview session is marked as APPLIED; full execution can proceed. @router.post("/jobs/{job_id}/preview/accept") async def accept_preview_session( job_id: str, @@ -120,7 +105,7 @@ async def accept_preview_session( config_manager: ConfigManager = Depends(get_config_manager), ): """Accept a preview session, enabling full translation execution.""" - log.reason("Accept preview session", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][accept_preview_session] Job: {job_id}, User: {current_user.username}") try: preview_service = TranslationPreview(db, config_manager, current_user.username) result = preview_service.accept_preview_session(job_id=job_id) @@ -130,12 +115,10 @@ async def accept_preview_session( # #endregion accept_preview_session -# #region apply_preview [C:4] [TYPE Function] [SEMANTICS api,translate,preview] +# #region apply_preview [TYPE Function] # @BRIEF Apply a preview session (alias for accept when accepting at session level). -# @PRE User has translate.job.execute permission. Session exists. -# @POST Preview session applied and marked as quality gate. -# @SIDE_EFFECT Updates PreviewSession status to APPLIED in DB. -# @RELATION DEPENDS_ON -> [TranslationPreview] +# @PRE: User has translate.job.execute permission. +# @POST: Preview is applied. @router.post("/preview/{session_id}/apply") async def apply_preview( session_id: str, @@ -145,9 +128,9 @@ async def apply_preview( config_manager: ConfigManager = Depends(get_config_manager), ): """Apply a preview session by session ID.""" - log.reason("Apply preview", payload={"session_id": session_id, "user": current_user.username}) + logger.info(f"[translate_routes][apply_preview] Session: {session_id}, User: {current_user.username}") # Find job_id from session - from ....models.translate import TranslationPreviewSession + from ...models.translate import TranslationPreviewSession session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() if not session: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") @@ -164,10 +147,10 @@ async def apply_preview( # Preview Records # ============================================================ -# #region get_preview_records [C:3] [TYPE Function] [SEMANTICS api,translate,preview] +# #region get_preview_records [TYPE Function] # @BRIEF Get records for a preview session. -# @RELATION DEPENDS_ON -> [TranslationPreviewSession] -# @RELATION DEPENDS_ON -> [TranslationPreviewRecord] +# @PRE: User has translate.job.view permission. +# @POST: Returns list of preview records. @router.get("/preview/{session_id}/records", response_model=List[PreviewRow]) async def get_preview_records( session_id: str, @@ -176,9 +159,9 @@ async def get_preview_records( db: Session = Depends(get_db), ): """Get records for a preview session.""" - log.reason("Get preview records", payload={"session_id": session_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_preview_records] Session: {session_id}, User: {current_user.username}") try: - from ....models.translate import TranslationPreviewSession, TranslationPreviewRecord + from ...models.translate import TranslationPreviewSession, TranslationPreviewRecord session = db.query(TranslationPreviewSession).filter(TranslationPreviewSession.id == session_id).first() if not session: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Preview session '{session_id}' not found") @@ -203,7 +186,7 @@ async def get_preview_records( except HTTPException: raise except Exception as e: - log.explore("Get preview records error", error=str(e)) + logger.error(f"[translate_routes][get_preview_records] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_preview_records diff --git a/backend/src/api/routes/translate/_router.py b/backend/src/api/routes/translate/_router.py index bb01b6b5..2c4aa9f8 100644 --- a/backend/src/api/routes/translate/_router.py +++ b/backend/src/api/routes/translate/_router.py @@ -1,9 +1,11 @@ -# #region TranslateRouterModule [C:1] [TYPE Module] [SEMANTICS api,routes,translate,router] -# @LAYER UI +# #region TranslateRouterModule [C:1] [TYPE Module] [SEMANTICS api, routes, translate, router] +# @BRIEF APIRouter instance for translate routes. +# @LAYER: API from fastapi import APIRouter -# #region translate_router [C:1] [TYPE Global] +# #region translate_router [TYPE Global] +# @BRIEF APIRouter instance for all translate sub-routes. router = APIRouter(prefix="/api/translate", tags=["translate"]) # #endregion translate_router diff --git a/backend/src/api/routes/translate/_run_list_routes.py b/backend/src/api/routes/translate/_run_list_routes.py index 8f2f5a8d..9226dd78 100644 --- a/backend/src/api/routes/translate/_run_list_routes.py +++ b/backend/src/api/routes/translate/_run_list_routes.py @@ -1,20 +1,14 @@ -# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api,routes,translate,runs,list,detail] +# #region TranslateRunListRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, runs, list, detail] # @BRIEF Translation Run listing, detail, and CSV download routes (cross-job). -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] -# @RELATION DEPENDS_ON -> [TranslationRun] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslateRunListRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.orchestrator import TranslationOrchestrator @@ -27,13 +21,14 @@ from ._router import router # List Runs (cross-job) # ============================================================ -# #region list_runs [C:3] [TYPE Function] [SEMANTICS api,translate,runs,list] +# #region list_runs [TYPE Function] # @BRIEF List all runs with cross-job filtering and pagination. -# @RELATION DEPENDS_ON -> [TranslationRun] +# @PRE: User has translate.history.view permission. +# @POST: Returns paginated list of runs. @router.get("/runs") async def list_runs( job_id: Optional[str] = Query(None), - run_status: Optional[str] = Query(None, alias="status"), + run_status: Optional[str] = Query(None), trigger_type: Optional[str] = Query(None), created_by: Optional[str] = Query(None), date_from: Optional[str] = Query(None), @@ -45,9 +40,9 @@ async def list_runs( db: Session = Depends(get_db), ): """List all runs with cross-job filtering and pagination.""" - log.reason("Listing runs", payload={"user": current_user.username}) + logger.info(f"[translate_routes][list_runs] User: {current_user.username}") try: - from ....models.translate import TranslationRun + from ...models.translate import TranslationRun query = db.query(TranslationRun) if job_id: @@ -105,7 +100,7 @@ async def list_runs( return {"items": items, "total": total, "page": page, "page_size": page_size} except Exception as e: - log.explore("List runs error", error=str(e)) + logger.error(f"[translate_routes][list_runs] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion list_runs @@ -114,11 +109,10 @@ async def list_runs( # Run Detail # ============================================================ -# #region get_run_detail [C:3] [TYPE Function] [SEMANTICS api,translate,runs,detail] +# #region get_run_detail [TYPE Function] # @BRIEF Get detailed run info with config_snapshot, records, events. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] -# @RELATION DEPENDS_ON -> [TranslationEventLog] -# @RELATION DEPENDS_ON -> [TranslationRun] +# @PRE: User has translate.history.view permission. +# @POST: Returns run detail with records and events. @router.get("/runs/{run_id}/detail") async def get_run_detail( run_id: str, @@ -128,7 +122,7 @@ async def get_run_detail( config_manager: ConfigManager = Depends(get_config_manager), ): """Get detailed run info with config_snapshot, records, events.""" - log.reason("Get run detail", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_run_detail] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) status_info = orch.get_run_status(run_id) @@ -137,7 +131,7 @@ async def get_run_detail( event_summary = TranslationEventLog(db).get_run_event_summary(run_id) # Get config snapshot from run - from ....models.translate import TranslationRun + from ...models.translate import TranslationRun run = db.query(TranslationRun).filter(TranslationRun.id == run_id).first() config_snapshot = run.config_snapshot if run else None @@ -158,9 +152,10 @@ async def get_run_detail( # Download Skipped CSV # ============================================================ -# #region download_skipped_csv [C:3] [TYPE Function] [SEMANTICS api,translate,runs,csv] +# #region download_skipped_csv [TYPE Function] # @BRIEF Download a CSV of skipped translation records from a run. -# @RELATION DEPENDS_ON -> [TranslationRecord] +# @PRE: User has translate.job.view permission. +# @POST: Returns CSV file of skipped records. @router.get("/runs/{run_id}/skipped.csv") async def download_skipped_csv( run_id: str, @@ -169,9 +164,9 @@ async def download_skipped_csv( db: Session = Depends(get_db), ): """Download skipped translation records as CSV.""" - log.reason("Download skipped CSV", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][download_skipped_csv] Run: {run_id}, User: {current_user.username}") try: - from ....models.translate import TranslationRecord + from ...models.translate import TranslationRecord from fastapi.responses import StreamingResponse import csv import io @@ -202,7 +197,7 @@ async def download_skipped_csv( headers={"Content-Disposition": f"attachment; filename=skipped_{run_id}.csv"}, ) except Exception as e: - log.explore("Download skipped CSV error", error=str(e)) + logger.error(f"[translate_routes][download_skipped_csv] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion download_skipped_csv diff --git a/backend/src/api/routes/translate/_run_routes.py b/backend/src/api/routes/translate/_run_routes.py index f98fe312..ad811c52 100644 --- a/backend/src/api/routes/translate/_run_routes.py +++ b/backend/src/api/routes/translate/_run_routes.py @@ -1,28 +1,18 @@ -# #region TranslateRunRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,runs] +# #region TranslateRunRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, runs] # @BRIEF Translation Run execution, history, status, records and batches routes. -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] -# @RELATION DEPENDS_ON -> [ConfigManager] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE ConfigManager and DB session initialized. User authenticated with translate permissions. -# @POST Translation run executed, manipulated, or queried. Results returned to caller. -# @SIDE_EFFECT Creates/updates TranslationRun records in DB; spawns background threads for execution; queries Superset API. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from ....core.database import get_db, SessionLocal -from ....core.cot_logger import MarkerLogger +from ....core.database import get_db +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslateRunRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.orchestrator import TranslationOrchestrator from ....plugins.translate.events import TranslationEventLog -from ....models.translate import TranslationRun from ._router import router from ._helpers import _run_to_response @@ -32,64 +22,43 @@ from ._helpers import _run_to_response # Translation Run / Execute # ============================================================ -# #region run_translation [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# #region run_translation [TYPE Function] # @BRIEF Execute a translation job (trigger a run). -# @PRE User has translate.job.execute permission. Job exists and preview is accepted. -# @POST Translation run created and started in background. Run object returned. -# @SIDE_EFFECT Creates TranslationRun record in DB; spawns background thread for translation execution. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.job.execute permission. +# @POST: Returns the created translation run. @router.post("/jobs/{job_id}/run", status_code=status.HTTP_201_CREATED) async def run_translation( job_id: str, - full_translation: bool = Query(False, description="If True, fetch ALL rows from Superset dataset instead of preview-only rows"), current_user: User = Depends(get_current_user), _ = Depends(has_permission("translate.job", "EXECUTE")), db: Session = Depends(get_db), config_manager: ConfigManager = Depends(get_config_manager), ): - """Execute a translation job (trigger a run). - - By default runs translation on preview-approved rows only. - Set full_translation=true to fetch ALL rows from the Superset source dataset. - """ - log.reason("Run translation", payload={"job_id": job_id, "user": current_user.username, "full_translation": full_translation}) + """Execute a translation job (trigger a run).""" + logger.info(f"[translate_routes][run_translation] Job: {job_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.start_run(job_id=job_id, is_scheduled=False) - # Execute asynchronously in background with a FRESH session. - # The request-scoped session from Depends(get_db) is closed after the handler returns, - # so the background thread must create its own session to avoid "This transaction is closed". + # Execute asynchronously in background import threading - - def _execute_background(): - thread_db = SessionLocal() - try: - thread_orch = TranslationOrchestrator(thread_db, config_manager, current_user.username) - # Reload run from DB with the new session (the passed run object is detached) - thread_run = thread_db.query(TranslationRun).filter(TranslationRun.id == run.id).first() - if thread_run: - thread_orch.execute_run(thread_run, full_translation=full_translation) - except Exception as e: - log.explore("Background execution error", error=str(e)) - finally: - thread_db.close() - - threading.Thread(target=_execute_background, daemon=True).start() + threading.Thread( + target=orch.execute_run, + args=(run,), + daemon=True, + ).start() return _run_to_response(run) except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - log.explore("Run translation error", error=str(e)) + logger.error(f"[translate_routes][run_translation] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Run failed: {e}") # #endregion run_translation -# #region retry_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# #region retry_run [TYPE Function] # @BRIEF Retry failed batches in a translation run. -# @PRE User has translate.job.execute permission. Run exists and has failed batches. -# @POST Failed batches re-executed. Updated run returned. -# @SIDE_EFFECT Updates TranslationRun and TranslationBatch records in DB. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.job.execute permission. +# @POST: Returns the updated translation run. @router.post("/runs/{run_id}/retry") async def retry_run( run_id: str, @@ -99,7 +68,7 @@ async def retry_run( config_manager: ConfigManager = Depends(get_config_manager), ): """Retry failed batches in a translation run.""" - log.reason("Retry run", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][retry_run] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.retry_failed_batches(run_id) @@ -107,17 +76,15 @@ async def retry_run( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - log.explore("Retry run error", error=str(e)) + logger.error(f"[translate_routes][retry_run] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry failed: {e}") # #endregion retry_run -# #region retry_insert [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# #region retry_insert [TYPE Function] # @BRIEF Retry the SQL insert phase for a completed run. -# @PRE User has translate.job.execute permission. Run completed with insert_status FAILED. -# @POST SQL insert phase re-executed. Updated run returned. -# @SIDE_EFFECT Re-executes SQL insert into Superset; updates TranslationRun insert_status. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.job.execute permission. +# @POST: Returns the updated run. @router.post("/runs/{run_id}/retry-insert") async def retry_insert( run_id: str, @@ -127,7 +94,7 @@ async def retry_insert( config_manager: ConfigManager = Depends(get_config_manager), ): """Retry the SQL insert phase for a completed run.""" - log.reason("Retry insert", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][retry_insert] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.retry_insert(run_id) @@ -135,17 +102,15 @@ async def retry_insert( except ValueError as e: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) except Exception as e: - log.explore("Retry insert error", error=str(e)) + logger.error(f"[translate_routes][retry_insert] Error: {e}") raise HTTPException(status_code=status.HTTP_502_BAD_GATEWAY, detail=f"Retry insert failed: {e}") # #endregion retry_insert -# #region cancel_run [C:4] [TYPE Function] [SEMANTICS api,translate,runs] +# #region cancel_run [TYPE Function] # @BRIEF Cancel a running translation. -# @PRE User has translate.job.execute permission. Run is in RUNNING state. -# @POST Run is cancelled. Updated run returned. -# @SIDE_EFFECT Updates TranslationRun status to CANCELLED in DB. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.job.execute permission. +# @POST: Run is cancelled. @router.post("/runs/{run_id}/cancel") async def cancel_run( run_id: str, @@ -155,7 +120,7 @@ async def cancel_run( config_manager: ConfigManager = Depends(get_config_manager), ): """Cancel a running translation.""" - log.reason("Cancel run", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][cancel_run] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) run = orch.cancel_run(run_id) @@ -165,9 +130,10 @@ async def cancel_run( # #endregion cancel_run -# #region get_run_history [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# #region get_run_history [TYPE Function] # @BRIEF Get run history for a translation job. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.history.view permission. +# @POST: Returns list of runs. @router.get("/jobs/{job_id}/runs") async def get_run_history( job_id: str, @@ -179,7 +145,7 @@ async def get_run_history( config_manager: ConfigManager = Depends(get_config_manager), ): """Get run history for a translation job.""" - log.reason("Get run history", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_run_history] Job: {job_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) total, runs = orch.get_run_history(job_id, page=page, page_size=page_size) @@ -189,9 +155,10 @@ async def get_run_history( # #endregion get_run_history -# #region get_run_status [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# #region get_run_status [TYPE Function] # @BRIEF Get status and statistics for a translation run. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.history.view permission. +# @POST: Returns run details with statistics. @router.get("/runs/{run_id}") async def get_run_status( run_id: str, @@ -201,7 +168,7 @@ async def get_run_status( config_manager: ConfigManager = Depends(get_config_manager), ): """Get status and statistics for a translation run.""" - log.reason("Get run status", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_run_status] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) return orch.get_run_status(run_id) @@ -210,9 +177,10 @@ async def get_run_status( # #endregion get_run_status -# #region get_run_records [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# #region get_run_records [TYPE Function] # @BRIEF Get paginated records for a translation run. -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] +# @PRE: User has translate.history.view permission. +# @POST: Returns paginated records. @router.get("/runs/{run_id}/records") async def get_run_records( run_id: str, @@ -225,7 +193,7 @@ async def get_run_records( config_manager: ConfigManager = Depends(get_config_manager), ): """Get paginated records for a translation run.""" - log.reason("Get run records", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_run_records] Run: {run_id}, User: {current_user.username}") try: orch = TranslationOrchestrator(db, config_manager, current_user.username) return orch.get_run_records(run_id, page=page, page_size=page_size, status_filter=status) @@ -238,9 +206,10 @@ async def get_run_records( # Batches # ============================================================ -# #region get_batches [C:3] [TYPE Function] [SEMANTICS api,translate,runs] +# #region get_batches [TYPE Function] # @BRIEF Get batches for a translation run. -# @RELATION DEPENDS_ON -> [TranslationBatch] +# @PRE: User has translate.job.view permission. +# @POST: Returns list of batches. @router.get("/runs/{run_id}/batches") async def get_batches( run_id: str, @@ -249,9 +218,9 @@ async def get_batches( db: Session = Depends(get_db), ): """Get batches for a translation run.""" - log.reason("Get batches", payload={"run_id": run_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_batches] Run: {run_id}, User: {current_user.username}") try: - from ....models.translate import TranslationBatch + from ...models.translate import TranslationBatch batches = ( db.query(TranslationBatch) .filter(TranslationBatch.run_id == run_id) @@ -274,7 +243,7 @@ async def get_batches( for b in batches ] except Exception as e: - log.explore("Get batches error", error=str(e)) + logger.error(f"[translate_routes][get_batches] Error: {e}") raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e)) # #endregion get_batches diff --git a/backend/src/api/routes/translate/_schedule_routes.py b/backend/src/api/routes/translate/_schedule_routes.py index adad895d..95bf02af 100644 --- a/backend/src/api/routes/translate/_schedule_routes.py +++ b/backend/src/api/routes/translate/_schedule_routes.py @@ -1,23 +1,14 @@ -# #region TranslateScheduleRoutesModule [C:4] [TYPE Module] [SEMANTICS api,routes,translate,schedule] +# #region TranslateScheduleRoutesModule [C:3] [TYPE Module] [SEMANTICS api, routes, translate, schedule] # @BRIEF Translation Schedule management routes. -# @LAYER UI -# @RELATION DEPENDS_ON -> [TranslationScheduler] -# @RELATION DEPENDS_ON -> [ConfigManager] -# @RELATION DEPENDS_ON -> [get_current_user] -# @RELATION DEPENDS_ON -> [get_db] -# @PRE ConfigManager and DB session initialized. User authenticated with translate.schedule permissions. -# @POST Schedule CRUD operations completed. APScheduler jobs registered/unregistered. -# @SIDE_EFFECT Creates/updates/deletes TranslationSchedule records in DB; registers/removes jobs in APScheduler. +# @LAYER: API from fastapi import APIRouter, Depends, HTTPException, status, Query from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from ....core.database import get_db -from ....core.cot_logger import MarkerLogger +from ....core.logger import logger, belief_scope from ....schemas.auth import User - -log = MarkerLogger("TranslateScheduleRoutes") from ....dependencies import get_current_user, has_permission, get_config_manager from ....core.config_manager import ConfigManager from ....plugins.translate.scheduler import TranslationScheduler @@ -30,9 +21,10 @@ from ._router import router # Schedule # ============================================================ -# #region get_schedule [C:3] [TYPE Function] [SEMANTICS api,translate,schedule] -# @BRIEF Get the schedule configuration for a translation job. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# #region get_schedule [TYPE Function] +# @BRIEF Get the schedule for a translation job. +# @PRE: User has translate.schedule.view permission. +# @POST: Returns the schedule configuration. @router.get("/jobs/{job_id}/schedule") async def get_schedule( job_id: str, @@ -42,7 +34,7 @@ async def get_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Get schedule for a translation job.""" - log.reason("Get schedule", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_schedule] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.get_schedule(job_id) @@ -63,12 +55,10 @@ async def get_schedule( # #endregion get_schedule -# #region set_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# #region set_schedule [TYPE Function] # @BRIEF Set or update the schedule for a translation job. -# @PRE User has translate.schedule.manage permission. Job exists. -# @POST Schedule created or updated. APScheduler job registered. -# @SIDE_EFFECT Creates/updates TranslationSchedule record in DB; registers job in APScheduler. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is created or updated. @router.put("/jobs/{job_id}/schedule") async def set_schedule( job_id: str, @@ -79,11 +69,11 @@ async def set_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Set or update schedule for a translation job.""" - log.reason("Set schedule", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][set_schedule] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) # Check if schedule already exists - from ....models.translate import TranslationSchedule + from ...models.translate import TranslationSchedule existing = db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() @@ -102,7 +92,7 @@ async def set_schedule( is_active=payload.is_active, ) # Register with APScheduler via SchedulerService - from ....dependencies import get_scheduler_service + from ...dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.add_translation_job( schedule_id=schedule.id, @@ -126,12 +116,10 @@ async def set_schedule( # #endregion set_schedule -# #region enable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# #region enable_schedule [TYPE Function] # @BRIEF Enable a schedule for a translation job. -# @PRE User has translate.schedule.manage permission. Schedule exists. -# @POST Schedule is enabled. APScheduler job registered. -# @SIDE_EFFECT Updates TranslationSchedule is_active to True in DB; registers job in APScheduler. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is enabled. @router.post("/jobs/{job_id}/schedule/enable") async def enable_schedule( job_id: str, @@ -141,11 +129,11 @@ async def enable_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Enable a schedule for a translation job.""" - log.reason("Enable schedule", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][enable_schedule] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.set_schedule_active(job_id, is_active=True) - from ....dependencies import get_scheduler_service + from ...dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.add_translation_job( schedule_id=schedule.id, @@ -159,12 +147,10 @@ async def enable_schedule( # #endregion enable_schedule -# #region disable_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# #region disable_schedule [TYPE Function] # @BRIEF Disable a schedule for a translation job. -# @PRE User has translate.schedule.manage permission. Schedule exists. -# @POST Schedule is disabled. APScheduler job removed. -# @SIDE_EFFECT Updates TranslationSchedule is_active to False in DB; removes job from APScheduler. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is disabled. @router.post("/jobs/{job_id}/schedule/disable") async def disable_schedule( job_id: str, @@ -174,11 +160,11 @@ async def disable_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Disable a schedule for a translation job.""" - log.reason("Disable schedule", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][disable_schedule] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.set_schedule_active(job_id, is_active=False) - from ....dependencies import get_scheduler_service + from ...dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.remove_translation_job(schedule_id=job_id) return {"status": "disabled", "job_id": job_id} @@ -187,12 +173,10 @@ async def disable_schedule( # #endregion disable_schedule -# #region delete_schedule [C:4] [TYPE Function] [SEMANTICS api,translate,schedule] +# #region delete_schedule [TYPE Function] # @BRIEF Delete the schedule for a translation job. -# @PRE User has translate.schedule.manage permission. Schedule exists. -# @POST Schedule is removed. APScheduler job removed. -# @SIDE_EFFECT Deletes TranslationSchedule record from DB; removes job from APScheduler. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# @PRE: User has translate.schedule.manage permission. +# @POST: Schedule is removed. @router.delete("/jobs/{job_id}/schedule", status_code=status.HTTP_204_NO_CONTENT) async def delete_schedule( job_id: str, @@ -202,11 +186,11 @@ async def delete_schedule( config_manager: ConfigManager = Depends(get_config_manager), ): """Delete schedule for a translation job.""" - log.reason(f"Delete schedule", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][delete_schedule] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) scheduler.delete_schedule(job_id) - from ....dependencies import get_scheduler_service + from ...dependencies import get_scheduler_service sched_svc = get_scheduler_service() sched_svc.remove_translation_job(schedule_id=job_id) return None @@ -215,9 +199,10 @@ async def delete_schedule( # #endregion delete_schedule -# #region get_next_executions [C:3] [TYPE Function] [SEMANTICS api,translate,schedule] -# @BRIEF Preview next N execution times for a job's schedule. -# @RELATION DEPENDS_ON -> [TranslationScheduler] +# #region get_next_executions [TYPE Function] +# @BRIEF Preview next N executions for a job's schedule. +# @PRE: User has translate.schedule.view permission. +# @POST: Returns next execution times. @router.get("/jobs/{job_id}/schedule/next-executions") async def get_next_executions( job_id: str, @@ -228,7 +213,7 @@ async def get_next_executions( config_manager: ConfigManager = Depends(get_config_manager), ): """Preview next N executions for a job's schedule.""" - log.reason(f"Get next executions", payload={"job_id": job_id, "user": current_user.username}) + logger.info(f"[translate_routes][get_next_executions] Job: {job_id}, User: {current_user.username}") try: scheduler = TranslationScheduler(db, config_manager, current_user.username) schedule = scheduler.get_schedule(job_id) diff --git a/backend/src/app.py b/backend/src/app.py index 99a3dbd9..b0774f6f 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -1,14 +1,14 @@ # #region AppModule [C:5] [TYPE Module] [SEMANTICS app, main, entrypoint, fastapi] # @BRIEF The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming. -# @LAYER UI (API) -# @PRE Python environment and dependencies installed; configuration database available. -# @POST FastAPI app instance is created, middleware configured, and routes registered. -# @SIDE_EFFECT Starts background scheduler and binds network ports for HTTP/WS traffic. -# @DATA_CONTRACT [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream] -# @INVARIANT Only one FastAPI app instance exists per process. -# @INVARIANT All WebSocket connections must be properly cleaned up on disconnect. +# @LAYER: UI (API) # @RELATION DEPENDS_ON -> [AppDependencies] # @RELATION DEPENDS_ON -> [ApiRoutesModule] +# @INVARIANT: Only one FastAPI app instance exists per process. +# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect. +# @PRE: Python environment and dependencies installed; configuration database available. +# @POST: FastAPI app instance is created, middleware configured, and routes registered. +# @SIDE_EFFECT: Starts background scheduler and binds network ports for HTTP/WS traffic. +# @DATA_CONTRACT: [HTTP Request | WS Message] -> [HTTP Response | JSON Log Stream] import os from pathlib import Path @@ -27,9 +27,6 @@ from .dependencies import get_task_manager, get_scheduler_service from .core.encryption_key import ensure_encryption_key from .core.utils.network import NetworkError from .core.logger import logger, belief_scope -from .core.cot_logger import MarkerLogger - -startup_log = MarkerLogger("Startup") from .core.database import AuthSessionLocal from .core.auth.security import get_password_hash from .models.auth import User, Role @@ -72,7 +69,7 @@ app = FastAPI( # #region ensure_initial_admin_user [C:3] [TYPE Function] # @BRIEF Ensures initial admin user exists when bootstrap env flags are enabled. -# @RELATION DEPENDS_ON -> [AuthRepository] +# @RELATION DEPENDS_ON -> AuthRepository def ensure_initial_admin_user() -> None: raw_flag = os.getenv("INITIAL_ADMIN_CREATE", "false").strip().lower() if raw_flag not in {"1", "true", "yes", "on"}: @@ -127,17 +124,16 @@ def ensure_initial_admin_user() -> None: # #region startup_event [C:3] [TYPE Function] # @BRIEF Handles application startup tasks, such as starting the scheduler. -# @PRE None. -# @POST Scheduler is started. # @RELATION CALLS -> [AppDependencies] +# @PRE: None. +# @POST: Scheduler is started. # Startup event @app.on_event("startup") async def startup_event(): - startup_log.reason("Enter startup event") - ensure_encryption_key() - ensure_initial_admin_user() - scheduler = get_scheduler_service() - startup_log.reflect("Application startup completed") + with belief_scope("startup_event"): + ensure_encryption_key() + ensure_initial_admin_user() + scheduler = get_scheduler_service() scheduler.start() @@ -146,9 +142,9 @@ async def startup_event(): # #region shutdown_event [C:3] [TYPE Function] # @BRIEF Handles application shutdown tasks, such as stopping the scheduler. -# @PRE None. -# @POST Scheduler is stopped. # @RELATION CALLS -> [AppDependencies] +# @PRE: None. +# @POST: Scheduler is stopped. # Shutdown event @app.on_event("shutdown") async def shutdown_event(): @@ -160,7 +156,7 @@ async def shutdown_event(): # #endregion shutdown_event # #region app_middleware [TYPE Block] -# @BRIEF Configure application-wide middleware (Session, CORS, TraceContext). +# @BRIEF Configure application-wide middleware (Session, CORS). # Configure Session Middleware (required by Authlib for OAuth2 flow) from .core.auth.config import auth_config @@ -174,27 +170,17 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) - -# Configure Trace Context Middleware (seeds trace_id per request) -from .core.middleware.trace import TraceContextMiddleware - -app.add_middleware(TraceContextMiddleware) # #endregion app_middleware # #region network_error_handler [C:1] [TYPE Function] # @BRIEF Global exception handler for NetworkError. -# @PRE request is a FastAPI Request object. -# @POST Returns 503 HTTP Exception. -# @PARAM: request (Request) - The incoming request object. -# @PARAM: exc (NetworkError) - The exception instance. -eh_log = MarkerLogger("ExceptionHandler") - - +# @PRE: request is a FastAPI Request object. +# @POST: Returns 503 HTTP Exception. @app.exception_handler(NetworkError) async def network_error_handler(request: Request, exc: NetworkError): with belief_scope("network_error_handler"): - eh_log.explore("Unhandled NetworkError in request", error=str(exc), payload={"url": str(request.url)}) + logger.error(f"Network error: {exc}") return HTTPException( status_code=503, detail="Environment unavailable. Please check if the Superset instance is running.", @@ -206,11 +192,9 @@ async def network_error_handler(request: Request, exc: NetworkError): # #region log_requests [C:3] [TYPE Function] # @BRIEF Middleware to log incoming HTTP requests and their response status. -# @PRE request is a FastAPI Request object. -# @POST Logs request and response details. # @RELATION DEPENDS_ON -> [LoggerModule] -# @PARAM: request (Request) - The incoming request object. -# @PARAM: call_next (Callable) - The next middleware or route handler. +# @PRE: request is a FastAPI Request object. +# @POST: Logs request and response details. @app.middleware("http") async def log_requests(request: Request, call_next): with belief_scope("log_requests"): @@ -280,19 +264,19 @@ app.include_router(translate.router) # #region api.include_routers [C:1] [TYPE Action] [SEMANTICS routes, registration, api] # @BRIEF Registers all API routers with the FastAPI application. -# @LAYER API +# @LAYER: API # #endregion api.include_routers # #region websocket_endpoint [C:5] [TYPE Function] # @BRIEF Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering. -# @PRE task_id must be a valid task ID. -# @POST WebSocket connection is managed and logs are streamed until disconnect. -# @SIDE_EFFECT Subscribes to TaskManager log queue and broadcasts messages over network. -# @DATA_CONTRACT [task_id: str, source: str, level: str] -> [JSON log entry objects] -# @INVARIANT Every accepted WebSocket subscription is unsubscribed exactly once even when streaming fails or the client disconnects. # @RELATION CALLS -> [TaskManagerPackage] # @RELATION DEPENDS_ON -> [LoggerModule] +# @PRE: task_id must be a valid task ID. +# @POST: WebSocket connection is managed and logs are streamed until disconnect. +# @SIDE_EFFECT: Subscribes to TaskManager log queue and broadcasts messages over network. +# @DATA_CONTRACT: [task_id: str, source: str, level: str] -> [JSON log entry objects] +# @INVARIANT: Every accepted WebSocket subscription is unsubscribed exactly once even when streaming fails or the client disconnects. # @UX_STATE: Connecting -> Streaming -> (Disconnected) # # @TEST_CONTRACT: WebSocketLogStreamApi -> @@ -454,10 +438,11 @@ if frontend_path.exists(): "/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static" ) - # #region serve_spa [C:1] [TYPE Function] - # @BRIEF Serves the SPA frontend for any path not matched by API routes. - # @PRE frontend_path exists. - # @POST Returns the requested file or index.html. + # [DEF:serve_spa:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Serves the SPA frontend for any path not matched by API routes. + # @PRE: frontend_path exists. + # @POST: Returns the requested file or index.html. @app.get("/{file_path:path}", include_in_schema=False) async def serve_spa(file_path: str): with belief_scope("serve_spa"): @@ -479,12 +464,13 @@ if frontend_path.exists(): return FileResponse(str(full_path)) return FileResponse(str(frontend_path / "index.html")) - # #endregion serve_spa + # [/DEF:serve_spa:Function] else: - # #region read_root [C:1] [TYPE Function] - # @BRIEF A simple root endpoint to confirm that the API is running when frontend is missing. - # @PRE None. - # @POST Returns a JSON message indicating API status. + # [DEF:read_root:Function] + # @COMPLEXITY: 1 + # @PURPOSE: A simple root endpoint to confirm that the API is running when frontend is missing. + # @PRE: None. + # @POST: Returns a JSON message indicating API status. @app.get("/") async def read_root(): with belief_scope("read_root"): @@ -492,5 +478,6 @@ else: "message": "Superset Tools API is running (Frontend build not found)" } - # #endregion read_root + # [/DEF:read_root:Function] # #endregion StaticFiles +# #endregion AppModule diff --git a/backend/src/core/__tests__/test_config_manager_compat.py b/backend/src/core/__tests__/test_config_manager_compat.py index 9dd908f6..86a3dcf9 100644 --- a/backend/src/core/__tests__/test_config_manager_compat.py +++ b/backend/src/core/__tests__/test_config_manager_compat.py @@ -1,7 +1,9 @@ -# #region TestConfigManagerCompat [C:3] [TYPE Module] [SEMANTICS config-manager, compatibility, payload, tests] -# @BRIEF Verifies ConfigManager compatibility wrappers preserve legacy payload sections. -# @LAYER Domain -# @RELATION VERIFIES -> [ConfigManager] +# [DEF:TestConfigManagerCompat:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: config-manager, compatibility, payload, tests +# @PURPOSE: Verifies ConfigManager compatibility wrappers preserve legacy payload sections. +# @LAYER: Domain +# @RELATION: VERIFIES -> ConfigManager from types import SimpleNamespace @@ -9,9 +11,9 @@ from src.core.config_manager import ConfigManager from src.core.config_models import AppConfig, Environment, GlobalSettings -# #region test_get_payload_preserves_legacy_sections [TYPE Function] -# @BRIEF Ensure get_payload merges typed config into raw payload without dropping legacy sections. -# @RELATION BINDS_TO -> [TestConfigManagerCompat] +# [DEF:test_get_payload_preserves_legacy_sections:Function] +# @RELATION: BINDS_TO -> TestConfigManagerCompat +# @PURPOSE: Ensure get_payload merges typed config into raw payload without dropping legacy sections. def test_get_payload_preserves_legacy_sections(): manager = ConfigManager.__new__(ConfigManager) manager.raw_payload = {"notifications": {"smtp": {"host": "mail.local"}}} @@ -23,7 +25,7 @@ def test_get_payload_preserves_legacy_sections(): assert payload["notifications"]["smtp"]["host"] == "mail.local" -# #endregion test_get_payload_preserves_legacy_sections +# [/DEF:test_get_payload_preserves_legacy_sections:Function] # [DEF:test_save_config_accepts_raw_payload_and_keeps_extras:Function] @@ -54,9 +56,13 @@ def test_save_config_accepts_raw_payload_and_keeps_extras(monkeypatch): assert persisted["payload"]["notifications"]["telegram"]["bot_token"] == "secret" -# #region test_save_config_syncs_environment_records_for_fk_backed_flows [TYPE Function] -# @BRIEF Ensure saving config mirrors typed environments into relational records required by FK-backed session persistence. -# @RELATION BINDS_TO -> [TestConfigManagerCompat] +# [/DEF:test_save_config_accepts_raw_payload_and_keeps_extras:Function] + + +# [DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function] +# @RELATION: BINDS_TO -> TestConfigManagerCompat +# @PURPOSE: Ensure saving config mirrors typed environments into relational records required by FK-backed session persistence. +# Deletion of stale records is tested separately in test_save_config_syncs_deletions_to_persistence. def test_save_config_syncs_environment_records_for_fk_backed_flows(): manager = ConfigManager.__new__(ConfigManager) manager.raw_payload = {} @@ -71,20 +77,22 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): credentials_id="legacy-user", ) - # #region _FakeQuery [C:1] [TYPE Class] - # @BRIEF Minimal query stub returning hardcoded existing environment record list for sync tests. - # @INVARIANT all() always returns [existing_record]; no parameterization or filtering. - # @RELATION BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # [DEF:_FakeQuery:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal query stub returning hardcoded existing environment record list for sync tests. + # @INVARIANT: all() always returns [existing_record]; no parameterization or filtering. class _FakeQuery: def all(self): return [existing_record] - # #endregion _FakeQuery + # [/DEF:_FakeQuery:Class] - # #region _FakeSession [C:1] [TYPE Class] - # @BRIEF Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions. - # @INVARIANT query() always returns _FakeQuery; no real DB interaction. - # @RELATION BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # [DEF:_FakeSession:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions. + # @INVARIANT: query() always returns _FakeQuery; no real DB interaction. class _FakeSession: def query(self, model): return _FakeQuery() @@ -95,7 +103,7 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): def delete(self, value): deleted_records.append(value) - # #endregion _FakeSession + # [/DEF:_FakeSession:Class] session = _FakeSession() config = AppConfig( @@ -118,13 +126,91 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows(): assert added_records[0].name == "DEV" assert added_records[0].url == "http://superset.local" assert added_records[0].credentials_id == "demo" - assert deleted_records == [existing_record] + # Sync does NOT delete; deletion is in _delete_stale_environment_records + assert len(deleted_records) == 0 -# #endregion test_save_config_syncs_environment_records_for_fk_backed_flows -# #region test_load_config_syncs_environment_records_from_existing_db_payload [TYPE Function] -# @BRIEF Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows. -# @RELATION BINDS_TO -> [TestConfigManagerCompat] +# [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function] + + +# [DEF:test_save_config_syncs_deletions_to_persistence:Function] +# @RELATION: BINDS_TO -> TestConfigManagerCompat +# @PURPOSE: Ensure stale environment records are deleted during save (in _delete_stale_environment_records) +# and NOT during _load_config → _sync_environment_records (which would violate FK constraints +# from task_records, database_mappings, etc.). +def test_save_config_syncs_deletions_to_persistence(): + manager = ConfigManager.__new__(ConfigManager) + manager.raw_payload = {} + manager.config = AppConfig(environments=[], settings=GlobalSettings()) + + added_records = [] + deleted_records = [] + existing_record = SimpleNamespace( + id="legacy-env", + name="Legacy", + url="http://legacy.local", + credentials_id="legacy-user", + ) + + # [DEF:_FakeQueryDel:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_deletions_to_persistence] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal query stub for deletion test. + class _FakeQuery: + def all(self): + return [existing_record] + + # [/DEF:_FakeQueryDel:Class] + + # [DEF:_FakeSessionDel:Class] + # @RELATION: BINDS_TO -> [test_save_config_syncs_deletions_to_persistence] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal session stub that captures add/delete for deletion assertions. + class _FakeSession: + def query(self, model): + return _FakeQuery() + + def add(self, value): + added_records.append(value) + + def delete(self, value): + deleted_records.append(value) + + # [/DEF:_FakeSessionDel:Class] + + session = _FakeSession() + config = AppConfig( + environments=[ + Environment( + id="dev", + name="DEV", + url="http://superset.local", + username="demo", + password="secret", + ) + ], + settings=GlobalSettings(), + ) + + # Step 1: sync creates/updates (no deletion) + manager._sync_environment_records(session, config) + assert len(added_records) == 1 + assert added_records[0].id == "dev" + assert len(deleted_records) == 0 + + # Step 2: stale deletion removes envs not in the configured set + manager._delete_stale_environment_records(session, config) + assert len(deleted_records) == 1 + assert deleted_records[0] == existing_record + assert deleted_records[0].id == "legacy-env" + + +# [/DEF:test_save_config_syncs_deletions_to_persistence:Function] + + +# [DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function] +# @RELATION: BINDS_TO -> TestConfigManagerCompat +# @PURPOSE: Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows. def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypatch): manager = ConfigManager.__new__(ConfigManager) manager.config_path = None @@ -135,10 +221,11 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa closed = {"value": False} committed = {"value": False} - # #region _FakeSession [C:1] [TYPE Class] - # @BRIEF Minimal session stub tracking commit/close signals for config load lifecycle assertions. - # @INVARIANT No query or add semantics; only lifecycle signal tracking. - # @RELATION BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload] + # [DEF:_FakeSession:Class] + # @RELATION: BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload] + # @COMPLEXITY: 1 + # @PURPOSE: Minimal session stub tracking commit/close signals for config load lifecycle assertions. + # @INVARIANT: No query or add semantics; only lifecycle signal tracking. class _FakeSession: def commit(self): committed["value"] = True @@ -146,7 +233,7 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa def close(self): closed["value"] = True - # #endregion _FakeSession + # [/DEF:_FakeSession:Class] fake_session = _FakeSession() fake_record = SimpleNamespace( @@ -183,5 +270,6 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa assert closed["value"] is True -# #endregion test_load_config_syncs_environment_records_from_existing_db_payload -# #endregion TestConfigManagerCompat +# [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function] + +# [/DEF:TestConfigManagerCompat:Module] diff --git a/backend/src/core/__tests__/test_native_filters.py b/backend/src/core/__tests__/test_native_filters.py index d22a1704..ca824f44 100644 --- a/backend/src/core/__tests__/test_native_filters.py +++ b/backend/src/core/__tests__/test_native_filters.py @@ -1,9 +1,11 @@ -# #region NativeFilterExtractionTests [C:3] [TYPE Module] [SEMANTICS tests, superset, native, filters, permalink, filter_state] -# @BRIEF Verify native filter extraction from permalinks and native_filters_key URLs. -# @LAYER Domain -# @RELATION BINDS_TO -> [SupersetClient] -# @RELATION BINDS_TO -> [AsyncSupersetClient] -# @RELATION BINDS_TO -> [FilterState, ParsedNativeFilters, ExtraFormDataMerge] +# [DEF:NativeFilterExtractionTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, superset, native, filters, permalink, filter_state +# @PURPOSE: Verify native filter extraction from permalinks and native_filters_key URLs. +# @LAYER: Domain +# @RELATION: [BINDS_TO] ->[SupersetClient] +# @RELATION: [BINDS_TO] ->[AsyncSupersetClient] +# @RELATION: [BINDS_TO] ->[FilterState, ParsedNativeFilters, ExtraFormDataMerge] import json from unittest.mock import MagicMock @@ -26,8 +28,8 @@ from src.models.filter_state import ( ) -# #region _make_environment [TYPE Function] -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:_make_environment:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests def _make_environment() -> Environment: return Environment( id="env-1", @@ -38,12 +40,12 @@ def _make_environment() -> Environment: ) -# #endregion _make_environment +# [/DEF:_make_environment:Function] -# #region test_extract_native_filters_from_permalink [TYPE Function] -# @BRIEF Extract native filters from a permalink key. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_native_filters_from_permalink:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Extract native filters from a permalink key. def test_extract_native_filters_from_permalink(): client = SupersetClient(_make_environment()) client.get_dashboard_permalink_state = MagicMock( @@ -87,12 +89,12 @@ def test_extract_native_filters_from_permalink(): assert result["anchor"] == "SECTION1" -# #endregion test_extract_native_filters_from_permalink +# [/DEF:test_extract_native_filters_from_permalink] -# #region test_extract_native_filters_from_permalink_direct_response [TYPE Function] -# @BRIEF Handle permalink response without result wrapper. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_native_filters_from_permalink_direct_response:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Handle permalink response without result wrapper. def test_extract_native_filters_from_permalink_direct_response(): client = SupersetClient(_make_environment()) client.get_dashboard_permalink_state = MagicMock( @@ -115,12 +117,12 @@ def test_extract_native_filters_from_permalink_direct_response(): assert "filter_1" in result["dataMask"] -# #endregion test_extract_native_filters_from_permalink_direct_response +# [/DEF:test_extract_native_filters_from_permalink_direct_response] -# #region test_extract_native_filters_from_key [TYPE Function] -# @BRIEF Extract native filters from a native_filters_key. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_native_filters_from_key:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Extract native filters from a native_filters_key. def test_extract_native_filters_from_key(): client = SupersetClient(_make_environment()) client.get_native_filter_state = MagicMock( @@ -154,12 +156,12 @@ def test_extract_native_filters_from_key(): ] == ["EMEA"] -# #endregion test_extract_native_filters_from_key +# [/DEF:test_extract_native_filters_from_key] -# #region test_extract_native_filters_from_key_single_filter [TYPE Function] -# @BRIEF Handle single filter format in native filter state. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_native_filters_from_key_single_filter:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Handle single filter format in native filter state. def test_extract_native_filters_from_key_single_filter(): client = SupersetClient(_make_environment()) client.get_native_filter_state = MagicMock( @@ -188,12 +190,12 @@ def test_extract_native_filters_from_key_single_filter(): ) -# #endregion test_extract_native_filters_from_key_single_filter +# [/DEF:test_extract_native_filters_from_key_single_filter] -# #region test_extract_native_filters_from_key_dict_value [TYPE Function] -# @BRIEF Handle filter state value as dict instead of JSON string. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_native_filters_from_key_dict_value:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Handle filter state value as dict instead of JSON string. def test_extract_native_filters_from_key_dict_value(): client = SupersetClient(_make_environment()) client.get_native_filter_state = MagicMock( @@ -215,12 +217,12 @@ def test_extract_native_filters_from_key_dict_value(): assert "filter_id" in result["dataMask"] -# #endregion test_extract_native_filters_from_key_dict_value +# [/DEF:test_extract_native_filters_from_key_dict_value] -# #region test_parse_dashboard_url_for_filters_permalink [TYPE Function] -# @BRIEF Parse permalink URL format. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_permalink:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Parse permalink URL format. def test_parse_dashboard_url_for_filters_permalink(): client = SupersetClient(_make_environment()) client.extract_native_filters_from_permalink = MagicMock( @@ -235,12 +237,12 @@ def test_parse_dashboard_url_for_filters_permalink(): assert result["filters"]["dataMask"]["f1"] == {} -# #endregion test_parse_dashboard_url_for_filters_permalink +# [/DEF:test_parse_dashboard_url_for_filters_permalink] -# #region test_parse_dashboard_url_for_filters_native_key [TYPE Function] -# @BRIEF Parse native_filters_key URL format with numeric dashboard ID. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_native_key:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Parse native_filters_key URL format with numeric dashboard ID. def test_parse_dashboard_url_for_filters_native_key(): client = SupersetClient(_make_environment()) client.extract_native_filters_from_key = MagicMock( @@ -260,12 +262,12 @@ def test_parse_dashboard_url_for_filters_native_key(): assert result["filters"]["dataMask"]["f2"] == {} -# #endregion test_parse_dashboard_url_for_filters_native_key +# [/DEF:test_parse_dashboard_url_for_filters_native_key] -# #region test_parse_dashboard_url_for_filters_native_key_slug [TYPE Function] -# @BRIEF Parse native_filters_key URL format when dashboard reference is a slug, not a numeric ID. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_native_key_slug:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Parse native_filters_key URL format when dashboard reference is a slug, not a numeric ID. def test_parse_dashboard_url_for_filters_native_key_slug(): client = SupersetClient(_make_environment()) # Simulate slug resolution: get_dashboard returns the dashboard with numeric ID @@ -293,12 +295,12 @@ def test_parse_dashboard_url_for_filters_native_key_slug(): client.extract_native_filters_from_key.assert_called_once_with(99, "abc123") -# #endregion test_parse_dashboard_url_for_filters_native_key_slug +# [/DEF:test_parse_dashboard_url_for_filters_native_key_slug] -# #region test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails [TYPE Function] -# @BRIEF Gracefully handle slug resolution failure for native_filters_key URL. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Gracefully handle slug resolution failure for native_filters_key URL. def test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails(): client = SupersetClient(_make_environment()) client.get_dashboard = MagicMock(side_effect=Exception("Not found")) @@ -311,12 +313,12 @@ def test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails(): assert result["dashboard_id"] is None -# #endregion test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails +# [/DEF:test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails] -# #region test_parse_dashboard_url_for_filters_native_filters_direct [TYPE Function] -# @BRIEF Parse native_filters direct query param. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_native_filters_direct:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Parse native_filters direct query param. def test_parse_dashboard_url_for_filters_native_filters_direct(): client = SupersetClient(_make_environment()) @@ -329,12 +331,12 @@ def test_parse_dashboard_url_for_filters_native_filters_direct(): assert "dataMask" in result["filters"] -# #endregion test_parse_dashboard_url_for_filters_native_filters_direct +# [/DEF:test_parse_dashboard_url_for_filters_native_filters_direct] -# #region test_parse_dashboard_url_for_filters_no_filters [TYPE Function] -# @BRIEF Return empty result when no filters present. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parse_dashboard_url_for_filters_no_filters:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Return empty result when no filters present. def test_parse_dashboard_url_for_filters_no_filters(): client = SupersetClient(_make_environment()) @@ -346,12 +348,12 @@ def test_parse_dashboard_url_for_filters_no_filters(): assert result["filters"] == {} -# #endregion test_parse_dashboard_url_for_filters_no_filters +# [/DEF:test_parse_dashboard_url_for_filters_no_filters] -# #region test_extra_form_data_merge [TYPE Function] -# @BRIEF Test ExtraFormDataMerge correctly merges dictionaries. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extra_form_data_merge:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Test ExtraFormDataMerge correctly merges dictionaries. def test_extra_form_data_merge(): merger = ExtraFormDataMerge() @@ -384,12 +386,12 @@ def test_extra_form_data_merge(): assert result["columns"] == ["col1", "col2"] -# #endregion test_extra_form_data_merge +# [/DEF:test_extra_form_data_merge] -# #region test_filter_state_model [TYPE Function] -# @BRIEF Test FilterState Pydantic model. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_filter_state_model:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Test FilterState Pydantic model. def test_filter_state_model(): state = FilterState( extraFormData={"filters": [{"col": "x", "op": "==", "val": "y"}]}, @@ -402,12 +404,12 @@ def test_filter_state_model(): assert state.ownState["selectedValues"] == ["y"] -# #endregion test_filter_state_model +# [/DEF:test_filter_state_model] -# #region test_parsed_native_filters_model [TYPE Function] -# @BRIEF Test ParsedNativeFilters Pydantic model. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parsed_native_filters_model:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Test ParsedNativeFilters Pydantic model. def test_parsed_native_filters_model(): filters = ParsedNativeFilters( dataMask={"f1": {"extraFormData": {}, "filterState": {}}}, @@ -421,12 +423,12 @@ def test_parsed_native_filters_model(): assert filters.filter_type == "permalink" -# #endregion test_parsed_native_filters_model +# [/DEF:test_parsed_native_filters_model] -# #region test_parsed_native_filters_empty [TYPE Function] -# @BRIEF Test ParsedNativeFilters with no filters. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_parsed_native_filters_empty:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Test ParsedNativeFilters with no filters. def test_parsed_native_filters_empty(): filters = ParsedNativeFilters() @@ -434,12 +436,12 @@ def test_parsed_native_filters_empty(): assert filters.get_filter_count() == 0 -# #endregion test_parsed_native_filters_empty +# [/DEF:test_parsed_native_filters_empty] -# #region test_native_filter_data_mask_model [TYPE Function] -# @BRIEF Test NativeFilterDataMask model. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_native_filter_data_mask_model:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Test NativeFilterDataMask model. def test_native_filter_data_mask_model(): data_mask = NativeFilterDataMask( filters={ @@ -455,12 +457,12 @@ def test_native_filter_data_mask_model(): assert data_mask.get_extra_form_data("nonexistent") == {} -# #endregion test_native_filter_data_mask_model +# [/DEF:test_native_filter_data_mask_model] -# #region test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names [TYPE Function] -# @BRIEF Reconcile raw native filter ids from state to canonical metadata filter names. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Reconcile raw native filter ids from state to canonical metadata filter names. def test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names(): client = MagicMock() client.get_dashboard.return_value = { @@ -522,12 +524,12 @@ def test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_n } -# #endregion test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names +# [/DEF:test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names:Function] -# #region test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter [TYPE Function] -# @BRIEF Collapse raw-id state entries and metadata entries into one canonical filter. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Collapse raw-id state entries and metadata entries into one canonical filter. def test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter(): client = MagicMock() client.get_dashboard.return_value = { @@ -580,12 +582,12 @@ def test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_o assert region_filter["recovery_status"] == "partial" -# #endregion test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter +# [/DEF:test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter:Function] -# #region test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids [TYPE Function] -# @BRIEF Preserve unmatched raw native filter ids as fallback diagnostics when metadata mapping is unavailable. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Preserve unmatched raw native filter ids as fallback diagnostics when metadata mapping is unavailable. def test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids(): client = MagicMock() client.get_dashboard.return_value = { @@ -637,12 +639,12 @@ def test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids(): ) -# #endregion test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids +# [/DEF:test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids:Function] -# #region test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview [TYPE Function] -# @BRIEF Recovered native filter state should preserve exact Superset clause payload and time extras for preview compilation. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Recovered native filter state should preserve exact Superset clause payload and time extras for preview compilation. def test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview(): extractor = SupersetContextExtractor(_make_environment(), client=MagicMock()) @@ -682,12 +684,12 @@ def test_extract_imported_filters_preserves_clause_level_native_filter_payload_f ] -# #endregion test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview +# [/DEF:test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview:Function] -# #region test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values [TYPE Function] -# @BRIEF Assistant-facing imported-filter payload should redact likely PII while preserving structure metadata. -# @RELATION BINDS_TO -> [NativeFilterExtractionTests] +# [DEF:test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values:Function] +# @RELATION: BINDS_TO -> NativeFilterExtractionTests +# @PURPOSE: Assistant-facing imported-filter payload should redact likely PII while preserving structure metadata. def test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values(): result = sanitize_imported_filter_for_assistant( { @@ -703,7 +705,7 @@ def test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values(): assert result["normalized_value"] == {"filter_clauses": []} -# #endregion test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values +# [/DEF:test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values:Function] -# #endregion NativeFilterExtractionTests +# [/DEF:NativeFilterExtractionTests:Module] diff --git a/backend/src/core/__tests__/test_superset_preview_pipeline.py b/backend/src/core/__tests__/test_superset_preview_pipeline.py index a3959add..96a33417 100644 --- a/backend/src/core/__tests__/test_superset_preview_pipeline.py +++ b/backend/src/core/__tests__/test_superset_preview_pipeline.py @@ -1,7 +1,9 @@ -# #region SupersetPreviewPipelineTests [C:3] [TYPE Module] [SEMANTICS tests, superset, preview, chart_data, network, 404-mapping] -# @BRIEF Verify explicit chart-data preview compilation and ensure non-dashboard 404 errors remain generic across sync and async clients. -# @LAYER Domain -# @RELATION BINDS_TO -> [AsyncNetworkModule] +# [DEF:SupersetPreviewPipelineTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, superset, preview, chart_data, network, 404-mapping +# @PURPOSE: Verify explicit chart-data preview compilation and ensure non-dashboard 404 errors remain generic across sync and async clients. +# @LAYER: Domain +# @RELATION: [BINDS_TO] ->[AsyncNetworkModule] import json from unittest.mock import MagicMock @@ -16,8 +18,8 @@ from src.core.utils.async_network import AsyncAPIClient from src.core.utils.network import APIClient, DashboardNotFoundError, SupersetAPIError -# #region _make_environment [TYPE Function] -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:_make_environment:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests def _make_environment() -> Environment: return Environment( id="env-1", @@ -28,11 +30,11 @@ def _make_environment() -> Environment: ) -# #endregion _make_environment +# [/DEF:_make_environment:Function] -# #region _make_requests_http_error [TYPE Function] -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:_make_requests_http_error:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests def _make_requests_http_error( status_code: int, url: str ) -> requests.exceptions.HTTPError: @@ -45,11 +47,11 @@ def _make_requests_http_error( return requests.exceptions.HTTPError(response=response, request=request) -# #endregion _make_requests_http_error +# [/DEF:_make_requests_http_error:Function] -# #region _make_httpx_status_error [TYPE Function] -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:_make_httpx_status_error:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests def _make_httpx_status_error(status_code: int, url: str) -> httpx.HTTPStatusError: request = httpx.Request("GET", url) response = httpx.Response( @@ -58,12 +60,12 @@ def _make_httpx_status_error(status_code: int, url: str) -> httpx.HTTPStatusErro return httpx.HTTPStatusError("upstream error", request=request, response=response) -# #endregion _make_httpx_status_error +# [/DEF:_make_httpx_status_error:Function] -# #region test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy [TYPE Function] -# @BRIEF Superset preview compilation should prefer the legacy form_data transport inferred from browser traffic before falling back to chart-data. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Superset preview compilation should prefer the legacy form_data transport inferred from browser traffic before falling back to chart-data. def test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy(): client = SupersetClient(_make_environment()) client.get_dataset = MagicMock( @@ -144,12 +146,12 @@ def test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy(): ] -# #endregion test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy +# [/DEF:test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy:Function] -# #region test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures [TYPE Function] -# @BRIEF Superset preview compilation should fall back to chart-data when legacy form_data strategies are rejected. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Superset preview compilation should fall back to chart-data when legacy form_data strategies are rejected. def test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures(): client = SupersetClient(_make_environment()) client.get_dataset = MagicMock( @@ -241,12 +243,12 @@ def test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures( } -# #endregion test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures +# [/DEF:test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures:Function] -# #region test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data [TYPE Function] -# @BRIEF Preview query context should mirror chart-style filter transport so recovered native filters reach Superset compilation. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Preview query context should mirror chart-style filter transport so recovered native filters reach Superset compilation. def test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data(): client = SupersetClient(_make_environment()) @@ -304,12 +306,12 @@ def test_build_dataset_preview_query_context_places_recovered_filters_in_chart_s assert query_context["form_data"]["url_params"] == {"country": "DE"} -# #endregion test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data +# [/DEF:test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data:Function] -# #region test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values [TYPE Function] -# @BRIEF Preview query context should merge dataset template params for parity with real dataset definitions while preserving explicit session overrides. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Preview query context should merge dataset template params for parity with real dataset definitions while preserving explicit session overrides. def test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values(): client = SupersetClient(_make_environment()) @@ -335,12 +337,12 @@ def test_build_dataset_preview_query_context_merges_dataset_template_params_and_ } -# #endregion test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values +# [/DEF:test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values:Function] -# #region test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload [TYPE Function] -# @BRIEF Preview query context should preserve time-range native filter extras even when dataset defaults differ. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Preview query context should preserve time-range native filter extras even when dataset defaults differ. def test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload(): client = SupersetClient(_make_environment()) @@ -374,12 +376,12 @@ def test_build_dataset_preview_query_context_preserves_time_range_from_native_fi assert query_context["queries"][0]["filters"] == [] -# #endregion test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload +# [/DEF:test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload:Function] -# #region test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses [TYPE Function] -# @BRIEF Legacy preview form_data should preserve recovered native filter clauses in browser-style fields without duplicating datasource for QueryObjectFactory. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Legacy preview form_data should preserve recovered native filter clauses in browser-style fields without duplicating datasource for QueryObjectFactory. def test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses(): client = SupersetClient(_make_environment()) @@ -428,12 +430,12 @@ def test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses( assert legacy_form_data["result_type"] == "query" -# #endregion test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses +# [/DEF:test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses:Function] -# #region test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function] -# @BRIEF Sync network client should reserve dashboard-not-found translation for dashboard endpoints only. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Sync network client should reserve dashboard-not-found translation for dashboard endpoints only. def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic(): client = APIClient( config={ @@ -452,12 +454,12 @@ def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic(): assert "API resource not found at endpoint '/chart/data'" in str(exc_info.value) -# #endregion test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic +# [/DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function] -# #region test_sync_network_404_mapping_translates_dashboard_endpoints [TYPE Function] -# @BRIEF Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors. def test_sync_network_404_mapping_translates_dashboard_endpoints(): client = APIClient( config={ @@ -475,12 +477,12 @@ def test_sync_network_404_mapping_translates_dashboard_endpoints(): assert "Dashboard '/dashboard/10' Dashboard not found" in str(exc_info.value) -# #endregion test_sync_network_404_mapping_translates_dashboard_endpoints +# [/DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function] -# #region test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function] -# @BRIEF Async network client should reserve dashboard-not-found translation for dashboard endpoints only. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Async network client should reserve dashboard-not-found translation for dashboard endpoints only. @pytest.mark.asyncio async def test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic(): client = AsyncAPIClient( @@ -505,12 +507,12 @@ async def test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic() await client.aclose() -# #endregion test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic +# [/DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function] -# #region test_async_network_404_mapping_translates_dashboard_endpoints [TYPE Function] -# @BRIEF Async network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors. -# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests] +# [DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function] +# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests +# @PURPOSE: Async network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors. @pytest.mark.asyncio async def test_async_network_404_mapping_translates_dashboard_endpoints(): client = AsyncAPIClient( @@ -534,7 +536,7 @@ async def test_async_network_404_mapping_translates_dashboard_endpoints(): await client.aclose() -# #endregion test_async_network_404_mapping_translates_dashboard_endpoints +# [/DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function] -# #endregion SupersetPreviewPipelineTests +# [/DEF:SupersetPreviewPipelineTests:Module] diff --git a/backend/src/core/__tests__/test_superset_profile_lookup.py b/backend/src/core/__tests__/test_superset_profile_lookup.py index ae0185d8..8690f5e5 100644 --- a/backend/src/core/__tests__/test_superset_profile_lookup.py +++ b/backend/src/core/__tests__/test_superset_profile_lookup.py @@ -1,7 +1,9 @@ -# #region TestSupersetProfileLookup [C:3] [TYPE Module] [SEMANTICS tests, superset, profile, lookup, fallback, sorting] -# @BRIEF Verifies Superset profile lookup adapter payload normalization and fallback error precedence. -# @LAYER Domain -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestSupersetProfileLookup:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, superset, profile, lookup, fallback, sorting +# @PURPOSE: Verifies Superset profile lookup adapter payload normalization and fallback error precedence. +# @LAYER: Domain # [SECTION: IMPORTS] import json @@ -20,25 +22,26 @@ from src.core.utils.network import AuthenticationError, SupersetAPIError # [/SECTION] -# #region _RecordingNetworkClient [C:2] [TYPE Class] -# @BRIEF Records request payloads and returns scripted responses for deterministic adapter tests. -# @INVARIANT Each request consumes one scripted response in call order and persists call metadata. -# @RELATION BINDS_TO -> [TestSupersetProfileLookup] +# [DEF:_RecordingNetworkClient:Class] +# @RELATION: BINDS_TO -> TestSupersetProfileLookup +# @COMPLEXITY: 2 +# @PURPOSE: Records request payloads and returns scripted responses for deterministic adapter tests. +# @INVARIANT: Each request consumes one scripted response in call order and persists call metadata. class _RecordingNetworkClient: - # #region __init__ [TYPE Function] - # @BRIEF Initializes scripted network responses. - # @PRE scripted_responses is ordered per expected request sequence. - # @POST Instance stores response script and captures subsequent request calls. + # [DEF:__init__:Function] + # @PURPOSE: Initializes scripted network responses. + # @PRE: scripted_responses is ordered per expected request sequence. + # @POST: Instance stores response script and captures subsequent request calls. def __init__(self, scripted_responses: List[Any]): self._scripted_responses = scripted_responses self.calls: List[Dict[str, Any]] = [] - # #endregion __init__ + # [/DEF:__init__:Function] - # #region request [TYPE Function] - # @BRIEF Mimics APIClient.request while capturing call arguments. - # @PRE method and endpoint are provided. - # @POST Returns scripted response or raises scripted exception. + # [DEF:request:Function] + # @PURPOSE: Mimics APIClient.request while capturing call arguments. + # @PRE: method and endpoint are provided. + # @POST: Returns scripted response or raises scripted exception. def request( self, method: str, @@ -59,17 +62,17 @@ class _RecordingNetworkClient: raise response return response - # #endregion request + # [/DEF:request:Function] -# #endregion _RecordingNetworkClient +# [/DEF:_RecordingNetworkClient:Class] -# #region test_get_users_page_sends_lowercase_order_direction [TYPE Function] -# @BRIEF Ensures adapter sends lowercase order_direction compatible with Superset rison schema. -# @PRE Adapter is initialized with recording network client. -# @POST First request query payload contains order_direction='asc' for asc sort. -# @RELATION BINDS_TO -> [TestSupersetProfileLookup] +# [DEF:test_get_users_page_sends_lowercase_order_direction:Function] +# @RELATION: BINDS_TO -> TestSupersetProfileLookup +# @PURPOSE: Ensures adapter sends lowercase order_direction compatible with Superset rison schema. +# @PRE: Adapter is initialized with recording network client. +# @POST: First request query payload contains order_direction='asc' for asc sort. def test_get_users_page_sends_lowercase_order_direction(): client = _RecordingNetworkClient( scripted_responses=[{"result": [{"username": "admin"}], "count": 1}] @@ -90,14 +93,14 @@ def test_get_users_page_sends_lowercase_order_direction(): assert sent_query["order_direction"] == "asc" -# #endregion test_get_users_page_sends_lowercase_order_direction +# [/DEF:test_get_users_page_sends_lowercase_order_direction:Function] -# #region test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error [TYPE Function] -# @BRIEF Ensures fallback auth error does not mask primary schema/query failure. -# @PRE Primary endpoint fails with SupersetAPIError and fallback fails with AuthenticationError. -# @POST Raised exception remains primary SupersetAPIError (non-auth) to preserve root cause. -# @RELATION BINDS_TO -> [TestSupersetProfileLookup] +# [DEF:test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error:Function] +# @RELATION: BINDS_TO -> TestSupersetProfileLookup +# @PURPOSE: Ensures fallback auth error does not mask primary schema/query failure. +# @PRE: Primary endpoint fails with SupersetAPIError and fallback fails with AuthenticationError. +# @POST: Raised exception remains primary SupersetAPIError (non-auth) to preserve root cause. def test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error(): client = _RecordingNetworkClient( scripted_responses=[ @@ -116,14 +119,14 @@ def test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error( assert not isinstance(exc_info.value, AuthenticationError) -# #endregion test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error +# [/DEF:test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error:Function] -# #region test_get_users_page_uses_fallback_endpoint_when_primary_fails [TYPE Function] -# @BRIEF Verifies adapter retries second users endpoint and succeeds when fallback is healthy. -# @PRE Primary endpoint fails; fallback returns valid users payload. -# @POST Result status is success and both endpoints were attempted in order. -# @RELATION BINDS_TO -> [TestSupersetProfileLookup] +# [DEF:test_get_users_page_uses_fallback_endpoint_when_primary_fails:Function] +# @RELATION: BINDS_TO -> TestSupersetProfileLookup +# @PURPOSE: Verifies adapter retries second users endpoint and succeeds when fallback is healthy. +# @PRE: Primary endpoint fails; fallback returns valid users payload. +# @POST: Result status is success and both endpoints were attempted in order. def test_get_users_page_uses_fallback_endpoint_when_primary_fails(): client = _RecordingNetworkClient( scripted_responses=[ @@ -144,7 +147,7 @@ def test_get_users_page_uses_fallback_endpoint_when_primary_fails(): ] -# #endregion test_get_users_page_uses_fallback_endpoint_when_primary_fails +# [/DEF:test_get_users_page_uses_fallback_endpoint_when_primary_fails:Function] -# #endregion TestSupersetProfileLookup +# [/DEF:TestSupersetProfileLookup:Module] diff --git a/backend/src/core/__tests__/test_throttled_scheduler.py b/backend/src/core/__tests__/test_throttled_scheduler.py index cd1fd799..ef47ddfb 100644 --- a/backend/src/core/__tests__/test_throttled_scheduler.py +++ b/backend/src/core/__tests__/test_throttled_scheduler.py @@ -2,14 +2,15 @@ import pytest from datetime import time, date, datetime, timedelta from src.core.scheduler import ThrottledSchedulerConfigurator -# #region test_throttled_scheduler [C:3] [TYPE Module] -# @BRIEF Unit tests for ThrottledSchedulerConfigurator distribution logic. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:test_throttled_scheduler:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic. -# #region test_calculate_schedule_even_distribution [TYPE Function] -# @BRIEF Validate even spacing across a two-hour scheduling window for three tasks. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_even_distribution:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate even spacing across a two-hour scheduling window for three tasks. def test_calculate_schedule_even_distribution(): """ @TEST_SCENARIO: 3 tasks in a 2-hour window should be spaced 1 hour apart. @@ -29,12 +30,12 @@ def test_calculate_schedule_even_distribution(): assert schedule[2] == datetime(2024, 1, 1, 3, 0) -# #endregion test_calculate_schedule_even_distribution +# [/DEF:test_calculate_schedule_even_distribution:Function] -# #region test_calculate_schedule_midnight_crossing [TYPE Function] -# @BRIEF Validate scheduler correctly rolls timestamps into the next day across midnight. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_midnight_crossing:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate scheduler correctly rolls timestamps into the next day across midnight. def test_calculate_schedule_midnight_crossing(): """ @TEST_SCENARIO: Window from 23:00 to 01:00 (next day). @@ -54,12 +55,12 @@ def test_calculate_schedule_midnight_crossing(): assert schedule[2] == datetime(2024, 1, 2, 1, 0) -# #endregion test_calculate_schedule_midnight_crossing +# [/DEF:test_calculate_schedule_midnight_crossing:Function] -# #region test_calculate_schedule_single_task [TYPE Function] -# @BRIEF Validate single-task schedule returns only the window start timestamp. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_single_task:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate single-task schedule returns only the window start timestamp. def test_calculate_schedule_single_task(): """ @TEST_SCENARIO: Single task should be scheduled at start time. @@ -77,12 +78,12 @@ def test_calculate_schedule_single_task(): assert schedule[0] == datetime(2024, 1, 1, 1, 0) -# #endregion test_calculate_schedule_single_task +# [/DEF:test_calculate_schedule_single_task:Function] -# #region test_calculate_schedule_empty_list [TYPE Function] -# @BRIEF Validate empty dashboard list produces an empty schedule. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_empty_list:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate empty dashboard list produces an empty schedule. def test_calculate_schedule_empty_list(): """ @TEST_SCENARIO: Empty dashboard list returns empty schedule. @@ -99,12 +100,12 @@ def test_calculate_schedule_empty_list(): assert schedule == [] -# #endregion test_calculate_schedule_empty_list +# [/DEF:test_calculate_schedule_empty_list:Function] -# #region test_calculate_schedule_zero_window [TYPE Function] -# @BRIEF Validate zero-length window schedules all tasks at identical start timestamp. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_zero_window:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate zero-length window schedules all tasks at identical start timestamp. def test_calculate_schedule_zero_window(): """ @TEST_SCENARIO: Window start == end. All tasks at start time. @@ -123,12 +124,12 @@ def test_calculate_schedule_zero_window(): assert schedule[1] == datetime(2024, 1, 1, 1, 0) -# #endregion test_calculate_schedule_zero_window +# [/DEF:test_calculate_schedule_zero_window:Function] -# #region test_calculate_schedule_very_small_window [TYPE Function] -# @BRIEF Validate sub-second interpolation when task count exceeds near-zero window granularity. -# @RELATION BINDS_TO -> [test_throttled_scheduler] +# [DEF:test_calculate_schedule_very_small_window:Function] +# @RELATION: BINDS_TO -> test_throttled_scheduler +# @PURPOSE: Validate sub-second interpolation when task count exceeds near-zero window granularity. def test_calculate_schedule_very_small_window(): """ @TEST_SCENARIO: Window smaller than number of tasks (in seconds). @@ -148,5 +149,5 @@ def test_calculate_schedule_very_small_window(): assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1) -# #endregion test_calculate_schedule_very_small_window -# #endregion test_throttled_scheduler +# [/DEF:test_calculate_schedule_very_small_window:Function] +# [/DEF:test_throttled_scheduler:Module] diff --git a/backend/src/core/async_superset_client.py b/backend/src/core/async_superset_client.py index b0da74aa..6ec6b286 100644 --- a/backend/src/core/async_superset_client.py +++ b/backend/src/core/async_superset_client.py @@ -1,9 +1,8 @@ # #region AsyncSupersetClientModule [C:3] [TYPE Module] [SEMANTICS superset, async, client, httpx, dashboards, datasets] # @BRIEF Parse a Superset dashboard URL and extract native filter state asynchronously. -# @LAYER Core +# @LAYER: Core -# [SECTION: IMPORTS] import asyncio import json import re @@ -13,7 +12,6 @@ from .config_models import Environment from .logger import logger as app_logger, belief_scope from .superset_client import SupersetClient from .utils.async_network import AsyncAPIClient -# [/SECTION] # #region AsyncSupersetClient [C:3] [TYPE Class] @@ -22,12 +20,13 @@ from .utils.async_network import AsyncAPIClient # @RELATION DEPENDS_ON -> [AsyncAPIClient] # @RELATION CALLS -> [AsyncAPIClient.request] class AsyncSupersetClient(SupersetClient): - # #region AsyncSupersetClientInit [C:3] [TYPE Function] - # @BRIEF Initialize async Superset client with AsyncAPIClient transport. - # @PRE env is valid Environment instance. - # @POST Client uses async network transport and inherited projection helpers. - # @DATA_CONTRACT Input[Environment] -> self.network[AsyncAPIClient] - # @RELATION DEPENDS_ON -> [AsyncAPIClient] + # [DEF:AsyncSupersetClientInit:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Initialize async Superset client with AsyncAPIClient transport. + # @PRE: env is valid Environment instance. + # @POST: Client uses async network transport and inherited projection helpers. + # @DATA_CONTRACT: Input[Environment] -> self.network[AsyncAPIClient] + # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient] def __init__(self, env: Environment): self.env = env auth_payload = { @@ -43,23 +42,25 @@ class AsyncSupersetClient(SupersetClient): ) self.delete_before_reimport = False - # #endregion AsyncSupersetClientInit + # [/DEF:AsyncSupersetClientInit:Function] - # #region AsyncSupersetClientClose [C:3] [TYPE Function] - # @BRIEF Close async transport resources. - # @POST Underlying AsyncAPIClient is closed. - # @SIDE_EFFECT Closes network sockets. - # @RELATION CALLS -> [AsyncAPIClient.aclose] + # [DEF:AsyncSupersetClientClose:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Close async transport resources. + # @POST: Underlying AsyncAPIClient is closed. + # @SIDE_EFFECT: Closes network sockets. + # @RELATION: [CALLS] ->[AsyncAPIClient.aclose] async def aclose(self) -> None: await self.network.aclose() - # #endregion AsyncSupersetClientClose + # [/DEF:AsyncSupersetClientClose:Function] - # #region get_dashboards_page_async [C:3] [TYPE Function] - # @BRIEF Fetch one dashboards page asynchronously. - # @POST Returns total count and page result list. - # @DATA_CONTRACT Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] - # @RELATION CALLS -> [AsyncAPIClient.request] + # [DEF:get_dashboards_page_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch one dashboards page asynchronously. + # @POST: Returns total count and page result list. + # @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]] + # @RELATION: [CALLS] -> [AsyncAPIClient.request] async def get_dashboards_page_async( self, query: Optional[Dict] = None ) -> Tuple[int, List[Dict]]: @@ -91,13 +92,14 @@ class AsyncSupersetClient(SupersetClient): total_count = response_json.get("count", len(result)) return total_count, result - # #endregion get_dashboards_page_async + # [/DEF:get_dashboards_page_async:Function] - # #region get_dashboard_async [C:3] [TYPE Function] - # @BRIEF Fetch one dashboard payload asynchronously. - # @POST Returns raw dashboard payload from Superset API. - # @DATA_CONTRACT Input[dashboard_id: int] -> Output[Dict] - # @RELATION CALLS -> [AsyncAPIClient.request] + # [DEF:get_dashboard_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch one dashboard payload asynchronously. + # @POST: Returns raw dashboard payload from Superset API. + # @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict] + # @RELATION: [CALLS] ->[AsyncAPIClient.request] async def get_dashboard_async(self, dashboard_id: int) -> Dict: with belief_scope( "AsyncSupersetClient.get_dashboard_async", f"id={dashboard_id}" @@ -107,13 +109,14 @@ class AsyncSupersetClient(SupersetClient): ) return cast(Dict, response) - # #endregion get_dashboard_async + # [/DEF:get_dashboard_async:Function] - # #region get_chart_async [C:3] [TYPE Function] - # @BRIEF Fetch one chart payload asynchronously. - # @POST Returns raw chart payload from Superset API. - # @DATA_CONTRACT Input[chart_id: int] -> Output[Dict] - # @RELATION CALLS -> [AsyncAPIClient.request] + # [DEF:get_chart_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch one chart payload asynchronously. + # @POST: Returns raw chart payload from Superset API. + # @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict] + # @RELATION: [CALLS] ->[AsyncAPIClient.request] async def get_chart_async(self, chart_id: int) -> Dict: with belief_scope("AsyncSupersetClient.get_chart_async", f"id={chart_id}"): response = await self.network.request( @@ -121,14 +124,15 @@ class AsyncSupersetClient(SupersetClient): ) return cast(Dict, response) - # #endregion get_chart_async + # [/DEF:get_chart_async:Function] - # #region get_dashboard_detail_async [C:3] [TYPE Function] - # @BRIEF Fetch dashboard detail asynchronously with concurrent charts/datasets requests. - # @POST Returns dashboard detail payload for overview page. - # @DATA_CONTRACT Input[dashboard_id: int] -> Output[Dict] - # @RELATION CALLS -> [get_dashboard_async] - # @RELATION CALLS -> [get_chart_async] + # [DEF:get_dashboard_detail_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests. + # @POST: Returns dashboard detail payload for overview page. + # @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict] + # @RELATION: [CALLS] ->[get_dashboard_async] + # @RELATION: [CALLS] ->[get_chart_async] async def get_dashboard_detail_async(self, dashboard_id: int) -> Dict: with belief_scope( "AsyncSupersetClient.get_dashboard_detail_async", f"id={dashboard_id}" @@ -418,12 +422,13 @@ class AsyncSupersetClient(SupersetClient): "dataset_count": len(datasets), } - # #endregion get_dashboard_detail_async + # [/DEF:get_dashboard_detail_async:Function] - # #region get_dashboard_permalink_state_async [C:2] [TYPE Function] - # @BRIEF Fetch stored dashboard permalink state asynchronously. - # @POST Returns dashboard permalink state payload from Superset API. - # @DATA_CONTRACT Input[permalink_key: str] -> Output[Dict] + # [DEF:get_dashboard_permalink_state_async:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Fetch stored dashboard permalink state asynchronously. + # @POST: Returns dashboard permalink state payload from Superset API. + # @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict] async def get_dashboard_permalink_state_async(self, permalink_key: str) -> Dict: with belief_scope( "AsyncSupersetClient.get_dashboard_permalink_state_async", @@ -434,12 +439,13 @@ class AsyncSupersetClient(SupersetClient): ) return cast(Dict, response) - # #endregion get_dashboard_permalink_state_async + # [/DEF:get_dashboard_permalink_state_async:Function] - # #region get_native_filter_state_async [C:2] [TYPE Function] - # @BRIEF Fetch stored native filter state asynchronously. - # @POST Returns native filter state payload from Superset API. - # @DATA_CONTRACT Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] + # [DEF:get_native_filter_state_async:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Fetch stored native filter state asynchronously. + # @POST: Returns native filter state payload from Superset API. + # @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] async def get_native_filter_state_async( self, dashboard_id: int, filter_state_key: str ) -> Dict: @@ -453,13 +459,14 @@ class AsyncSupersetClient(SupersetClient): ) return cast(Dict, response) - # #endregion get_native_filter_state_async + # [/DEF:get_native_filter_state_async:Function] - # #region extract_native_filters_from_permalink_async [C:3] [TYPE Function] - # @BRIEF Extract native filters dataMask from a permalink key asynchronously. - # @POST Returns extracted dataMask with filter states. - # @DATA_CONTRACT Input[permalink_key: str] -> Output[Dict] - # @RELATION CALLS -> [get_dashboard_permalink_state_async] + # [DEF:extract_native_filters_from_permalink_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract native filters dataMask from a permalink key asynchronously. + # @POST: Returns extracted dataMask with filter states. + # @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict] + # @RELATION: [CALLS] ->[get_dashboard_permalink_state_async] async def extract_native_filters_from_permalink_async( self, permalink_key: str ) -> Dict: @@ -493,13 +500,14 @@ class AsyncSupersetClient(SupersetClient): "permalink_key": permalink_key, } - # #endregion extract_native_filters_from_permalink_async + # [/DEF:extract_native_filters_from_permalink_async:Function] - # #region extract_native_filters_from_key_async [C:3] [TYPE Function] - # @BRIEF Extract native filters from a native_filters_key URL parameter asynchronously. - # @POST Returns extracted filter state with extraFormData. - # @DATA_CONTRACT Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] - # @RELATION CALLS -> [get_native_filter_state_async] + # [DEF:extract_native_filters_from_key_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract native filters from a native_filters_key URL parameter asynchronously. + # @POST: Returns extracted filter state with extraFormData. + # @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict] + # @RELATION: [CALLS] ->[get_native_filter_state_async] async def extract_native_filters_from_key_async( self, dashboard_id: int, filter_state_key: str ) -> Dict: @@ -553,14 +561,15 @@ class AsyncSupersetClient(SupersetClient): "filter_state_key": filter_state_key, } - # #endregion extract_native_filters_from_key_async + # [/DEF:extract_native_filters_from_key_async:Function] - # #region parse_dashboard_url_for_filters_async [C:3] [TYPE Function] - # @BRIEF Parse a Superset dashboard URL and extract native filter state asynchronously. - # @POST Returns extracted filter state or empty dict if no filters found. - # @DATA_CONTRACT Input[url: str] -> Output[Dict] - # @RELATION CALLS -> [extract_native_filters_from_permalink_async] - # @RELATION CALLS -> [extract_native_filters_from_key_async] + # [DEF:parse_dashboard_url_for_filters_async:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously. + # @POST: Returns extracted filter state or empty dict if no filters found. + # @DATA_CONTRACT: Input[url: str] -> Output[Dict] + # @RELATION: [CALLS] ->[extract_native_filters_from_permalink_async] + # @RELATION: [CALLS] ->[extract_native_filters_from_key_async] async def parse_dashboard_url_for_filters_async(self, url: str) -> Dict: with belief_scope( "AsyncSupersetClient.parse_dashboard_url_for_filters_async", f"url={url}" @@ -662,7 +671,9 @@ class AsyncSupersetClient(SupersetClient): return result - # #endregion parse_dashboard_url_for_filters_async + # [/DEF:parse_dashboard_url_for_filters_async:Function] # #endregion AsyncSupersetClient + +# #endregion AsyncSupersetClientModule diff --git a/backend/src/core/auth/__tests__/test_auth.py b/backend/src/core/auth/__tests__/test_auth.py index 89b3fc5b..3da432d0 100644 --- a/backend/src/core/auth/__tests__/test_auth.py +++ b/backend/src/core/auth/__tests__/test_auth.py @@ -1,7 +1,8 @@ -# #region test_auth [C:3] [TYPE Module] -# @BRIEF Unit tests for authentication module -# @LAYER Domain -# @RELATION VERIFIES -> [AuthPackage] +# [DEF:test_auth:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for authentication module +# @LAYER: Domain +# @RELATION: VERIFIES -> AuthPackage import sys from pathlib import Path @@ -57,9 +58,9 @@ def auth_repo(db_session): return AuthRepository(db_session) -# #region test_create_user [TYPE Function] -# @BRIEF Verifies that a persisted user can be retrieved with intact credential hash. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_create_user:Function] +# @PURPOSE: Verifies that a persisted user can be retrieved with intact credential hash. +# @RELATION: BINDS_TO -> test_auth def test_create_user(auth_repo): """Test user creation""" user = User( @@ -79,12 +80,12 @@ def test_create_user(auth_repo): assert verify_password("testpassword123", retrieved_user.password_hash) -# #endregion test_create_user +# [/DEF:test_create_user:Function] -# #region test_authenticate_user [TYPE Function] -# @BRIEF Validates authentication outcomes for valid, wrong-password, and unknown-user cases. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_authenticate_user:Function] +# @PURPOSE: Validates authentication outcomes for valid, wrong-password, and unknown-user cases. +# @RELATION: BINDS_TO -> test_auth def test_authenticate_user(auth_service, auth_repo): """Test user authentication with valid and invalid credentials""" user = User( @@ -111,12 +112,12 @@ def test_authenticate_user(auth_service, auth_repo): assert invalid_user is None -# #endregion test_authenticate_user +# [/DEF:test_authenticate_user:Function] -# #region test_create_session [TYPE Function] -# @BRIEF Ensures session creation returns bearer token payload fields. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_create_session:Function] +# @PURPOSE: Ensures session creation returns bearer token payload fields. +# @RELATION: BINDS_TO -> test_auth def test_create_session(auth_service, auth_repo): """Test session token creation""" user = User( @@ -136,12 +137,12 @@ def test_create_session(auth_service, auth_repo): assert len(session["access_token"]) > 0 -# #endregion test_create_session +# [/DEF:test_create_session:Function] -# #region test_role_permission_association [TYPE Function] -# @BRIEF Confirms role-permission many-to-many assignments persist and reload correctly. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_role_permission_association:Function] +# @PURPOSE: Confirms role-permission many-to-many assignments persist and reload correctly. +# @RELATION: BINDS_TO -> test_auth def test_role_permission_association(auth_repo): """Test role and permission association""" role = Role(name="Admin", description="System administrator") @@ -162,12 +163,12 @@ def test_role_permission_association(auth_repo): assert "admin:users:WRITE" in permissions -# #endregion test_role_permission_association +# [/DEF:test_role_permission_association:Function] -# #region test_user_role_association [TYPE Function] -# @BRIEF Confirms user-role assignment persists and is queryable from repository reads. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_user_role_association:Function] +# @PURPOSE: Confirms user-role assignment persists and is queryable from repository reads. +# @RELATION: BINDS_TO -> test_auth def test_user_role_association(auth_repo): """Test user and role association""" role = Role(name="Admin", description="System administrator") @@ -190,12 +191,12 @@ def test_user_role_association(auth_repo): assert retrieved_user.roles[0].name == "Admin" -# #endregion test_user_role_association +# [/DEF:test_user_role_association:Function] -# #region test_ad_group_mapping [TYPE Function] -# @BRIEF Verifies AD group mapping rows persist and reference the expected role. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_ad_group_mapping:Function] +# @PURPOSE: Verifies AD group mapping rows persist and reference the expected role. +# @RELATION: BINDS_TO -> test_auth def test_ad_group_mapping(auth_repo): """Test AD group mapping""" role = Role(name="ADFS_Admin", description="ADFS administrators") @@ -217,12 +218,12 @@ def test_ad_group_mapping(auth_repo): assert retrieved_mapping.role_id == role.id -# #endregion test_ad_group_mapping +# [/DEF:test_ad_group_mapping:Function] -# #region test_authenticate_user_updates_last_login [TYPE Function] -# @BRIEF Verifies successful authentication updates last_login audit field. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_authenticate_user_updates_last_login:Function] +# @PURPOSE: Verifies successful authentication updates last_login audit field. +# @RELATION: BINDS_TO -> test_auth def test_authenticate_user_updates_last_login(auth_service, auth_repo): """@SIDE_EFFECT: authenticate_user updates last_login timestamp on success.""" user = User( @@ -241,12 +242,12 @@ def test_authenticate_user_updates_last_login(auth_service, auth_repo): assert authenticated.last_login is not None -# #endregion test_authenticate_user_updates_last_login +# [/DEF:test_authenticate_user_updates_last_login:Function] -# #region test_authenticate_inactive_user [TYPE Function] -# @BRIEF Verifies inactive accounts are rejected during password authentication. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_authenticate_inactive_user:Function] +# @PURPOSE: Verifies inactive accounts are rejected during password authentication. +# @RELATION: BINDS_TO -> test_auth def test_authenticate_inactive_user(auth_service, auth_repo): """@PRE: User with is_active=False should not authenticate.""" user = User( @@ -263,24 +264,24 @@ def test_authenticate_inactive_user(auth_service, auth_repo): assert result is None -# #endregion test_authenticate_inactive_user +# [/DEF:test_authenticate_inactive_user:Function] -# #region test_verify_password_empty_hash [TYPE Function] -# @BRIEF Verifies password verification safely rejects empty or null password hashes. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_verify_password_empty_hash:Function] +# @PURPOSE: Verifies password verification safely rejects empty or null password hashes. +# @RELATION: BINDS_TO -> test_auth def test_verify_password_empty_hash(): """@PRE: verify_password with empty/None hash returns False.""" assert verify_password("anypassword", "") is False assert verify_password("anypassword", None) is False -# #endregion test_verify_password_empty_hash +# [/DEF:test_verify_password_empty_hash:Function] -# #region test_provision_adfs_user_new [TYPE Function] -# @BRIEF Verifies JIT provisioning creates a new ADFS user and maps group-derived roles. -# @RELATION BINDS_TO -> [test_auth] +# [DEF:test_provision_adfs_user_new:Function] +# @PURPOSE: Verifies JIT provisioning creates a new ADFS user and maps group-derived roles. +# @RELATION: BINDS_TO -> test_auth def test_provision_adfs_user_new(auth_service, auth_repo): """@POST: provision_adfs_user creates a new ADFS user with correct roles.""" # Set up a role and AD group mapping @@ -307,7 +308,7 @@ def test_provision_adfs_user_new(auth_service, auth_repo): assert user.roles[0].name == "ADFS_Viewer" -# #endregion test_provision_adfs_user_new +# [/DEF:test_provision_adfs_user_new:Function] # [DEF:test_provision_adfs_user_existing:Function] @@ -337,5 +338,5 @@ def test_provision_adfs_user_existing(auth_service, auth_repo): assert len(user.roles) == 0 # No matching group mappings -# #endregion test_auth -# #endregion test_provision_adfs_user_existing +# [/DEF:test_auth:Module] +# [/DEF:test_provision_adfs_user_existing:Function] diff --git a/backend/src/core/auth/config.py b/backend/src/core/auth/config.py index 677d3821..516c56db 100644 --- a/backend/src/core/auth/config.py +++ b/backend/src/core/auth/config.py @@ -1,21 +1,19 @@ # #region AuthConfigModule [C:2] [TYPE Module] [SEMANTICS auth, config, settings, jwt, adfs] +# # @BRIEF Centralized configuration for authentication and authorization. -# @LAYER Core -# @INVARIANT All sensitive configuration must have defaults or be loaded from environment. -# @RELATION DEPENDS_ON -> [pydantic] -# +# @LAYER: Core +# @RELATION DEPENDS_ON -> pydantic # +# @INVARIANT: All sensitive configuration must have defaults or be loaded from environment. -# [SECTION: IMPORTS] from pydantic import Field from pydantic_settings import BaseSettings -# [/SECTION] # #region AuthConfig [TYPE Class] # @BRIEF Holds authentication-related settings. -# @PRE Environment variables may be provided via .env file. -# @POST Returns a configuration object with validated settings. -# @RELATION INHERITS -> [pydantic_settings.BaseSettings] +# @PRE: Environment variables may be provided via .env file. +# @POST: Returns a configuration object with validated settings. +# @RELATION INHERITS -> pydantic_settings.BaseSettings class AuthConfig(BaseSettings): # JWT Settings SECRET_KEY: str = Field(default="super-secret-key-change-in-production", env="AUTH_SECRET_KEY") @@ -41,7 +39,7 @@ class AuthConfig(BaseSettings): # #region auth_config [TYPE Variable] # @BRIEF Singleton instance of AuthConfig. -# @RELATION DEPENDS_ON -> [AuthConfig] +# @RELATION DEPENDS_ON -> AuthConfig auth_config = AuthConfig() # #endregion auth_config diff --git a/backend/src/core/auth/jwt.py b/backend/src/core/auth/jwt.py index 41406869..c2818f0e 100644 --- a/backend/src/core/auth/jwt.py +++ b/backend/src/core/auth/jwt.py @@ -1,30 +1,25 @@ # #region AuthJwtModule [C:3] [TYPE Module] [SEMANTICS jwt, token, session, auth] +# # @BRIEF JWT token generation and validation logic. -# @LAYER Core -# @INVARIANT Tokens must include expiration time and user identifier. +# @LAYER: Core # @RELATION DEPENDS_ON -> [auth_config] # @RELATION USES -> [auth_config] # -# +# @INVARIANT: Tokens must include expiration time and user identifier. -# [SECTION: IMPORTS] from datetime import datetime, timedelta from typing import Optional from jose import jwt from .config import auth_config from ..logger import belief_scope -# [/SECTION] # #region create_access_token [TYPE Function] # @BRIEF Generates a new JWT access token. -# @PRE data dict contains 'sub' (user_id) and optional 'scopes' (roles). -# @POST Returns a signed JWT string. +# @PRE: data dict contains 'sub' (user_id) and optional 'scopes' (roles). +# @POST: Returns a signed JWT string. # @RELATION DEPENDS_ON -> [auth_config] # -# @PARAM: data (dict) - Payload data for the token. -# @PARAM: expires_delta (Optional[timedelta]) - Custom expiration time. -# @RETURN: str - The encoded JWT. def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str: with belief_scope("create_access_token"): to_encode = data.copy() @@ -47,13 +42,10 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) - # #region decode_token [TYPE Function] # @BRIEF Decodes and validates a JWT token. -# @PRE token is a signed JWT string. -# @POST Returns the decoded payload if valid. +# @PRE: token is a signed JWT string. +# @POST: Returns the decoded payload if valid. # @RELATION DEPENDS_ON -> [auth_config] # -# @PARAM: token (str) - The JWT to decode. -# @RETURN: dict - The decoded payload. -# @THROW: jose.JWTError - If token is invalid or expired. def decode_token(token: str) -> dict: with belief_scope("decode_token"): payload = jwt.decode( diff --git a/backend/src/core/auth/logger.py b/backend/src/core/auth/logger.py index f2a37f4f..85a49d67 100644 --- a/backend/src/core/auth/logger.py +++ b/backend/src/core/auth/logger.py @@ -1,24 +1,19 @@ # #region AuthLoggerModule [C:3] [TYPE Module] [SEMANTICS auth, logger, audit, security] +# # @BRIEF Audit logging for security-related events. -# @LAYER Core -# @INVARIANT Must not log sensitive data like passwords or full tokens. -# @RELATION USES -> [belief_scope] -# +# @LAYER: Core +# @RELATION USES -> belief_scope # +# @INVARIANT: Must not log sensitive data like passwords or full tokens. -# [SECTION: IMPORTS] from ..logger import logger, belief_scope from datetime import datetime -# [/SECTION] # #region log_security_event [TYPE Function] # @BRIEF Logs a security-related event for audit trails. -# @PRE event_type and username are strings. -# @POST Security event is written to the application log. -# @RELATION USES -> [logger] -# @PARAM: event_type (str) - Type of event (e.g., LOGIN_SUCCESS, PERMISSION_DENIED). -# @PARAM: username (str) - The user involved in the event. -# @PARAM: details (dict) - Additional non-sensitive metadata. +# @PRE: event_type and username are strings. +# @POST: Security event is written to the application log. +# @RELATION USES -> logger def log_security_event(event_type: str, username: str, details: dict = None): with belief_scope("log_security_event", f"{event_type}:{username}"): timestamp = datetime.utcnow().isoformat() @@ -28,4 +23,4 @@ def log_security_event(event_type: str, username: str, details: dict = None): logger.info(msg) # #endregion log_security_event -# #endregion AuthLoggerModule +# #endregion AuthLoggerModule \ No newline at end of file diff --git a/backend/src/core/auth/oauth.py b/backend/src/core/auth/oauth.py index 4d466644..c74a1bf0 100644 --- a/backend/src/core/auth/oauth.py +++ b/backend/src/core/auth/oauth.py @@ -1,29 +1,27 @@ # #region AuthOauthModule [C:2] [TYPE Module] [SEMANTICS auth, oauth, oidc, adfs] +# # @BRIEF ADFS OIDC configuration and client using Authlib. -# @LAYER Core -# @INVARIANT Must use secure OIDC flows. -# @RELATION DEPENDS_ON -> [authlib] -# @RELATION USES -> [auth_config] -# +# @LAYER: Core +# @RELATION DEPENDS_ON -> authlib +# @RELATION USES -> auth_config # +# @INVARIANT: Must use secure OIDC flows. -# [SECTION: IMPORTS] from authlib.integrations.starlette_client import OAuth from .config import auth_config -# [/SECTION] # #region oauth [TYPE Variable] # @BRIEF Global Authlib OAuth registry. -# @RELATION DEPENDS_ON -> [OAuth] +# @RELATION DEPENDS_ON -> OAuth oauth = OAuth() # #endregion oauth # #region register_adfs [TYPE Function] # @BRIEF Registers the ADFS OIDC client. -# @PRE ADFS configuration is provided in auth_config. -# @POST ADFS client is registered in oauth registry. -# @RELATION USES -> [oauth] -# @RELATION USES -> [auth_config] +# @PRE: ADFS configuration is provided in auth_config. +# @POST: ADFS client is registered in oauth registry. +# @RELATION USES -> oauth +# @RELATION USES -> auth_config def register_adfs(): if auth_config.ADFS_CLIENT_ID: oauth.register( @@ -39,10 +37,9 @@ def register_adfs(): # #region is_adfs_configured [TYPE Function] # @BRIEF Checks if ADFS is properly configured. -# @PRE None. -# @POST Returns True if ADFS client is registered, False otherwise. -# @RETURN bool - Configuration status. -# @RELATION USES -> [oauth] +# @PRE: None. +# @POST: Returns True if ADFS client is registered, False otherwise. +# @RELATION USES -> oauth def is_adfs_configured() -> bool: """Check if ADFS OAuth client is registered.""" return 'adfs' in oauth._registry @@ -51,4 +48,4 @@ def is_adfs_configured() -> bool: # Initial registration register_adfs() -# #endregion AuthOauthModule +# #endregion AuthOauthModule \ No newline at end of file diff --git a/backend/src/core/auth/repository.py b/backend/src/core/auth/repository.py index eb596398..8c07e9bf 100644 --- a/backend/src/core/auth/repository.py +++ b/backend/src/core/auth/repository.py @@ -1,39 +1,37 @@ # #region AuthRepositoryModule [C:5] [TYPE Module] [SEMANTICS auth, repository, database, user, role, permission] # @BRIEF Data access layer for authentication and user preference entities. -# @LAYER Domain -# @PRE Database connection is active. -# @POST Provides valid access to identity data. -# @SIDE_EFFECT Executes database read queries through the injected SQLAlchemy session boundary. -# @DATA_CONTRACT Input[sqlalchemy.orm.Session] -> Output[User|Role|Permission|UserDashboardPreference access] -# @INVARIANT All database read/write operations must execute via the injected SQLAlchemy session boundary. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [AuthModels] # @RELATION DEPENDS_ON -> [ProfileModels] # @RELATION DEPENDS_ON -> [belief_scope] +# @INVARIANT: All database read/write operations must execute via the injected SQLAlchemy session boundary. +# @DATA_CONTRACT: Input[sqlalchemy.orm.Session] -> Output[User|Role|Permission|UserDashboardPreference access] +# @PRE: Database connection is active. +# @POST: Provides valid access to identity data. +# @SIDE_EFFECT: Executes database read queries through the injected SQLAlchemy session boundary. -# [SECTION: IMPORTS] from typing import List, Optional from sqlalchemy.orm import Session, selectinload from ...models.auth import Permission, Role, User, ADGroupMapping from ...models.profile import UserDashboardPreference from ..logger import belief_scope, logger -# [/SECTION] # #region AuthRepository [TYPE Class] # @BRIEF Provides low-level CRUD operations for identity and authorization records. -# @PRE Database session is bound. -# @POST Entity instances returned safely. -# @SIDE_EFFECT Performs database reads. +# @PRE: Database session is bound. +# @POST: Entity instances returned safely. +# @SIDE_EFFECT: Performs database reads. # @RELATION DEPENDS_ON -> [AuthModels] class AuthRepository: def __init__(self, db: Session): self.db = db - # #region get_user_by_id [TYPE Function] - # @BRIEF Retrieve user by UUID. - # @PRE user_id is a valid UUID string. - # @POST Returns User object if found, else None. - # @RELATION DEPENDS_ON -> [User] + # [DEF:get_user_by_id:Function] + # @PURPOSE: Retrieve user by UUID. + # @PRE: user_id is a valid UUID string. + # @POST: Returns User object if found, else None. + # @RELATION: DEPENDS_ON -> [User] def get_user_by_id(self, user_id: str) -> Optional[User]: with belief_scope("AuthRepository.get_user_by_id"): logger.reason(f"Fetching user by id: {user_id}") @@ -41,13 +39,13 @@ class AuthRepository: logger.reflect(f"User found: {result is not None}") return result - # #endregion get_user_by_id + # [/DEF:get_user_by_id:Function] - # #region get_user_by_username [TYPE Function] - # @BRIEF Retrieve user by username. - # @PRE username is a non-empty string. - # @POST Returns User object if found, else None. - # @RELATION DEPENDS_ON -> [User] + # [DEF:get_user_by_username:Function] + # @PURPOSE: Retrieve user by username. + # @PRE: username is a non-empty string. + # @POST: Returns User object if found, else None. + # @RELATION: DEPENDS_ON -> [User] def get_user_by_username(self, username: str) -> Optional[User]: with belief_scope("AuthRepository.get_user_by_username"): logger.reason(f"Fetching user by username: {username}") @@ -55,12 +53,12 @@ class AuthRepository: logger.reflect(f"User found: {result is not None}") return result - # #endregion get_user_by_username + # [/DEF:get_user_by_username:Function] - # #region get_role_by_id [TYPE Function] - # @BRIEF Retrieve role by UUID with permissions preloaded. - # @RELATION DEPENDS_ON -> [Role] - # @RELATION DEPENDS_ON -> [Permission] + # [DEF:get_role_by_id:Function] + # @PURPOSE: Retrieve role by UUID with permissions preloaded. + # @RELATION: DEPENDS_ON -> [Role] + # @RELATION: DEPENDS_ON -> [Permission] def get_role_by_id(self, role_id: str) -> Optional[Role]: with belief_scope("AuthRepository.get_role_by_id"): return ( @@ -70,31 +68,31 @@ class AuthRepository: .first() ) - # #endregion get_role_by_id + # [/DEF:get_role_by_id:Function] - # #region get_role_by_name [TYPE Function] - # @BRIEF Retrieve role by unique name. - # @RELATION DEPENDS_ON -> [Role] + # [DEF:get_role_by_name:Function] + # @PURPOSE: Retrieve role by unique name. + # @RELATION: DEPENDS_ON -> [Role] def get_role_by_name(self, name: str) -> Optional[Role]: with belief_scope("AuthRepository.get_role_by_name"): return self.db.query(Role).filter(Role.name == name).first() - # #endregion get_role_by_name + # [/DEF:get_role_by_name:Function] - # #region get_permission_by_id [TYPE Function] - # @BRIEF Retrieve permission by UUID. - # @RELATION DEPENDS_ON -> [Permission] + # [DEF:get_permission_by_id:Function] + # @PURPOSE: Retrieve permission by UUID. + # @RELATION: DEPENDS_ON -> [Permission] def get_permission_by_id(self, permission_id: str) -> Optional[Permission]: with belief_scope("AuthRepository.get_permission_by_id"): return ( self.db.query(Permission).filter(Permission.id == permission_id).first() ) - # #endregion get_permission_by_id + # [/DEF:get_permission_by_id:Function] - # #region get_permission_by_resource_action [TYPE Function] - # @BRIEF Retrieve permission by resource and action tuple. - # @RELATION DEPENDS_ON -> [Permission] + # [DEF:get_permission_by_resource_action:Function] + # @PURPOSE: Retrieve permission by resource and action tuple. + # @RELATION: DEPENDS_ON -> [Permission] def get_permission_by_resource_action( self, resource: str, action: str ) -> Optional[Permission]: @@ -105,20 +103,20 @@ class AuthRepository: .first() ) - # #endregion get_permission_by_resource_action + # [/DEF:get_permission_by_resource_action:Function] - # #region list_permissions [TYPE Function] - # @BRIEF List all system permissions. - # @RELATION DEPENDS_ON -> [Permission] + # [DEF:list_permissions:Function] + # @PURPOSE: List all system permissions. + # @RELATION: DEPENDS_ON -> [Permission] def list_permissions(self) -> List[Permission]: with belief_scope("AuthRepository.list_permissions"): return self.db.query(Permission).all() - # #endregion list_permissions + # [/DEF:list_permissions:Function] - # #region get_user_dashboard_preference [TYPE Function] - # @BRIEF Retrieve dashboard filters/preferences for a user. - # @RELATION DEPENDS_ON -> [UserDashboardPreference] + # [DEF:get_user_dashboard_preference:Function] + # @PURPOSE: Retrieve dashboard filters/preferences for a user. + # @RELATION: DEPENDS_ON -> [UserDashboardPreference] def get_user_dashboard_preference( self, user_id: str ) -> Optional[UserDashboardPreference]: @@ -129,14 +127,14 @@ class AuthRepository: .first() ) - # #endregion get_user_dashboard_preference + # [/DEF:get_user_dashboard_preference:Function] - # #region get_roles_by_ad_groups [TYPE Function] - # @BRIEF Retrieve roles that match a list of AD group names. - # @PRE groups is a list of strings representing AD group identifiers. - # @POST Returns a list of Role objects mapped to the provided AD groups. - # @RELATION DEPENDS_ON -> [Role] - # @RELATION DEPENDS_ON -> [ADGroupMapping] + # [DEF:get_roles_by_ad_groups:Function] + # @PURPOSE: Retrieve roles that match a list of AD group names. + # @PRE: groups is a list of strings representing AD group identifiers. + # @POST: Returns a list of Role objects mapped to the provided AD groups. + # @RELATION: DEPENDS_ON -> [Role] + # @RELATION: DEPENDS_ON -> [ADGroupMapping] def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]: with belief_scope("AuthRepository.get_roles_by_ad_groups"): logger.reason(f"Fetching roles for AD groups: {groups}") @@ -149,7 +147,7 @@ class AuthRepository: .all() ) - # #endregion get_roles_by_ad_groups + # [/DEF:get_roles_by_ad_groups:Function] # #endregion AuthRepository diff --git a/backend/src/core/auth/security.py b/backend/src/core/auth/security.py index 921de333..9792cfa1 100644 --- a/backend/src/core/auth/security.py +++ b/backend/src/core/auth/security.py @@ -1,24 +1,19 @@ # #region AuthSecurityModule [C:2] [TYPE Module] [SEMANTICS security, password, hashing, bcrypt] +# # @BRIEF Utility for password hashing and verification using Passlib. -# @LAYER Core -# @INVARIANT Uses bcrypt for hashing with standard work factor. -# @RELATION DEPENDS_ON -> [bcrypt] -# +# @LAYER: Core +# @RELATION DEPENDS_ON -> bcrypt # +# @INVARIANT: Uses bcrypt for hashing with standard work factor. -# [SECTION: IMPORTS] import bcrypt -# [/SECTION] # #region verify_password [TYPE Function] # @BRIEF Verifies a plain password against a hashed password. -# @PRE plain_password is a string, hashed_password is a bcrypt hash. -# @POST Returns True if password matches, False otherwise. -# @RELATION DEPENDS_ON -> [bcrypt] +# @PRE: plain_password is a string, hashed_password is a bcrypt hash. +# @POST: Returns True if password matches, False otherwise. +# @RELATION DEPENDS_ON -> bcrypt # -# @PARAM: plain_password (str) - The unhashed password. -# @PARAM: hashed_password (str) - The stored hash. -# @RETURN: bool - Verification result. def verify_password(plain_password: str, hashed_password: str) -> bool: if not hashed_password: return False @@ -33,12 +28,10 @@ def verify_password(plain_password: str, hashed_password: str) -> bool: # #region get_password_hash [TYPE Function] # @BRIEF Generates a bcrypt hash for a plain password. -# @PRE password is a string. -# @POST Returns a secure bcrypt hash string. -# @RELATION DEPENDS_ON -> [bcrypt] +# @PRE: password is a string. +# @POST: Returns a secure bcrypt hash string. +# @RELATION DEPENDS_ON -> bcrypt # -# @PARAM: password (str) - The password to hash. -# @RETURN: str - The generated hash. def get_password_hash(password: str) -> str: return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") # #endregion get_password_hash diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py index 8ac92ddf..703db898 100644 --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -1,18 +1,18 @@ # #region ConfigManager [C:5] [TYPE Module] [SEMANTICS config, manager, persistence, migration, postgresql] +# # @BRIEF Manages application configuration persistence in DB with one-time migration from legacy JSON. -# @LAYER Domain -# @PRE Database schema for AppConfigRecord must be initialized. -# @POST Configuration is loaded into memory and logger is configured. -# @SIDE_EFFECT Performs DB I/O and may update global logging level. -# @DATA_CONTRACT Input[json, record] -> Model[AppConfig] -# @INVARIANT Configuration must always be representable by AppConfig and persisted under global record id. +# @LAYER: Domain +# @PRE: Database schema for AppConfigRecord must be initialized. +# @POST: Configuration is loaded into memory and logger is configured. +# @SIDE_EFFECT: Performs DB I/O and may update global logging level. +# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig] +# @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id. # @RELATION DEPENDS_ON -> [AppConfig] # @RELATION DEPENDS_ON -> [SessionLocal] # @RELATION DEPENDS_ON -> [AppConfigRecord] # @RELATION CALLS -> [logger] # @RELATION CALLS -> [configure_logger] # -# import json import os from pathlib import Path @@ -24,30 +24,30 @@ from .config_models import AppConfig, Environment, GlobalSettings from .database import SessionLocal from ..models.config import AppConfigRecord from ..models.mapping import Environment as EnvironmentRecord -from .logger import configure_logger, belief_scope -from .cot_logger import MarkerLogger +from .logger import logger, configure_logger, belief_scope -log = MarkerLogger("ConfigManager") # #region ConfigManager [C:5] [TYPE Class] # @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle. -# @PRE Database is accessible and AppConfigRecord schema is loaded. -# @POST Configuration state is synchronized between memory and database. -# @SIDE_EFFECT Performs DB I/O, OS path validation, and logger reconfiguration. +# @PRE: Database is accessible and AppConfigRecord schema is loaded. +# @POST: Configuration state is synchronized between memory and database. +# @SIDE_EFFECT: Performs DB I/O, OS path validation, and logger reconfiguration. class ConfigManager: - # #region __init__ [TYPE Function] - # @BRIEF Initialize manager state from persisted or migrated configuration. - # @PRE config_path is a non-empty string path. - # @POST self.config is initialized as AppConfig and logger is configured. - # @SIDE_EFFECT Reads config sources and updates logging configuration. - # @DATA_CONTRACT Input(str config_path) -> Output(None; self.config: AppConfig) + # [DEF:__init__:Function] + # @PURPOSE: Initialize manager state from persisted or migrated configuration. + # @PRE: config_path is a non-empty string path. + # @POST: self.config is initialized as AppConfig and logger is configured. + # @SIDE_EFFECT: Reads config sources and updates logging configuration. + # @DATA_CONTRACT: Input(str config_path) -> Output(None; self.config: AppConfig) def __init__(self, config_path: str = "config.json"): with belief_scope("ConfigManager.__init__"): if not isinstance(config_path, str) or not config_path: - log.explore("Invalid config_path provided", error="config_path must be a non-empty string", payload={"path": config_path}) + logger.explore( + "Invalid config_path provided", extra={"path": config_path} + ) raise ValueError("config_path must be a non-empty string") - log.reason(f"Initializing ConfigManager with legacy path: {config_path}") + logger.reason(f"Initializing ConfigManager with legacy path: {config_path}") self.config_path = Path(config_path) self.raw_payload: dict[str, Any] = {} @@ -56,17 +56,20 @@ class ConfigManager: configure_logger(self.config.settings.logging) if not isinstance(self.config, AppConfig): - log.explore("Config loading resulted in invalid type", error="Loaded config is not an AppConfig instance", payload={"type": type(self.config)}) + logger.explore( + "Config loading resulted in invalid type", + extra={"type": type(self.config)}, + ) raise TypeError("self.config must be an instance of AppConfig") - log.reflect("ConfigManager initialization complete") + logger.reflect("ConfigManager initialization complete") - # #endregion __init__ + # [/DEF:__init__:Function] - # #region _apply_features_from_env [TYPE Function] - # @BRIEF Read FEATURES__* env vars and apply them to a GlobalSettings features config. - # @SIDE_EFFECT Reads os.environ; mutates settings.features in-place. - # @RATIONALE Env vars seed the initial defaults. After first bootstrap, DB is source of truth. + # [DEF:_apply_features_from_env:Function] + # @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config. + # @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place. + # @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth. @staticmethod def _apply_features_from_env(settings: GlobalSettings) -> None: with belief_scope("ConfigManager._apply_features_from_env"): @@ -74,35 +77,35 @@ class ConfigManager: if dataset_review_env is not None: parsed = dataset_review_env.strip().lower() in ("true", "1", "yes") settings.features.dataset_review = parsed - log.reason( + logger.reason( "Applied FEATURES__DATASET_REVIEW from env", - payload={"value": parsed, "raw": dataset_review_env}, + extra={"value": parsed, "raw": dataset_review_env}, ) health_monitor_env = os.getenv("FEATURES__HEALTH_MONITOR") if health_monitor_env is not None: parsed = health_monitor_env.strip().lower() in ("true", "1", "yes") settings.features.health_monitor = parsed - log.reason( + logger.reason( "Applied FEATURES__HEALTH_MONITOR from env", - payload={"value": parsed, "raw": health_monitor_env}, + extra={"value": parsed, "raw": health_monitor_env}, ) - # #endregion _apply_features_from_env + # [/DEF:_apply_features_from_env:Function] - # #region _default_config [TYPE Function] - # @BRIEF Build default application configuration fallback. + # [DEF:_default_config:Function] + # @PURPOSE: Build default application configuration fallback. def _default_config(self) -> AppConfig: with belief_scope("ConfigManager._default_config"): - log.reason("Building default AppConfig fallback") + logger.reason("Building default AppConfig fallback") config = AppConfig(environments=[], settings=GlobalSettings()) self._apply_features_from_env(config.settings) return config - # #endregion _default_config + # [/DEF:_default_config:Function] - # #region _sync_raw_payload_from_config [TYPE Function] - # @BRIEF Merge typed AppConfig state into raw payload while preserving unsupported legacy sections. + # [DEF:_sync_raw_payload_from_config:Function] + # @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections. def _sync_raw_payload_from_config(self) -> dict[str, Any]: with belief_scope("ConfigManager._sync_raw_payload_from_config"): typed_payload = self.config.model_dump() @@ -110,9 +113,9 @@ class ConfigManager: merged_payload["environments"] = typed_payload.get("environments", []) merged_payload["settings"] = typed_payload.get("settings", {}) self.raw_payload = merged_payload - log.reason( + logger.reason( "Synchronized raw payload from typed config", - payload={ + extra={ "environments_count": len( merged_payload.get("environments", []) or [] ), @@ -126,42 +129,45 @@ class ConfigManager: ) return merged_payload - # #endregion _sync_raw_payload_from_config + # [/DEF:_sync_raw_payload_from_config:Function] - # #region _load_from_legacy_file [TYPE Function] - # @BRIEF Load legacy JSON configuration for migration fallback path. + # [DEF:_load_from_legacy_file:Function] + # @PURPOSE: Load legacy JSON configuration for migration fallback path. def _load_from_legacy_file(self) -> dict[str, Any]: with belief_scope("ConfigManager._load_from_legacy_file"): if not self.config_path.exists(): - log.reason( + logger.reason( "Legacy config file not found; using default payload", - payload={"path": str(self.config_path)}, + extra={"path": str(self.config_path)}, ) return {} - log.reason( - "Loading legacy config file", payload={"path": str(self.config_path)} + logger.reason( + "Loading legacy config file", extra={"path": str(self.config_path)} ) with self.config_path.open("r", encoding="utf-8") as fh: payload = json.load(fh) if not isinstance(payload, dict): - log.explore("Legacy config payload is not a JSON object", error=f"Expected dict, got {type(payload).__name__}", payload={ - "path": str(self.config_path), - "type": type(payload).__name__, - }) + logger.explore( + "Legacy config payload is not a JSON object", + extra={ + "path": str(self.config_path), + "type": type(payload).__name__, + }, + ) raise ValueError("Legacy config payload must be a JSON object") - log.reason( + logger.reason( "Legacy config file loaded successfully", - payload={"path": str(self.config_path), "keys": sorted(payload.keys())}, + extra={"path": str(self.config_path), "keys": sorted(payload.keys())}, ) return payload - # #endregion _load_from_legacy_file + # [/DEF:_load_from_legacy_file:Function] - # #region _get_record [TYPE Function] - # @BRIEF Resolve global configuration record from DB. + # [DEF:_get_record:Function] + # @PURPOSE: Resolve global configuration record from DB. def _get_record(self, session: Session) -> Optional[AppConfigRecord]: with belief_scope("ConfigManager._get_record"): record = ( @@ -169,24 +175,24 @@ class ConfigManager: .filter(AppConfigRecord.id == "global") .first() ) - log.reason( - "Resolved app config record", payload={"exists": record is not None} + logger.reason( + "Resolved app config record", extra={"exists": record is not None} ) return record - # #endregion _get_record + # [/DEF:_get_record:Function] - # #region _load_config [TYPE Function] - # @BRIEF Load configuration from DB or perform one-time migration from legacy JSON. + # [DEF:_load_config:Function] + # @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON. def _load_config(self) -> AppConfig: with belief_scope("ConfigManager._load_config"): session = SessionLocal() try: record = self._get_record(session) if record and isinstance(record.payload, dict): - log.reason( + logger.reason( "Loading configuration from database", - payload={"record_id": record.id}, + extra={"record_id": record.id}, ) self.raw_payload = dict(record.payload) config = AppConfig.model_validate( @@ -197,18 +203,18 @@ class ConfigManager: ) self._sync_environment_records(session, config) session.commit() - log.reason( + logger.reason( "Database configuration validated successfully", - payload={ + extra={ "environments_count": len(config.environments), "payload_keys": sorted(self.raw_payload.keys()), }, ) return config - log.reason( + logger.reason( "Database configuration record missing; attempting legacy file migration", - payload={"legacy_path": str(self.config_path)}, + extra={"legacy_path": str(self.config_path)}, ) legacy_payload = self._load_from_legacy_file() @@ -221,9 +227,9 @@ class ConfigManager: } ) self._apply_features_from_env(config.settings) - log.reason( + logger.reason( "Legacy payload validated; persisting migrated configuration to database", - payload={ + extra={ "environments_count": len(config.environments), "payload_keys": sorted(self.raw_payload.keys()), }, @@ -231,7 +237,7 @@ class ConfigManager: self._save_config_to_db(config, session=session) return config - log.reason( + logger.reason( "No persisted config found; falling back to default configuration" ) config = self._default_config() @@ -239,20 +245,26 @@ class ConfigManager: self._save_config_to_db(config, session=session) return config except (json.JSONDecodeError, TypeError, ValueError) as exc: - log.explore("Recoverable config load failure; falling back to default configuration", error=str(exc), payload={"legacy_path": str(self.config_path)}) + logger.explore( + "Recoverable config load failure; falling back to default configuration", + extra={"error": str(exc), "legacy_path": str(self.config_path)}, + ) config = self._default_config() self.raw_payload = config.model_dump() return config except Exception as exc: - log.explore("Critical config load failure; re-raising persistence or validation error", error=str(exc)) + logger.explore( + "Critical config load failure; re-raising persistence or validation error", + extra={"error": str(exc)}, + ) raise finally: session.close() - # #endregion _load_config + # [/DEF:_load_config:Function] - # #region _sync_environment_records [TYPE Function] - # @BRIEF Mirror configured environments into the relational environments table used by FK-backed domain models. + # [DEF:_sync_environment_records:Function] + # @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models. def _sync_environment_records(self, session: Session, config: AppConfig) -> None: with belief_scope("ConfigManager._sync_environment_records"): configured_envs = list(config.environments or []) @@ -276,9 +288,9 @@ class ConfigManager: record = persisted_by_id.get(normalized_id) if record is None: - log.reason( + logger.reason( "Creating relational environment record from typed config", - payload={ + extra={ "environment_id": normalized_id, "environment_name": display_name, }, @@ -297,10 +309,37 @@ class ConfigManager: record.url = normalized_url record.credentials_id = credentials_id - # #endregion _sync_environment_records + # [/DEF:_sync_environment_records:Function] - # #region _save_config_to_db [TYPE Function] - # @BRIEF Persist provided AppConfig into the global DB configuration record. + # [DEF:_delete_stale_environment_records:Function] + # @PURPOSE: Remove persisted environment records that are no longer in the configured environments. + # @PRE: _sync_environment_records must already have run so the query returns a current view. + # @POST: Stale Environment rows are deleted via the session (caller must commit). + # @SIDE_EFFECT: Only safe to call during explicit save (_save_config_to_db), NOT during _load_config, + # because FK-dependent rows (task_records, database_mappings, etc.) may reference deleted rows. + def _delete_stale_environment_records( + self, session: Session, config: AppConfig + ) -> None: + with belief_scope("ConfigManager._delete_stale_environment_records"): + persisted_records = session.query(EnvironmentRecord).all() + configured_ids = { + str(env.id or "").strip() + for env in (config.environments or []) + if str(env.id or "").strip() + } + for record in persisted_records: + record_id = str(record.id or "").strip() + if record_id and record_id not in configured_ids: + logger.reason( + "Deleting stale environment record", + extra={"environment_id": record_id}, + ) + session.delete(record) + + # [/DEF:_delete_stale_environment_records:Function] + + # [DEF:_save_config_to_db:Function] + # @PURPOSE: Persist provided AppConfig into the global DB configuration record. def _save_config_to_db( self, config: AppConfig, session: Optional[Session] = None ) -> None: @@ -312,22 +351,23 @@ class ConfigManager: payload = self._sync_raw_payload_from_config() record = self._get_record(db) if record is None: - log.reason("Creating new global app config record") + logger.reason("Creating new global app config record") record = AppConfigRecord(id="global", payload=payload) db.add(record) else: - log.reason( + logger.reason( "Updating existing global app config record", - payload={"record_id": record.id}, + extra={"record_id": record.id}, ) record.payload = payload self._sync_environment_records(db, config) + self._delete_stale_environment_records(db, config) db.commit() - log.reason( + logger.reason( "Configuration persisted to database", - payload={ + extra={ "environments_count": len( payload.get("environments", []) or [] ), @@ -336,54 +376,54 @@ class ConfigManager: ) except Exception: db.rollback() - log.explore("Database save failed; transaction rolled back", error="Database transaction rolled back") + logger.explore("Database save failed; transaction rolled back") raise finally: if owns_session: db.close() - # #endregion _save_config_to_db + # [/DEF:_save_config_to_db:Function] - # #region save [TYPE Function] - # @BRIEF Persist current in-memory configuration state. + # [DEF:save:Function] + # @PURPOSE: Persist current in-memory configuration state. def save(self) -> None: with belief_scope("ConfigManager.save"): - log.reason("Persisting current in-memory configuration") + logger.reason("Persisting current in-memory configuration") self._save_config_to_db(self.config) - # #endregion save + # [/DEF:save:Function] - # #region get_config [TYPE Function] - # @BRIEF Return current in-memory configuration snapshot. + # [DEF:get_config:Function] + # @PURPOSE: Return current in-memory configuration snapshot. def get_config(self) -> AppConfig: with belief_scope("ConfigManager.get_config"): return self.config - # #endregion get_config + # [/DEF:get_config:Function] - # #region get_payload [TYPE Function] - # @BRIEF Return full persisted payload including sections outside typed AppConfig schema. + # [DEF:get_payload:Function] + # @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema. def get_payload(self) -> dict[str, Any]: with belief_scope("ConfigManager.get_payload"): return self._sync_raw_payload_from_config() - # #endregion get_payload + # [/DEF:get_payload:Function] - # #region save_config [TYPE Function] - # @BRIEF Persist configuration provided either as typed AppConfig or raw payload dict. + # [DEF:save_config:Function] + # @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict. def save_config(self, config: Any) -> AppConfig: with belief_scope("ConfigManager.save_config"): if isinstance(config, AppConfig): - log.reason("Saving typed AppConfig payload") + logger.reason("Saving typed AppConfig payload") self.config = config self.raw_payload = config.model_dump() self._save_config_to_db(config) return self.config if isinstance(config, dict): - log.reason( + logger.reason( "Saving raw config payload", - payload={"keys": sorted(config.keys())}, + extra={"keys": sorted(config.keys())}, ) self.raw_payload = dict(config) typed_config = AppConfig.model_validate( @@ -396,24 +436,27 @@ class ConfigManager: self._save_config_to_db(typed_config) return self.config - log.explore("Unsupported config type supplied to save_config", error=f"Expected AppConfig or dict, got {type(config).__name__}", payload={"type": type(config).__name__}) + logger.explore( + "Unsupported config type supplied to save_config", + extra={"type": type(config).__name__}, + ) raise TypeError("config must be AppConfig or dict") - # #endregion save_config + # [/DEF:save_config:Function] - # #region update_global_settings [TYPE Function] - # @BRIEF Replace global settings and persist the resulting configuration. + # [DEF:update_global_settings:Function] + # @PURPOSE: Replace global settings and persist the resulting configuration. def update_global_settings(self, settings: GlobalSettings) -> AppConfig: with belief_scope("ConfigManager.update_global_settings"): - log.reason("Updating global settings") + logger.reason("Updating global settings") self.config.settings = settings self.save() return self.config - # #endregion update_global_settings + # [/DEF:update_global_settings:Function] - # #region validate_path [TYPE Function] - # @BRIEF Validate that path exists and is writable, creating it when absent. + # [DEF:validate_path:Function] + # @PURPOSE: Validate that path exists and is writable, creating it when absent. def validate_path(self, path: str) -> tuple[bool, str]: with belief_scope("ConfigManager.validate_path", f"path={path}"): try: @@ -431,32 +474,34 @@ class ConfigManager: fh.write("ok") test_file.unlink(missing_ok=True) - log.reason("Path validation succeeded", payload={"path": str(target)}) + logger.reason("Path validation succeeded", extra={"path": str(target)}) return True, "OK" except Exception as exc: - log.explore("Path validation failed", error=str(exc), payload={"path": path}) + logger.explore( + "Path validation failed", extra={"path": path, "error": str(exc)} + ) return False, str(exc) - # #endregion validate_path + # [/DEF:validate_path:Function] - # #region get_environments [TYPE Function] - # @BRIEF Return all configured environments. + # [DEF:get_environments:Function] + # @PURPOSE: Return all configured environments. def get_environments(self) -> List[Environment]: with belief_scope("ConfigManager.get_environments"): return list(self.config.environments) - # #endregion get_environments + # [/DEF:get_environments:Function] - # #region has_environments [TYPE Function] - # @BRIEF Check whether at least one environment exists in configuration. + # [DEF:has_environments:Function] + # @PURPOSE: Check whether at least one environment exists in configuration. def has_environments(self) -> bool: with belief_scope("ConfigManager.has_environments"): return len(self.config.environments) > 0 - # #endregion has_environments + # [/DEF:has_environments:Function] - # #region get_environment [TYPE Function] - # @BRIEF Resolve a configured environment by identifier. + # [DEF:get_environment:Function] + # @PURPOSE: Resolve a configured environment by identifier. def get_environment(self, env_id: str) -> Optional[Environment]: with belief_scope("ConfigManager.get_environment", f"env_id={env_id}"): normalized = str(env_id or "").strip() @@ -468,10 +513,10 @@ class ConfigManager: return env return None - # #endregion get_environment + # [/DEF:get_environment:Function] - # #region add_environment [TYPE Function] - # @BRIEF Upsert environment by id into configuration and persist. + # [DEF:add_environment:Function] + # @PURPOSE: Upsert environment by id into configuration and persist. def add_environment(self, env: Environment) -> AppConfig: with belief_scope("ConfigManager.add_environment", f"env_id={env.id}"): existing_index = next( @@ -487,12 +532,12 @@ class ConfigManager: item.is_default = False if existing_index is None: - log.reason("Appending new environment", payload={"env_id": env.id}) + logger.reason("Appending new environment", extra={"env_id": env.id}) self.config.environments.append(env) else: - log.reason( + logger.reason( "Replacing existing environment during add", - payload={"env_id": env.id}, + extra={"env_id": env.id}, ) self.config.environments[existing_index] = env @@ -504,10 +549,10 @@ class ConfigManager: self.save() return self.config - # #endregion add_environment + # [/DEF:add_environment:Function] - # #region update_environment [TYPE Function] - # @BRIEF Update existing environment by id and preserve masked password placeholder behavior. + # [DEF:update_environment:Function] + # @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior. def update_environment(self, env_id: str, env: Environment) -> bool: with belief_scope("ConfigManager.update_environment", f"env_id={env_id}"): for index, existing in enumerate(self.config.environments): @@ -527,17 +572,19 @@ class ConfigManager: updated.is_default = True self.config.environments[index] = updated - log.reason("Environment updated", payload={"env_id": env_id}) + logger.reason("Environment updated", extra={"env_id": env_id}) self.save() return True - log.explore("Environment update skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id}) + logger.explore( + "Environment update skipped; env not found", extra={"env_id": env_id} + ) return False - # #endregion update_environment + # [/DEF:update_environment:Function] - # #region delete_environment [TYPE Function] - # @BRIEF Delete environment by id and persist when deletion occurs. + # [DEF:delete_environment:Function] + # @PURPOSE: Delete environment by id and persist when deletion occurs. def delete_environment(self, env_id: str) -> bool: with belief_scope("ConfigManager.delete_environment", f"env_id={env_id}"): before = len(self.config.environments) @@ -547,7 +594,10 @@ class ConfigManager: ] if len(self.config.environments) == before: - log.explore("Environment delete skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id}) + logger.explore( + "Environment delete skipped; env not found", + extra={"env_id": env_id}, + ) return False if removed and removed[0].is_default and self.config.environments: @@ -559,14 +609,14 @@ class ConfigManager: ) self.config.settings.default_environment_id = replacement - log.reason( + logger.reason( "Environment deleted", - payload={"env_id": env_id, "remaining": len(self.config.environments)}, + extra={"env_id": env_id, "remaining": len(self.config.environments)}, ) self.save() return True - # #endregion delete_environment + # [/DEF:delete_environment:Function] # #endregion ConfigManager diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 32b7c9f7..bd39467a 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -1,6 +1,6 @@ # #region ConfigModels [C:3] [TYPE Module] [SEMANTICS config, models, pydantic] # @BRIEF Defines the data models for application configuration using Pydantic. -# @LAYER Core +# @LAYER: Core # @RELATION IMPLEMENTS -> [CoreContracts] # @RELATION IMPLEMENTS -> [ConnectionContracts] @@ -71,7 +71,7 @@ class CleanReleaseConfig(BaseModel): # #region FeaturesConfig [C:1] [TYPE DataClass] # @BRIEF Top-level feature flags that toggle entire project features on/off. -# @RATIONALE Features are read from environment variables on bootstrap and persisted in DB. +# @RATIONALE: Features are read from environment variables on bootstrap and persisted in DB. # DB is source of truth after initial bootstrap; env vars only seed defaults. class FeaturesConfig(BaseModel): dataset_review: bool = True diff --git a/backend/src/core/cot_logger.py b/backend/src/core/cot_logger.py index f168075d..3eb438c2 100644 --- a/backend/src/core/cot_logger.py +++ b/backend/src/core/cot_logger.py @@ -3,8 +3,8 @@ # Uses ContextVar for trace_id and span_id propagation across async contexts. # Provides log(), MarkerLogger, seed_trace_id(), push_span(), pop_span(). # @LAYER: Core -# @RELATION: [CALLED_BY] -> [TraceContextMiddleware] -# @RELATION: [CALLED_BY] -> [All C4+ service and route modules] +# @RELATION CALLED_BY -> [TraceContextMiddleware] +# @RELATION CALLED_BY -> [All C4+ service and route modules] # @PRE: Python 3.7+ (ContextVar available). # @POST: JSON log records written to the 'cot' logger at appropriate levels. # @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger. diff --git a/backend/src/core/database.py b/backend/src/core/database.py index fc6d5aad..cab64619 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -1,14 +1,13 @@ # #region DatabaseModule [C:3] [TYPE Module] [SEMANTICS database, postgresql, sqlalchemy, session, persistence] +# # @BRIEF Configures database connection and session management (PostgreSQL-first). -# @LAYER Core -# @INVARIANT A single engine instance is used for the entire application. +# @LAYER: Core # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [auth_config] # @RELATION DEPENDS_ON -> [ConnectionConfig] # -# +# @INVARIANT: A single engine instance is used for the entire application. -# [SECTION: IMPORTS] from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker from ..models.mapping import Base @@ -28,7 +27,6 @@ from .logger import belief_scope, logger from .auth.config import auth_config import os from pathlib import Path -# [/SECTION] # #region BASE_DIR [C:1] [TYPE Variable] # @BRIEF Base directory for the backend. @@ -58,7 +56,7 @@ AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL # #region engine [C:1] [TYPE Variable] # @BRIEF SQLAlchemy engine for mappings database. -# @SIDE_EFFECT Creates database engine and manages connection pool. +# @SIDE_EFFECT: Creates database engine and manages connection pool. def _build_engine(db_url: str): with belief_scope("_build_engine"): if db_url.startswith("sqlite"): @@ -81,27 +79,27 @@ auth_engine = _build_engine(AUTH_DATABASE_URL) # #region SessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the main mappings database. -# @PRE engine is initialized. +# @PRE: engine is initialized. SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) # #endregion SessionLocal # #region TasksSessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the tasks execution database. -# @PRE tasks_engine is initialized. +# @PRE: tasks_engine is initialized. TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine) # #endregion TasksSessionLocal # #region AuthSessionLocal [C:1] [TYPE Class] # @BRIEF A session factory for the authentication database. -# @PRE auth_engine is initialized. +# @PRE: auth_engine is initialized. AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine) # #endregion AuthSessionLocal # #region _ensure_user_dashboard_preferences_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for user_dashboard_preferences table. -# @PRE bind_engine points to application database where profile table is stored. -# @POST Missing columns are added without data loss. +# @PRE: bind_engine points to application database where profile table is stored. +# @POST: Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_user_dashboard_preferences_columns(bind_engine): with belief_scope("_ensure_user_dashboard_preferences_columns"): @@ -257,8 +255,8 @@ def _ensure_llm_validation_results_columns(bind_engine): # #region _ensure_git_server_configs_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for git_server_configs table. -# @PRE bind_engine points to application database. -# @POST Missing columns are added without data loss. +# @PRE: bind_engine points to application database. +# @POST: Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_git_server_configs_columns(bind_engine): with belief_scope("_ensure_git_server_configs_columns"): @@ -297,8 +295,8 @@ def _ensure_git_server_configs_columns(bind_engine): # #region _ensure_auth_users_columns [C:3] [TYPE Function] # @BRIEF Applies additive schema upgrades for auth users table. -# @PRE bind_engine points to authentication database. -# @POST Missing columns are added without data loss. +# @PRE: bind_engine points to authentication database. +# @POST: Missing columns are added without data loss. # @RELATION DEPENDS_ON -> [auth_engine] def _ensure_auth_users_columns(bind_engine): with belief_scope("_ensure_auth_users_columns"): @@ -359,8 +357,8 @@ def _ensure_auth_users_columns(bind_engine): # #region ensure_connection_configs_table [C:3] [TYPE Function] # @BRIEF Ensures the external connection registry table exists in the main database. -# @PRE bind_engine points to the application database. -# @POST connection_configs table exists without dropping existing data. +# @PRE: bind_engine points to the application database. +# @POST: connection_configs table exists without dropping existing data. # @RELATION DEPENDS_ON -> [ConnectionConfig] def ensure_connection_configs_table(bind_engine): with belief_scope("ensure_connection_configs_table"): @@ -379,8 +377,8 @@ def ensure_connection_configs_table(bind_engine): # #region _ensure_filter_source_enum_values [C:3] [TYPE Function] # @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type. -# @PRE bind_engine points to application database with imported_filters table. -# @POST New enum values are available without data loss. +# @PRE: bind_engine points to application database with imported_filters table. +# @POST: New enum values are available without data loss. # @RELATION DEPENDS_ON -> [engine] def _ensure_filter_source_enum_values(bind_engine): with belief_scope("_ensure_filter_source_enum_values"): @@ -450,9 +448,9 @@ def _ensure_filter_source_enum_values(bind_engine): # #region _ensure_dataset_review_session_columns [C:4] [TYPE Function] # @BRIEF Apply additive schema upgrades for dataset review persistence required by optimistic-lock and recovery metadata semantics. -# @PRE bind_engine points to the application database where dataset review tables are stored. -# @POST Missing additive columns across legacy dataset review tables are created without removing existing data. -# @SIDE_EFFECT Executes ALTER TABLE statements against dataset review tables in the application database. +# @PRE: bind_engine points to the application database where dataset review tables are stored. +# @POST: Missing additive columns across legacy dataset review tables are created without removing existing data. +# @SIDE_EFFECT: Executes ALTER TABLE statements against dataset review tables in the application database. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [ImportedFilter] def _ensure_translation_jobs_columns(bind_engine): @@ -486,62 +484,6 @@ def _ensure_translation_jobs_columns(bind_engine): ) raise - if "target_database_id" not in existing_columns: - try: - with bind_engine.begin() as connection: - connection.execute( - text( - "ALTER TABLE translation_jobs " - "ADD COLUMN target_database_id VARCHAR" - ) - ) - logger.reflect( - "Added target_database_id column to translation_jobs", - ) - except Exception as migration_error: - logger.explore( - "Failed to add target_database_id to translation_jobs", - extra={"error": str(migration_error)}, - ) - raise - - -# #region _ensure_translation_records_columns [TYPE Function] -# @BRIEF Add source_data JSON column to translation_records and translation_preview_records. -# @PRE bind_engine points to the application database. -# @POST source_data column exists on both tables. -# @SIDE_EFFECT Executes ALTER TABLE statements. -def _ensure_translation_records_columns(bind_engine): - with belief_scope("_ensure_translation_records_columns"): - migrations = [ - ("translation_records", "source_data", - "ALTER TABLE translation_records ADD COLUMN source_data JSON"), - ("translation_preview_records", "source_data", - "ALTER TABLE translation_preview_records ADD COLUMN source_data JSON"), - ] - for table_name, column_name, alter_sql in migrations: - inspector = inspect(bind_engine) - if table_name not in inspector.get_table_names(): - continue - existing_columns = { - str(c.get("name") or "").strip() - for c in inspector.get_columns(table_name) - } - if column_name not in existing_columns: - try: - with bind_engine.begin() as connection: - connection.execute(text(alter_sql)) - logger.reflect( - f"Added {column_name} column to {table_name}", - ) - except Exception as migration_error: - logger.explore( - f"Failed to add {column_name} to {table_name}", - extra={"error": str(migration_error)}, - ) - raise -# #endregion _ensure_translation_records_columns - def _ensure_dataset_review_session_columns(bind_engine): with belief_scope("_ensure_dataset_review_session_columns"): @@ -621,9 +563,9 @@ def _ensure_dataset_review_session_columns(bind_engine): # #region init_db [C:3] [TYPE Function] # @BRIEF Initializes the database by creating all tables. -# @PRE engine, tasks_engine and auth_engine are initialized. -# @POST Database tables created in all databases. -# @SIDE_EFFECT Creates physical database files if they don't exist. +# @PRE: engine, tasks_engine and auth_engine are initialized. +# @POST: Database tables created in all databases. +# @SIDE_EFFECT: Creates physical database files if they don't exist. # @RELATION CALLS -> [ensure_connection_configs_table] # @RELATION CALLS -> [_ensure_filter_source_enum_values] # @RELATION CALLS -> [_ensure_dataset_review_session_columns] @@ -641,7 +583,6 @@ def init_db(): _ensure_filter_source_enum_values(engine) _ensure_dataset_review_session_columns(engine) _ensure_translation_jobs_columns(engine) - _ensure_translation_records_columns(engine) # #endregion init_db @@ -649,9 +590,8 @@ def init_db(): # #region get_db [C:3] [TYPE Function] # @BRIEF Dependency for getting a database session. -# @PRE SessionLocal is initialized. -# @POST Session is closed after use. -# @RETURN Generator[Session, None, None] +# @PRE: SessionLocal is initialized. +# @POST: Session is closed after use. # @RELATION DEPENDS_ON -> [SessionLocal] def get_db(): with belief_scope("get_db"): @@ -667,9 +607,8 @@ def get_db(): # #region get_tasks_db [C:3] [TYPE Function] # @BRIEF Dependency for getting a tasks database session. -# @PRE TasksSessionLocal is initialized. -# @POST Session is closed after use. -# @RETURN Generator[Session, None, None] +# @PRE: TasksSessionLocal is initialized. +# @POST: Session is closed after use. # @RELATION DEPENDS_ON -> [TasksSessionLocal] def get_tasks_db(): with belief_scope("get_tasks_db"): @@ -685,10 +624,9 @@ def get_tasks_db(): # #region get_auth_db [C:3] [TYPE Function] # @BRIEF Dependency for getting an authentication database session. -# @PRE AuthSessionLocal is initialized. -# @POST Session is closed after use. -# @DATA_CONTRACT None -> Output[sqlalchemy.orm.Session] -# @RETURN Generator[Session, None, None] +# @PRE: AuthSessionLocal is initialized. +# @POST: Session is closed after use. +# @DATA_CONTRACT: None -> Output[sqlalchemy.orm.Session] # @RELATION DEPENDS_ON -> [AuthSessionLocal] def get_auth_db(): with belief_scope("get_auth_db"): diff --git a/backend/src/core/encryption_key.py b/backend/src/core/encryption_key.py index b82b320d..f029f755 100644 --- a/backend/src/core/encryption_key.py +++ b/backend/src/core/encryption_key.py @@ -1,12 +1,12 @@ # #region EncryptionKeyModule [C:5] [TYPE Module] [SEMANTICS encryption, key, bootstrap, environment, startup] # @BRIEF Resolve and persist the Fernet encryption key required by runtime services. -# @LAYER Infra -# @PRE Runtime environment can read process variables and target .env path is writable when key generation is required. -# @POST A valid Fernet key is available to runtime services via ENCRYPTION_KEY. -# @SIDE_EFFECT May append ENCRYPTION_KEY entry into backend .env file and set process environment variable. -# @DATA_CONTRACT Input[env_file_path] -> Output[encryption_key] -# @INVARIANT Runtime key resolution never falls back to an ephemeral secret. +# @LAYER: Infra # @RELATION DEPENDS_ON -> [LoggerModule] +# @INVARIANT: Runtime key resolution never falls back to an ephemeral secret. +# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required. +# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY. +# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable. +# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key] from __future__ import annotations @@ -22,9 +22,9 @@ DEFAULT_ENV_FILE_PATH = Path(__file__).resolve().parents[2] / ".env" # #region ensure_encryption_key [TYPE Function] # @BRIEF Ensure backend runtime has a persistent valid Fernet key. -# @PRE env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment. -# @POST Returns a valid Fernet key and guarantees it is present in process environment. -# @SIDE_EFFECT May create or append backend/.env when key is missing. +# @PRE: env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment. +# @POST: Returns a valid Fernet key and guarantees it is present in process environment. +# @SIDE_EFFECT: May create or append backend/.env when key is missing. def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str: with belief_scope("ensure_encryption_key", f"env_file_path={env_file_path}"): existing_key = os.getenv("ENCRYPTION_KEY", "").strip() diff --git a/backend/src/core/logger.py b/backend/src/core/logger.py index b07aed90..c1e1c9a1 100755 --- a/backend/src/core/logger.py +++ b/backend/src/core/logger.py @@ -1,26 +1,16 @@ -# #region LoggerModule [C:5] [TYPE Module] [SEMANTICS logging,websocket,streaming,handler,cot,json] -# @BRIEF Application logging system with CotJsonFormatter producing molecular CoT JSON output. -# @LAYER Core -# @RELATION USED_BY -> [All application modules] -# @RELATION DEPENDS_ON -> [CotLoggerModule] -# @RELATION DEPENDS_ON -> [WebSocketLogHandler] -# @PRE Python 3.7+ with cot_logger ContextVars available. -# @POST All log output is single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id. -# @SIDE_EFFECT Configures global logger handlers and formatters; seeds global _enable_belief_state and _task_log_level; writes JSON to console/files. -# @RATIONALE Replaced BeliefFormatter text-based [Entry]/[Exit]/[Action] logging with CotJsonFormatter JSON output. Old format was not machine-parseable and mixed presentation with intent. CotJsonFormatter produces structured JSON consistent with cot_logger.py MarkerLogger protocol. -# @REJECTED Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers). -# @DATA_CONTRACT All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?} -# @INVARIANT CotJsonFormatter.format() always returns valid single-line JSON string. -import json +# #region LoggerModule [TYPE Module] [SEMANTICS logging, websocket, streaming, handler] +# @BRIEF Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets. +# @LAYER: Core +# @RELATION Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py. import logging import threading -import types -from datetime import datetime, timezone +from datetime import datetime +from typing import Dict, Any, List, Optional +from collections import deque from contextlib import contextmanager from logging.handlers import RotatingFileHandler -from .cot_logger import _trace_id, _span_id -from .ws_log_handler import WebSocketLogHandler, LogEntry # noqa: F401 +from pydantic import BaseModel, Field # Thread-local storage for belief state _belief_state = threading.local() @@ -31,122 +21,73 @@ _enable_belief_state = True # Global task log level filter _task_log_level = "INFO" - -# #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol] -# @BRIEF JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id. -# @RELATION IMPLEMENTS -> [CotJsonFormat] -class CotJsonFormatter(logging.Formatter): - """JSON formatter implementing the molecular CoT logging protocol. - - Reads structured data from the LogRecord's extra attributes (marker, intent, payload, - error, src) set via the ``extra`` kwarg. Falls back to plain message wrapping when no - structured extra is present. - - Output format (single-line JSON):: - - { - "ts": "2026-05-12T10:30:00.123", - "level": "INFO", - "trace_id": "uuid-or-no-trace", - "src": "module.name", - "marker": "REASON|REFLECT|EXPLORE", - "intent": "human-readable intent", - "span_id": "optional-span", - "payload": { ... }, # optional - "error": "..." # optional - } - """ - - # #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record] - # @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields. +# #region BeliefFormatter [TYPE Class] +# @BRIEF Custom logging formatter that adds belief state prefixes to log messages. +class BeliefFormatter(logging.Formatter): + # [DEF:format:Function] + # @PURPOSE: Formats the log record, adding belief state context if available. + # @PRE: record is a logging.LogRecord. + # @POST: Returns formatted string. + # @PARAM: record (logging.LogRecord) - The log record to format. + # @RETURN: str - The formatted log message. + # @SEMANTICS: logging, formatter, context def format(self, record): - # Try to extract structured data from extra kwargs on the record - marker = getattr(record, 'marker', None) - intent = getattr(record, 'intent', None) - payload = getattr(record, 'payload', None) - error = getattr(record, 'error', None) - src = getattr(record, 'src', None) or record.name + anchor_id = getattr(_belief_state, 'anchor_id', None) + if anchor_id: + msg = str(record.msg) + # Supported molecular topology markers + markers = ("[EXPLORE]", "[REASON]", "[REFLECT]", "[COHERENCE:", "[Action]", "[Entry]", "[Exit]") + + # Avoid duplicating anchor or overriding explicit markers + if msg.startswith(f"[{anchor_id}]"): + pass + elif any(msg.startswith(m) for m in markers): + record.msg = f"[{anchor_id}]{msg}" + else: + # Default covalent bond + record.msg = f"[{anchor_id}][Action] {msg}" + + return super().format(record) + # [/DEF:format:Function] +# #endregion BeliefFormatter - # For records that already come as JSON strings (e.g. from cot_logger.log()), - # detect and pass through the message directly as intent. - if not marker: - marker = "REASON" # default for plain messages - if not intent: - intent = record.getMessage() +# Re-using LogEntry from task_manager for consistency +# #region LogEntry [TYPE Class] [SEMANTICS log, entry, record, pydantic] +# @BRIEF A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py. +class LogEntry(BaseModel): + timestamp: datetime = Field(default_factory=datetime.utcnow) + level: str + message: str + context: Optional[Dict[str, Any]] = None - log_obj = { - "ts": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), - "level": record.levelname, - "trace_id": _trace_id.get() or "no-trace", - "src": src, - "marker": marker, - "intent": intent, - } +# #endregion LogEntry - span_id = _span_id.get() - if span_id: - log_obj["span_id"] = span_id - if payload: - log_obj["payload"] = payload - if error: - log_obj["error"] = error - - return json.dumps(log_obj, ensure_ascii=False, default=str) - # #endregion CotJsonFormatter.format - - -# #endregion CotJsonFormatter - - -# #region belief_scope [C:4] [TYPE Function] [SEMANTICS logging,context,belief_state,cot,marker] -# @BRIEF Context manager for structured Belief State logging with CoT markers (REASON on entry, REFLECT on success, EXPLORE on failure). -# @RELATION CALLED_BY -> [All C3+ modules using belief_scope] -# @RELATION BINDS_TO -> [CotJsonFormatter] -# @PRE anchor_id must be provided. -# @POST Thread-local belief state is updated. Entry logged as REASON marker, success coherence as REFLECT marker, failure as EXPLORE marker. -# @SIDE_EFFECT Writes debug-level log records with structured extra data; mutates _belief_state.anchor_id. +# #region belief_scope [TYPE Function] [SEMANTICS logging, context, belief_state] +# @BRIEF Context manager for structured Belief State logging. +# @PRE: anchor_id must be provided. +# @POST: Thread-local belief state is updated and entry/exit logs are generated. @contextmanager def belief_scope(anchor_id: str, message: str = ""): - """Context manager for structured Belief State logging with CoT markers. + # Log Entry if enabled (DEBUG level to reduce noise) + if _enable_belief_state: + entry_msg = f"[{anchor_id}][Entry]" + if message: + entry_msg += f" {message}" + logger.debug(entry_msg) - Emits structured markers via the ``extra`` dict so that CotJsonFormatter - produces proper JSON output. - - Usage:: - - with belief_scope("MyFunction", "Processing request"): - logger.info("Doing work") - """ + # Set thread-local anchor_id old_anchor = getattr(_belief_state, 'anchor_id', None) _belief_state.anchor_id = anchor_id - # Log Entry (REASON marker) — only when belief state logging is enabled - if _enable_belief_state: - entry_msg = message or f"Enter {anchor_id}" - logger.debug(entry_msg, extra={ - "marker": "REASON", - "intent": entry_msg, - "src": anchor_id, - }) - try: yield - # Coherence OK / scope exit — always logged for internal tracking - logger.debug("Coherence OK", extra={ - "marker": "REFLECT", - "intent": "Coherence OK", - "src": anchor_id, - "payload": {"contract_id": anchor_id}, - }) + # Log Coherence OK and Exit (DEBUG level to reduce noise) + logger.debug("[COHERENCE:OK]") + if _enable_belief_state: + logger.debug("[Exit]") except Exception as e: - # Coherence FAILED — always logged for error tracking - logger.debug(f"Coherence FAILED: {e}", extra={ - "marker": "EXPLORE", - "intent": "Coherence FAILED", - "error": str(e), - "src": anchor_id, - "payload": {"contract_id": anchor_id}, - }) + # Log Coherence Failed (DEBUG level to reduce noise) + logger.debug(f"[COHERENCE:FAILED] {str(e)}") raise finally: # Restore old anchor @@ -154,14 +95,10 @@ def belief_scope(anchor_id: str, message: str = ""): # #endregion belief_scope - -# #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot] -# @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags. -# @RELATION CALLS -> [CotJsonFormatter] -# @RELATION CALLS -> [CotLoggerModule] -# @PRE config is a valid LoggingConfig instance. -# @POST Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled. -# @SIDE_EFFECT Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories. +# #region configure_logger [TYPE Function] [SEMANTICS logging, configuration, initialization] +# @BRIEF Configures the logger with the provided logging settings. +# @PRE: config is a valid LoggingConfig instance. +# @POST: Logger level, handlers, belief state flag, and task log level are updated. def configure_logger(config): global _enable_belief_state, _task_log_level _enable_belief_state = config.enable_belief_state @@ -182,162 +119,183 @@ def configure_logger(config): from pathlib import Path log_file = Path(config.file_path) log_file.parent.mkdir(parents=True, exist_ok=True) - + file_handler = RotatingFileHandler( config.file_path, maxBytes=config.max_bytes, backupCount=config.backup_count ) - file_handler.setFormatter(CotJsonFormatter()) + file_handler.setFormatter(BeliefFormatter( + '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' + )) logger.addHandler(file_handler) - # Update existing handlers' formatters to CotJsonFormatter + # Update existing handlers' formatters to BeliefFormatter for handler in logger.handlers: if not isinstance(handler, RotatingFileHandler): - handler.setFormatter(CotJsonFormatter()) - - # Also configure the 'cot' logger for consistent JSON output - from .cot_logger import cot_logger - cot_logger.setLevel(level) - # Remove all existing handlers from the cot logger - for h in list(cot_logger.handlers): - cot_logger.removeHandler(h) - h.close() - # Add a console handler with CotJsonFormatter (if file path set, add file handler too) - cot_console = logging.StreamHandler() - cot_console.setFormatter(CotJsonFormatter()) - cot_logger.addHandler(cot_console) - if config.file_path: - from pathlib import Path - log_file = Path(config.file_path) - log_file.parent.mkdir(parents=True, exist_ok=True) - cot_file = RotatingFileHandler( - config.file_path, - maxBytes=config.max_bytes, - backupCount=config.backup_count - ) - cot_file.setFormatter(CotJsonFormatter()) - cot_logger.addHandler(cot_file) - # Disable propagation so cot messages don't double-emit via root loggers - cot_logger.propagate = False - + handler.setFormatter(BeliefFormatter( + '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' + )) # #endregion configure_logger - -# #region get_task_log_level [C:1] [TYPE Function] [SEMANTICS logging,configuration,getter] +# #region get_task_log_level [TYPE Function] [SEMANTICS logging, configuration, getter] +# @BRIEF Returns the current task log level filter. +# @PRE: None. +# @POST: Returns the task log level string. def get_task_log_level() -> str: """Returns the current task log level filter.""" return _task_log_level - # #endregion get_task_log_level - -# #region should_log_task_level [C:2] [TYPE Function] [SEMANTICS logging,filter,level] -# @BRIEF Checks if a log level should be recorded based on the current task log level threshold. +# #region should_log_task_level [TYPE Function] [SEMANTICS logging, filter, level] +# @BRIEF Checks if a log level should be recorded based on task_log_level setting. +# @PRE: level is a valid log level string. +# @POST: Returns True if level meets or exceeds task_log_level threshold. def should_log_task_level(level: str) -> bool: """Checks if a log level should be recorded based on task_log_level setting.""" level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3} current_level = _task_log_level.upper() check_level = level.upper() - + current_order = level_order.get(current_level, 1) # Default to INFO check_order = level_order.get(check_level, 1) - + return check_order >= current_order - # #endregion should_log_task_level +# #region WebSocketLogHandler [TYPE Class] [SEMANTICS logging, handler, websocket, buffer] +# @BRIEF A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets. +class WebSocketLogHandler(logging.Handler): + """ + A logging handler that stores log records and can be extended to send them + over WebSockets. + """ + # [DEF:__init__:Function] + # @PURPOSE: Initializes the handler with a fixed-capacity buffer. + # @PRE: capacity is an integer. + # @POST: Instance initialized with empty deque. + # @PARAM: capacity (int) - Maximum number of logs to keep in memory. + # @SEMANTICS: logging, initialization, buffer + def __init__(self, capacity: int = 1000): + super().__init__() + self.log_buffer: deque[LogEntry] = deque(maxlen=capacity) + # In a real implementation, you'd have a way to manage active WebSocket connections + # e.g., self.active_connections: Set[WebSocket] = set() + # [/DEF:__init__:Function] -# #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance] -# @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler. + # [DEF:emit:Function] + # @PURPOSE: Captures a log record, formats it, and stores it in the buffer. + # @PRE: record is a logging.LogRecord. + # @POST: Log is added to the log_buffer. + # @PARAM: record (logging.LogRecord) - The log record to emit. + # @SEMANTICS: logging, handler, buffer + def emit(self, record: logging.LogRecord): + try: + log_entry = LogEntry( + level=record.levelname, + message=self.format(record), + context={ + "name": record.name, + "pathname": record.pathname, + "lineno": record.lineno, + "funcName": record.funcName, + "process": record.process, + "thread": record.thread, + } + ) + self.log_buffer.append(log_entry) + # Here you would typically send the log_entry to all active WebSocket connections + # for real-time streaming to the frontend. + # Example: for ws in self.active_connections: await ws.send_json(log_entry.dict()) + except Exception: + self.handleError(record) + # [/DEF:emit:Function] + + # [DEF:get_recent_logs:Function] + # @PURPOSE: Returns a list of recent log entries from the buffer. + # @PRE: None. + # @POST: Returns list of LogEntry objects. + # @RETURN: List[LogEntry] - List of buffered log entries. + # @SEMANTICS: logging, buffer, retrieval + def get_recent_logs(self) -> List[LogEntry]: + """ + Returns a list of recent log entries from the buffer. + """ + return list(self.log_buffer) + # [/DEF:get_recent_logs:Function] + +# #endregion WebSocketLogHandler + +# #region Logger [TYPE Global] [SEMANTICS logger, global, instance] +# @BRIEF The global logger instance for the application, configured with both a console handler and the custom WebSocket handler. logger = logging.getLogger("superset_tools_app") + +# #region believed [TYPE Function] +# @BRIEF A decorator that wraps a function in a belief scope. +# @PRE: anchor_id must be a string. +# @POST: Returns a decorator function. +def believed(anchor_id: str): + # [DEF:decorator:Function] + # @PURPOSE: Internal decorator for belief scope. + # @PRE: func must be a callable. + # @POST: Returns the wrapped function. + def decorator(func): + # [DEF:wrapper:Function] + # @PURPOSE: Internal wrapper that enters belief scope. + # @PRE: None. + # @POST: Executes the function within a belief scope. + def wrapper(*args, **kwargs): + with belief_scope(anchor_id): + return func(*args, **kwargs) + # [/DEF:wrapper:Function] + return wrapper + # [/DEF:decorator:Function] + return decorator +# #endregion believed logger.setLevel(logging.INFO) -# Create CotJsonFormatter -_formatter = CotJsonFormatter() +# Create a formatter +formatter = BeliefFormatter( + '[%(asctime)s][%(levelname)s][%(name)s] %(message)s' +) # Add console handler console_handler = logging.StreamHandler() -console_handler.setFormatter(_formatter) +console_handler.setFormatter(formatter) logger.addHandler(console_handler) # Add WebSocket log handler websocket_log_handler = WebSocketLogHandler() -websocket_log_handler.setFormatter(_formatter) +websocket_log_handler.setFormatter(formatter) logger.addHandler(websocket_log_handler) # Example usage: # logger.info("Application started", extra={"context_key": "context_value"}) # logger.error("An error occurred", exc_info=True) +import types -# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning] -# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter. +# #region explore [TYPE Function] [SEMANTICS log, explore, molecule] +# @BRIEF Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses. def explore(self, msg, *args, **kwargs): - """Log an EXPLORE marker — searching, alternatives, violated assumptions. - - Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. - """ - user_extra = kwargs.pop('extra', {}) - extra = {'marker': 'EXPLORE', 'intent': msg} - extra.update(user_extra) - self.warning(msg, *args, extra=extra, **kwargs) - + self.warning(f"[EXPLORE] {msg}", *args, **kwargs) # #endregion explore - -# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info] -# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter. +# #region reason [TYPE Function] [SEMANTICS log, reason, molecule] +# @BRIEF Logs a REASON message (Covalent bond) for strict deduction and core logic. def reason(self, msg, *args, **kwargs): - """Log a REASON marker — strict deduction, core logic. - - Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. - """ - user_extra = kwargs.pop('extra', {}) - extra = {'marker': 'REASON', 'intent': msg} - extra.update(user_extra) - self.info(msg, *args, extra=extra, **kwargs) - + self.info(f"[REASON] {msg}", *args, **kwargs) # #endregion reason - -# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug] -# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter. +# #region reflect [TYPE Function] [SEMANTICS log, reflect, molecule] +# @BRIEF Logs a REFLECT message (Hydrogen bond) for self-check and structural validation. def reflect(self, msg, *args, **kwargs): - """Log a REFLECT marker — self-check, structural validation. - - Passes structured ``extra`` data so CotJsonFormatter produces proper JSON. - """ - user_extra = kwargs.pop('extra', {}) - extra = {'marker': 'REFLECT', 'intent': msg} - extra.update(user_extra) - self.debug(msg, *args, extra=extra, **kwargs) - + self.debug(f"[REFLECT] {msg}", *args, **kwargs) # #endregion reflect - logger.explore = types.MethodType(explore, logger) logger.reason = types.MethodType(reason, logger) logger.reflect = types.MethodType(reflect, logger) # #endregion Logger - - -# #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief_scope,wrapper] -# @BRIEF Decorator that wraps a function in a belief_scope context manager. -def believed(anchor_id: str): - # #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner] - def decorator(func): - # #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope] - def wrapper(*args, **kwargs): - with belief_scope(anchor_id): - return func(*args, **kwargs) - # #endregion believed.decorator.wrapper - return wrapper - # #endregion believed.decorator - return decorator - -# #endregion believed - - -# #endregion LoggerModule +# #endregion LoggerModule \ No newline at end of file diff --git a/backend/src/core/logger/__tests__/test_logger.py b/backend/src/core/logger/__tests__/test_logger.py index 7554e2fc..c93f9028 100644 --- a/backend/src/core/logger/__tests__/test_logger.py +++ b/backend/src/core/logger/__tests__/test_logger.py @@ -1,7 +1,8 @@ -# #region test_logger [C:3] [TYPE Module] -# @BRIEF Unit tests for logger module — CoT JSON format markers. -# @LAYER Infra -# @RELATION VERIFIES -> [src.core.logger] +# [DEF:test_logger:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for logger module +# @LAYER: Infra +# @RELATION: VERIFIES -> src.core.logger import sys from pathlib import Path @@ -11,14 +12,12 @@ sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src")) import pytest import logging -import json from src.core.logger import ( belief_scope, logger, configure_logger, get_task_log_level, - should_log_task_level, - CotJsonFormatter, + should_log_task_level ) from src.core.config_models import LoggingConfig @@ -44,13 +43,13 @@ def reset_logger_state(): configure_logger(config) -# #region test_belief_scope_logs_reason_reflect_at_debug [TYPE Function] -# @BRIEF Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level. -# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST Logs are verified to contain REASON (entry) and REFLECT (coherence) markers. -# @RELATION BINDS_TO -> [test_logger] -def test_belief_scope_logs_reason_reflect_at_debug(caplog): - """Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.""" +# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level. +# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. +# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level. +def test_belief_scope_logs_entry_action_exit_at_debug(caplog): + """Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -58,43 +57,32 @@ def test_belief_scope_logs_reason_reflect_at_debug(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("TestFunction"): logger.info("Doing something important") - # Check that the records contain the expected markers - reason_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction' - ] - reflect_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction' - ] - info_records = [ - r for r in caplog.records - if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() - ] - - assert len(reason_records) >= 1, "REASON marker not found for TestFunction" - assert len(reflect_records) >= 1, "REFLECT marker not found for TestFunction" - assert len(info_records) >= 1, "INFO log 'Doing something important' not found" + # Check that the logs contain the expected patterns + log_messages = [record.message for record in caplog.records] + assert any("[TestFunction][Entry]" in msg for msg in log_messages), "Entry log not found" + assert any("[TestFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" + assert any("[TestFunction][Exit]" in msg for msg in log_messages), "Exit log not found" + # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) -# #endregion test_belief_scope_logs_reason_reflect_at_debug +# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function] -# #region test_belief_scope_error_handling [TYPE Function] -# @BRIEF Test that belief_scope logs EXPLORE marker on exception. -# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST Logs are verified to contain EXPLORE marker with error info. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_belief_scope_error_handling:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception. +# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. +# @POST: Logs are verified to contain Coherence:Failed tag. def test_belief_scope_error_handling(caplog): - """Test that belief_scope logs EXPLORE marker on exception.""" + """Test that belief_scope logs Coherence:Failed on exception.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -102,37 +90,32 @@ def test_belief_scope_error_handling(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with pytest.raises(ValueError): with belief_scope("FailingFunction"): raise ValueError("Something went wrong") - # Check that an EXPLORE marker was emitted - explore_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'EXPLORE' - and getattr(r, 'src', None) == 'FailingFunction' - ] - - assert len(explore_records) >= 1, "EXPLORE marker not found for FailingFunction" - assert 'Something went wrong' in explore_records[0].getMessage() or \ - 'Something went wrong' in str(getattr(explore_records[0], 'error', '')) + log_messages = [record.message for record in caplog.records] + assert any("[FailingFunction][Entry]" in msg for msg in log_messages), "Entry log not found" + assert any("[FailingFunction][COHERENCE:FAILED]" in msg for msg in log_messages), "Failed coherence log not found" + # Exit should not be logged on failure + # Reset to INFO config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True) configure_logger(config) -# #endregion test_belief_scope_error_handling +# [/DEF:test_belief_scope_error_handling:Function] -# #region test_belief_scope_success_coherence [TYPE Function] -# @BRIEF Test that belief_scope logs REFLECT marker on success. -# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG. -# @POST Logs are verified to contain REFLECT marker. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_belief_scope_success_coherence:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that belief_scope logs Coherence:OK on success. +# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG. +# @POST: Logs are verified to contain Coherence:OK tag. def test_belief_scope_success_coherence(caplog): - """Test that belief_scope logs REFLECT marker on success.""" + """Test that belief_scope logs Coherence:OK on success.""" # Configure logger to DEBUG level config = LoggingConfig( level="DEBUG", @@ -140,77 +123,60 @@ def test_belief_scope_success_coherence(caplog): enable_belief_state=True ) configure_logger(config) - + caplog.set_level("DEBUG") with belief_scope("SuccessFunction"): pass - reflect_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REFLECT' - and getattr(r, 'src', None) == 'SuccessFunction' - ] + log_messages = [record.message for record in caplog.records] - assert len(reflect_records) >= 1, "REFLECT marker not found for SuccessFunction" - assert 'Coherence OK' in reflect_records[0].getMessage() or \ - getattr(reflect_records[0], 'intent', '') == 'Coherence OK' + assert any("[SuccessFunction][COHERENCE:OK]" in msg for msg in log_messages), "Success coherence log not found" + + +# [/DEF:test_belief_scope_success_coherence:Function] -# #endregion test_belief_scope_success_coherence - - -# #region test_belief_scope_reason_not_visible_at_info [TYPE Function] -# @BRIEF Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level. -# @PRE belief_scope is available. caplog fixture is used. -# @POST REASON/REFLECT markers are not captured at INFO level. -# @RELATION BINDS_TO -> [test_logger] -def test_belief_scope_reason_not_visible_at_info(caplog): - """Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.""" +# [DEF:test_belief_scope_not_visible_at_info:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level. +# @PRE: belief_scope is available. caplog fixture is used. +# @POST: Entry/Exit/Coherence logs are not captured at INFO level. +def test_belief_scope_not_visible_at_info(caplog): + """Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.""" caplog.set_level("INFO") with belief_scope("InfoLevelFunction"): logger.info("Doing something important") - # The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO - reason_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction' - ] - reflect_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction' - ] + log_messages = [record.message for record in caplog.records] - assert len(reason_records) == 0, "REASON marker should not be visible at INFO" - assert len(reflect_records) == 0, "REFLECT marker should not be visible at INFO" - - # But the INFO-level message should be visible - info_records = [ - r for r in caplog.records - if r.levelname == 'INFO' and 'Doing something important' in r.getMessage() - ] - assert len(info_records) >= 1, "INFO log 'Doing something important' should be visible" -# #endregion test_belief_scope_reason_not_visible_at_info + # Action log should be visible + assert any("[InfoLevelFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found" + # Entry/Exit/Coherence should NOT be visible at INFO level + assert not any("[InfoLevelFunction][Entry]" in msg for msg in log_messages), "Entry log should not be visible at INFO" + assert not any("[InfoLevelFunction][Exit]" in msg for msg in log_messages), "Exit log should not be visible at INFO" + assert not any("[InfoLevelFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence log should not be visible at INFO" +# [/DEF:test_belief_scope_not_visible_at_info:Function] -# #region test_task_log_level_default [TYPE Function] -# @BRIEF Test that default task log level is INFO. -# @PRE None. -# @POST Default level is INFO. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_task_log_level_default:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that default task log level is INFO. +# @PRE: None. +# @POST: Default level is INFO. def test_task_log_level_default(): """Test that default task log level is INFO (after reset fixture).""" level = get_task_log_level() assert level == "INFO" -# #endregion test_task_log_level_default +# [/DEF:test_task_log_level_default:Function] -# #region test_should_log_task_level [TYPE Function] -# @BRIEF Test that should_log_task_level correctly filters log levels. -# @PRE None. -# @POST Filtering works correctly for all level combinations. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_should_log_task_level:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that should_log_task_level correctly filters log levels. +# @PRE: None. +# @POST: Filtering works correctly for all level combinations. def test_should_log_task_level(): """Test that should_log_task_level correctly filters log levels.""" # Default level is INFO @@ -218,14 +184,14 @@ def test_should_log_task_level(): assert should_log_task_level("WARNING") is True, "WARNING should be logged at INFO threshold" assert should_log_task_level("INFO") is True, "INFO should be logged at INFO threshold" assert should_log_task_level("DEBUG") is False, "DEBUG should NOT be logged at INFO threshold" -# #endregion test_should_log_task_level +# [/DEF:test_should_log_task_level:Function] -# #region test_configure_logger_task_log_level [TYPE Function] -# @BRIEF Test that configure_logger updates task_log_level. -# @PRE LoggingConfig is available. -# @POST task_log_level is updated correctly. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_configure_logger_task_log_level:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that configure_logger updates task_log_level. +# @PRE: LoggingConfig is available. +# @POST: task_log_level is updated correctly. def test_configure_logger_task_log_level(): """Test that configure_logger updates task_log_level.""" config = LoggingConfig( @@ -234,19 +200,19 @@ def test_configure_logger_task_log_level(): enable_belief_state=True ) configure_logger(config) - + assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG" assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold" -# #endregion test_configure_logger_task_log_level +# [/DEF:test_configure_logger_task_log_level:Function] -# #region test_enable_belief_state_flag [TYPE Function] -# @BRIEF Test that enable_belief_state flag controls belief_scope entry logging. -# @PRE LoggingConfig is available. caplog fixture is used. -# @POST REASON entry marker is suppressed when disabled; REFLECT coherence still logged. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_enable_belief_state_flag:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging. +# @PRE: LoggingConfig is available. caplog fixture is used. +# @POST: belief_scope logs are controlled by the flag. def test_enable_belief_state_flag(caplog): - """Test that enable_belief_state flag controls belief_scope REASON entry logging.""" + """Test that enable_belief_state flag controls belief_scope logging.""" # Disable belief state config = LoggingConfig( level="DEBUG", @@ -254,31 +220,25 @@ def test_enable_belief_state_flag(caplog): enable_belief_state=False ) configure_logger(config) - + caplog.set_level("DEBUG") - + with belief_scope("DisabledFunction"): logger.info("Doing something") - - # REASON entry marker should NOT be logged when disabled - reason_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction' - ] - assert len(reason_records) == 0, "REASON entry should not be logged when disabled" - - # REFLECT coherence marker should still be logged (internal tracking) - reflect_records = [ - r for r in caplog.records - if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction' - ] - assert len(reflect_records) >= 1, "REFLECT coherence should still be logged" -# #endregion test_enable_belief_state_flag + + log_messages = [record.message for record in caplog.records] + + # Entry and Exit should NOT be logged when disabled + assert not any("[DisabledFunction][Entry]" in msg for msg in log_messages), "Entry should not be logged when disabled" + assert not any("[DisabledFunction][Exit]" in msg for msg in log_messages), "Exit should not be logged when disabled" + # Coherence:OK should still be logged (internal tracking) + assert any("[DisabledFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence should still be logged" +# [/DEF:test_enable_belief_state_flag:Function] -# #region test_belief_scope_missing_anchor [TYPE Function] -# @BRIEF Test @PRE condition: anchor_id must be provided -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_belief_scope_missing_anchor:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test @PRE condition: anchor_id must be provided def test_belief_scope_missing_anchor(): """Test that belief_scope enforces anchor_id to be provided.""" import pytest @@ -287,17 +247,17 @@ def test_belief_scope_missing_anchor(): # Missing required positional argument 'anchor_id' with belief_scope(): pass -# #endregion test_belief_scope_missing_anchor +# [/DEF:test_belief_scope_missing_anchor:Function] -# #region test_configure_logger_post_conditions [TYPE Function] -# @BRIEF Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated. -# @RELATION BINDS_TO -> [test_logger] +# [DEF:test_configure_logger_post_conditions:Function] +# @RELATION: BINDS_TO -> test_logger +# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated. def test_configure_logger_post_conditions(tmp_path): """Test that configure_logger satisfies all @POST conditions.""" import logging from logging.handlers import RotatingFileHandler from src.core.config_models import LoggingConfig - from src.core.logger import configure_logger, logger, CotJsonFormatter, get_task_log_level + from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level import src.core.logger as logger_module log_file = tmp_path / "test.log" @@ -307,93 +267,25 @@ def test_configure_logger_post_conditions(tmp_path): enable_belief_state=False, file_path=str(log_file) ) - + configure_logger(config) - + # 1. Logger level is updated assert logger.level == logging.WARNING - + # 2. Handlers are updated (file handler removed old ones, added new one) file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)] assert len(file_handlers) == 1 import pathlib assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve() - - # 3. Formatter is set to CotJsonFormatter + + # 3. Formatter is set to BeliefFormatter for handler in logger.handlers: - assert isinstance(handler.formatter, CotJsonFormatter) - + assert isinstance(handler.formatter, BeliefFormatter) + # 4. Global states assert getattr(logger_module, '_enable_belief_state') is False assert get_task_log_level() == "DEBUG" -# #endregion test_configure_logger_post_conditions +# [/DEF:test_configure_logger_post_conditions:Function] -# #region test_cot_json_formatter_output [TYPE Function] -# @BRIEF Test that CotJsonFormatter produces valid JSON with expected fields. -# @RELATION BINDS_TO -> [test_logger] -def test_cot_json_formatter_output(): - """Test that CotJsonFormatter produces valid JSON with expected fields.""" - import json - from datetime import datetime, timezone - - formatter = CotJsonFormatter() - - # Create a LogRecord with structured extra data - record = logging.LogRecord( - name="test.module", - level=logging.INFO, - pathname="/fake/path.py", - lineno=42, - msg="Test action", - args=(), - exc_info=None, - ) - record.marker = "REASON" - record.intent = "Test action" - record.src = "TestModule" - record.payload = {"key": "value"} - - output = formatter.format(record) - parsed = json.loads(output) - - assert parsed["level"] == "INFO" - assert parsed["marker"] == "REASON" - assert parsed["intent"] == "Test action" - assert parsed["src"] == "TestModule" - assert parsed["payload"] == {"key": "value"} - assert "ts" in parsed - assert "trace_id" in parsed -# #endregion test_cot_json_formatter_output - -# #region test_cot_json_formatter_plain_message [TYPE Function] -# @BRIEF Test that CotJsonFormatter wraps plain messages (no extra) with default marker. -# @RELATION BINDS_TO -> [test_logger] -def test_cot_json_formatter_plain_message(): - """Test that CotJsonFormatter wraps plain messages with default marker.""" - import json - - formatter = CotJsonFormatter() - - # Create a LogRecord WITHOUT extra data (plain message) - record = logging.LogRecord( - name="test.module", - level=logging.INFO, - pathname="/fake/path.py", - lineno=42, - msg="Plain info message", - args=(), - exc_info=None, - ) - - output = formatter.format(record) - parsed = json.loads(output) - - assert parsed["level"] == "INFO" - assert parsed["marker"] == "REASON" # default for plain messages - assert parsed["intent"] == "Plain info message" - assert parsed["src"] == "test.module" - assert "ts" in parsed - assert "trace_id" in parsed -# #endregion test_cot_json_formatter_plain_message - -# #endregion test_logger +# [/DEF:test_logger:Module] diff --git a/backend/src/core/mapping_service.py b/backend/src/core/mapping_service.py index 050f3541..a9652eaa 100644 --- a/backend/src/core/mapping_service.py +++ b/backend/src/core/mapping_service.py @@ -1,40 +1,35 @@ # #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS mapping, ids, synchronization, environments, cross-filters] +# # @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID) -# @LAYER Core -# @PRE Database session is valid and Superset client factory returns authenticated clients for requested environments. -# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution. -# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs. -# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None] +# @LAYER: Core # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [LoggerModule] -# +# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments. +# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution. +# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs. +# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None] # @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]} # # @INVARIANT: sync_environment must handle remote API failures gracefully. -# [SECTION: IMPORTS] from typing import Dict, List, Optional from datetime import datetime, timezone from sqlalchemy.orm import Session from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger from src.models.mapping import ResourceMapping, ResourceType -from src.core.cot_logger import MarkerLogger from src.core.logger import logger, belief_scope -log = MarkerLogger("IdMapping") -# [/SECTION] - # #region IdMappingService [C:5] [TYPE Class] # @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs. -# @PRE db_session is an active SQLAlchemy Session bound to mapping tables. -# @POST Service instance provides scheduler control and environment-scoped mapping synchronization APIs. -# @SIDE_EFFECT Instantiates an in-process scheduler and performs database writes during sync cycles. -# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService] -# @INVARIANT self.db remains the authoritative session for all mapping operations. +# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables. +# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs. # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [LoggerModule] +# @INVARIANT: self.db remains the authoritative session for all mapping operations. +# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles. +# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService] # # @TEST_CONTRACT: IdMappingServiceModel -> # { @@ -51,17 +46,17 @@ log = MarkerLogger("IdMapping") # @TEST_EDGE: get_batch_empty_list -> returns empty dict # @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure] class IdMappingService: - # #region __init__ [TYPE Function] - # @BRIEF Initializes the mapping service. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the mapping service. def __init__(self, db_session: Session): self.db = db_session self.scheduler = BackgroundScheduler() self._sync_job = None - # #endregion __init__ + # [/DEF:__init__:Function] - # #region start_scheduler [TYPE Function] - # @BRIEF Starts the background scheduler with a given cron string. + # [DEF:start_scheduler:Function] + # @PURPOSE: Starts the background scheduler with a given cron string. # @PARAM: cron_string (str) - Cron expression for the sync interval. # @PARAM: environments (List[str]) - List of environment IDs to sync. # @PARAM: superset_client_factory - Function to get a client for an environment. @@ -71,7 +66,9 @@ class IdMappingService: with belief_scope("IdMappingService.start_scheduler"): if self._sync_job: self.scheduler.remove_job(self._sync_job.id) - log.reflect("Removed existing sync job.") + logger.info( + "[IdMappingService.start_scheduler][Reflect] Removed existing sync job." + ) def sync_all(): for env_id in environments: @@ -88,14 +85,18 @@ class IdMappingService: if not self.scheduler.running: self.scheduler.start() - log.reason(f"Started background scheduler with cron: {cron_string}") + logger.info( + f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}" + ) else: - log.reason(f"Updated background scheduler with cron: {cron_string}") + logger.info( + f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}" + ) - # #endregion start_scheduler + # [/DEF:start_scheduler:Function] - # #region sync_environment [TYPE Function] - # @BRIEF Fully synchronizes mapping for a specific environment. + # [DEF:sync_environment:Function] + # @PURPOSE: Fully synchronizes mapping for a specific environment. # @PARAM: environment_id (str) - Target environment ID. # @PARAM: superset_client - Instance capable of hitting the Superset API. # @PRE: environment_id exists in the database. @@ -108,7 +109,9 @@ class IdMappingService: If incremental=True, only fetches items changed since the max last_synced_at date. """ with belief_scope("IdMappingService.sync_environment"): - log.reason(f"Starting sync for environment {environment_id} (incremental={incremental})") + logger.info( + f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})" + ) # Implementation Note: In a real scenario, superset_client needs to be an instance # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/ @@ -128,7 +131,9 @@ class IdMappingService: total_deleted = 0 try: for res_enum, endpoint, name_field in types_to_poll: - log.reason(f"Polling {endpoint} endpoint") + logger.debug( + f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint" + ) # Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... ) # This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] } @@ -153,8 +158,8 @@ class IdMappingService: from datetime import timedelta since_dttm = max_date - timedelta(minutes=5) - log.reason( - f"Incremental sync since {since_dttm}" + logger.debug( + f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}" ) resources = superset_client.get_all_resources( @@ -218,24 +223,32 @@ class IdMappingService: deleted = stale_query.delete(synchronize_session="fetch") if deleted: total_deleted += deleted - log.reason(f"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}") + logger.info( + f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}" + ) except Exception as loop_e: - log.explore(f"Error polling {endpoint}", error=str(loop_e)) + logger.error( + f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}" + ) # Continue to next resource type instead of blowing up the whole sync self.db.commit() - log.reflect(f"Successfully synced {total_synced} items and deleted {total_deleted} stale items.") + logger.info( + f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items." + ) except Exception as e: self.db.rollback() - log.explore("Critical sync failure", error=str(e)) + logger.error( + f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}" + ) raise - # #endregion sync_environment + # [/DEF:sync_environment:Function] - # #region get_remote_id [TYPE Function] - # @BRIEF Retrieves the remote integer ID for a given universal UUID. + # [DEF:get_remote_id:Function] + # @PURPOSE: Retrieves the remote integer ID for a given universal UUID. # @PARAM: environment_id (str) # @PARAM: resource_type (ResourceType) # @PARAM: uuid (str) @@ -258,10 +271,10 @@ class IdMappingService: return None return None - # #endregion get_remote_id + # [/DEF:get_remote_id:Function] - # #region get_remote_ids_batch [TYPE Function] - # @BRIEF Retrieves remote integer IDs for a list of universal UUIDs efficiently. + # [DEF:get_remote_ids_batch:Function] + # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently. # @PARAM: environment_id (str) # @PARAM: resource_type (ResourceType) # @PARAM: uuids (List[str]) @@ -291,7 +304,7 @@ class IdMappingService: return result - # #endregion get_remote_ids_batch + # [/DEF:get_remote_ids_batch:Function] # #endregion IdMappingService diff --git a/backend/src/core/middleware/trace.py b/backend/src/core/middleware/trace.py index 369f47c7..d327d024 100644 --- a/backend/src/core/middleware/trace.py +++ b/backend/src/core/middleware/trace.py @@ -2,8 +2,8 @@ # @BRIEF FastAPI/Starlette middleware that seeds a trace_id for every incoming HTTP request. # Optionally extracts X-Trace-ID header for cross-service trace propagation. # @LAYER: Core -# @RELATION: [DEPENDS_ON] -> [CotLoggerModule] -# @RELATION: [CALLED_BY] -> [AppModule] +# @RELATION DEPENDS_ON -> [CotLoggerModule] +# @RELATION CALLED_BY -> [AppModule] # @PRE: FastAPI app instance with Starlette middleware support. # @POST: Every request gets a trace_id via seed_trace_id(). Existing X-Trace-ID header is # preserved and used as the trace_id when present. @@ -68,3 +68,4 @@ class TraceContextMiddleware(BaseHTTPMiddleware): # #endregion TraceContextMiddleware +# #endregion TraceContextMiddlewareModule diff --git a/backend/src/core/migration/archive_parser.py b/backend/src/core/migration/archive_parser.py index 98dcb845..c1860f82 100644 --- a/backend/src/core/migration/archive_parser.py +++ b/backend/src/core/migration/archive_parser.py @@ -1,8 +1,8 @@ # #region MigrationArchiveParserModule [C:3] [TYPE Module] [SEMANTICS migration, zip, parser, yaml, metadata] # @BRIEF Parse Superset export ZIP archives into normalized object catalogs for diffing. -# @LAYER Core -# @INVARIANT Parsing is read-only and never mutates archive files. +# @LAYER: Core # @RELATION DEPENDS_ON -> [LoggerModule] +# @INVARIANT: Parsing is read-only and never mutates archive files. import json import tempfile @@ -21,12 +21,12 @@ from ..logger import logger, belief_scope # @RELATION CONTAINS -> [_collect_yaml_objects] # @RELATION CONTAINS -> [_normalize_object_payload] class MigrationArchiveParser: - # #region extract_objects_from_zip [TYPE Function] - # @BRIEF Extract object catalogs from Superset archive. - # @PRE zip_path points to a valid readable ZIP. - # @POST Returns object lists grouped by resource type. - # @RETURN Dict[str, List[Dict[str, Any]]] - # @RELATION DEPENDS_ON -> [_collect_yaml_objects] + # [DEF:extract_objects_from_zip:Function] + # @PURPOSE: Extract object catalogs from Superset archive. + # @RELATION: DEPENDS_ON -> [_collect_yaml_objects] + # @PRE: zip_path points to a valid readable ZIP. + # @POST: Returns object lists grouped by resource type. + # @RETURN: Dict[str, List[Dict[str, Any]]] def extract_objects_from_zip( self, zip_path: str ) -> Dict[str, List[Dict[str, Any]]]: @@ -49,13 +49,13 @@ class MigrationArchiveParser: return result - # #endregion extract_objects_from_zip + # [/DEF:extract_objects_from_zip:Function] - # #region _collect_yaml_objects [TYPE Function] - # @BRIEF Read and normalize YAML manifests for one object type. - # @PRE object_type is one of dashboards/charts/datasets. - # @POST Returns only valid normalized objects. - # @RELATION DEPENDS_ON -> [_normalize_object_payload] + # [DEF:_collect_yaml_objects:Function] + # @PURPOSE: Read and normalize YAML manifests for one object type. + # @RELATION: DEPENDS_ON -> [_normalize_object_payload] + # @PRE: object_type is one of dashboards/charts/datasets. + # @POST: Returns only valid normalized objects. def _collect_yaml_objects( self, root_dir: Path, object_type: str ) -> List[Dict[str, Any]]: @@ -79,12 +79,12 @@ class MigrationArchiveParser: ) return objects - # #endregion _collect_yaml_objects + # [/DEF:_collect_yaml_objects:Function] - # #region _normalize_object_payload [TYPE Function] - # @BRIEF Convert raw YAML payload to stable diff signature shape. - # @PRE payload is parsed YAML mapping. - # @POST Returns normalized descriptor with `uuid`, `title`, and `signature`. + # [DEF:_normalize_object_payload:Function] + # @PURPOSE: Convert raw YAML payload to stable diff signature shape. + # @PRE: payload is parsed YAML mapping. + # @POST: Returns normalized descriptor with `uuid`, `title`, and `signature`. def _normalize_object_payload( self, payload: Dict[str, Any], object_type: str ) -> Optional[Dict[str, Any]]: @@ -149,7 +149,7 @@ class MigrationArchiveParser: return None - # #endregion _normalize_object_payload + # [/DEF:_normalize_object_payload:Function] # #endregion MigrationArchiveParser diff --git a/backend/src/core/migration/dry_run_orchestrator.py b/backend/src/core/migration/dry_run_orchestrator.py index 10cc67cc..6d237d40 100644 --- a/backend/src/core/migration/dry_run_orchestrator.py +++ b/backend/src/core/migration/dry_run_orchestrator.py @@ -1,11 +1,11 @@ # #region MigrationDryRunOrchestratorModule [C:3] [TYPE Module] [SEMANTICS migration, dry_run, diff, risk, superset] # @BRIEF Compute pre-flight migration diff and risk scoring without apply. -# @LAYER Core -# @INVARIANT Dry run is informative only and must not mutate target environment. +# @LAYER: Core # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [MigrationEngine] # @RELATION DEPENDS_ON -> [MigrationArchiveParser] # @RELATION DEPENDS_ON -> [RiskAssessorModule] +# @INVARIANT: Dry run is informative only and must not mutate target environment. from datetime import datetime, timezone import json @@ -34,25 +34,25 @@ from ..utils.fileio import create_temp_file # @RELATION CONTAINS -> [_build_target_signatures] # @RELATION CONTAINS -> [_build_risks] class MigrationDryRunService: - # #region __init__ [TYPE Function] - # @BRIEF Wire parser dependency for archive object extraction. - # @PRE parser can be omitted to use default implementation. - # @POST Service is ready to calculate dry-run payload. + # [DEF:__init__:Function] + # @PURPOSE: Wire parser dependency for archive object extraction. + # @PRE: parser can be omitted to use default implementation. + # @POST: Service is ready to calculate dry-run payload. def __init__(self, parser: MigrationArchiveParser | None = None): self.parser = parser or MigrationArchiveParser() - # #endregion __init__ + # [/DEF:__init__:Function] - # #region run [TYPE Function] - # @BRIEF Execute full dry-run computation for selected dashboards. - # @PRE source/target clients are authenticated and selection validated by caller. - # @POST Returns JSON-serializable pre-flight payload with summary, diff and risk. - # @SIDE_EFFECT Reads source export archives and target metadata via network. - # @RELATION DEPENDS_ON -> [_load_db_mapping] - # @RELATION DEPENDS_ON -> [_accumulate_objects] - # @RELATION DEPENDS_ON -> [_build_target_signatures] - # @RELATION DEPENDS_ON -> [_build_object_diff] - # @RELATION DEPENDS_ON -> [_build_risks] + # [DEF:run:Function] + # @PURPOSE: Execute full dry-run computation for selected dashboards. + # @RELATION: DEPENDS_ON -> [_load_db_mapping] + # @RELATION: DEPENDS_ON -> [_accumulate_objects] + # @RELATION: DEPENDS_ON -> [_build_target_signatures] + # @RELATION: DEPENDS_ON -> [_build_object_diff] + # @RELATION: DEPENDS_ON -> [_build_risks] + # @PRE: source/target clients are authenticated and selection validated by caller. + # @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk. + # @SIDE_EFFECT: Reads source export archives and target metadata via network. def run( self, selection: DashboardSelection, @@ -154,10 +154,10 @@ class MigrationDryRunService: "risk": score_risks(risk), } - # #endregion run + # [/DEF:run:Function] - # #region _load_db_mapping [TYPE Function] - # @BRIEF Resolve UUID mapping for optional DB config replacement. + # [DEF:_load_db_mapping:Function] + # @PURPOSE: Resolve UUID mapping for optional DB config replacement. def _load_db_mapping( self, db: Session, selection: DashboardSelection ) -> Dict[str, str]: @@ -171,10 +171,10 @@ class MigrationDryRunService: ) return {row.source_db_uuid: row.target_db_uuid for row in rows} - # #endregion _load_db_mapping + # [/DEF:_load_db_mapping:Function] - # #region _accumulate_objects [TYPE Function] - # @BRIEF Merge extracted resources by UUID to avoid duplicates. + # [DEF:_accumulate_objects:Function] + # @PURPOSE: Merge extracted resources by UUID to avoid duplicates. def _accumulate_objects( self, target: Dict[str, Dict[str, Dict[str, Any]]], @@ -186,10 +186,10 @@ class MigrationDryRunService: if uuid: target[object_type][str(uuid)] = item - # #endregion _accumulate_objects + # [/DEF:_accumulate_objects:Function] - # #region _index_by_uuid [TYPE Function] - # @BRIEF Build UUID-index map for normalized resources. + # [DEF:_index_by_uuid:Function] + # @PURPOSE: Build UUID-index map for normalized resources. def _index_by_uuid( self, objects: List[Dict[str, Any]] ) -> Dict[str, Dict[str, Any]]: @@ -200,11 +200,11 @@ class MigrationDryRunService: indexed[str(uuid)] = obj return indexed - # #endregion _index_by_uuid + # [/DEF:_index_by_uuid:Function] - # #region _build_object_diff [TYPE Function] - # @BRIEF Compute create/update/delete buckets by UUID+signature. - # @RELATION DEPENDS_ON -> [_index_by_uuid] + # [DEF:_build_object_diff:Function] + # @PURPOSE: Compute create/update/delete buckets by UUID+signature. + # @RELATION: DEPENDS_ON -> [_index_by_uuid] def _build_object_diff( self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]] ) -> Dict[str, List[Dict[str, Any]]]: @@ -228,10 +228,10 @@ class MigrationDryRunService: ) return {"create": created, "update": updated, "delete": deleted} - # #endregion _build_object_diff + # [/DEF:_build_object_diff:Function] - # #region _build_target_signatures [TYPE Function] - # @BRIEF Pull target metadata and normalize it into comparable signatures. + # [DEF:_build_target_signatures:Function] + # @PURPOSE: Pull target metadata and normalize it into comparable signatures. def _build_target_signatures( self, client: SupersetClient ) -> Dict[str, List[Dict[str, Any]]]: @@ -341,10 +341,10 @@ class MigrationDryRunService: ], } - # #endregion _build_target_signatures + # [/DEF:_build_target_signatures:Function] - # #region _build_risks [TYPE Function] - # @BRIEF Build risk items for missing datasource, broken refs, overwrite, owner mismatch. + # [DEF:_build_risks:Function] + # @PURPOSE: Build risk items for missing datasource, broken refs, overwrite, owner mismatch. def _build_risks( self, source_objects: Dict[str, List[Dict[str, Any]]], @@ -354,7 +354,7 @@ class MigrationDryRunService: ) -> List[Dict[str, Any]]: return build_risks(source_objects, target_objects, diff, target_client) - # #endregion _build_risks + # [/DEF:_build_risks:Function] # #endregion MigrationDryRunService diff --git a/backend/src/core/migration/risk_assessor.py b/backend/src/core/migration/risk_assessor.py index e1c69ea6..a919c6a2 100644 --- a/backend/src/core/migration/risk_assessor.py +++ b/backend/src/core/migration/risk_assessor.py @@ -1,16 +1,16 @@ # #region RiskAssessorModule [C:5] [TYPE Module] [SEMANTICS migration, dry_run, risk, scoring, preflight] # @BRIEF Compute deterministic migration risk items and aggregate score for dry-run reporting. -# @LAYER Domain -# @PRE Risk assessor functions receive normalized migration object collections from dry-run orchestration. -# @POST Risk scoring output preserves item list and provides bounded score with derived level. -# @SIDE_EFFECT Emits diagnostic logs and performs read-only metadata requests via Superset client. -# @DATA_CONTRACT Module[build_risks, score_risks] -# @INVARIANT Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION CONTAINS -> [index_by_uuid] # @RELATION CONTAINS -> [extract_owner_identifiers] # @RELATION CONTAINS -> [build_risks] # @RELATION CONTAINS -> [score_risks] +# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping. +# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration. +# @POST: Risk scoring output preserves item list and provides bounded score with derived level. +# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client. +# @DATA_CONTRACT: Module[build_risks, score_risks] # @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]] # @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item] # @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted] @@ -26,28 +26,25 @@ from typing import Any, Dict, List -from ..logger import belief_scope -from ..cot_logger import MarkerLogger +from ..logger import logger, belief_scope from ..superset_client import SupersetClient -log = MarkerLogger("RiskAssessor") - # #region index_by_uuid [TYPE Function] # @BRIEF Build UUID-index from normalized objects. -# @PRE Input list items are dict-like payloads potentially containing "uuid". -# @POST Returns mapping keyed by string uuid; only truthy uuid values are included. -# @SIDE_EFFECT Emits reasoning/reflective logs only. -# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Dict[str, Any]] +# @PRE: Input list items are dict-like payloads potentially containing "uuid". +# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included. +# @SIDE_EFFECT: Emits reasoning/reflective logs only. +# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]] def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: with belief_scope("risk_assessor.index_by_uuid"): - log.reason("Building UUID index", payload={"objects_count": len(objects)}) + logger.reason("Building UUID index", extra={"objects_count": len(objects)}) indexed: Dict[str, Dict[str, Any]] = {} for obj in objects: uuid = obj.get("uuid") if uuid: indexed[str(uuid)] = obj - log.reflect("UUID index built", payload={"indexed_count": len(indexed)}) + logger.reflect("UUID index built", extra={"indexed_count": len(indexed)}) return indexed @@ -56,15 +53,15 @@ def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: # #region extract_owner_identifiers [TYPE Function] # @BRIEF Normalize owner payloads for stable comparison. -# @PRE Owners may be list payload, scalar values, or None. -# @POST Returns sorted unique owner identifiers as strings. -# @SIDE_EFFECT Emits reasoning/reflective logs only. -# @DATA_CONTRACT Any -> List[str] +# @PRE: Owners may be list payload, scalar values, or None. +# @POST: Returns sorted unique owner identifiers as strings. +# @SIDE_EFFECT: Emits reasoning/reflective logs only. +# @DATA_CONTRACT: Any -> List[str] def extract_owner_identifiers(owners: Any) -> List[str]: with belief_scope("risk_assessor.extract_owner_identifiers"): - log.reason("Normalizing owner identifiers") + logger.reason("Normalizing owner identifiers") if not isinstance(owners, list): - log.reflect("Owners payload is not list; returning empty identifiers") + logger.reflect("Owners payload is not list; returning empty identifiers") return [] ids: List[str] = [] for owner in owners: @@ -76,8 +73,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]: elif owner is not None: ids.append(str(owner)) normalized_ids = sorted(set(ids)) - log.reflect( - "Owner identifiers normalized", payload={"owner_count": len(normalized_ids)} + logger.reflect( + "Owner identifiers normalized", extra={"owner_count": len(normalized_ids)} ) return normalized_ids @@ -87,18 +84,18 @@ def extract_owner_identifiers(owners: Any) -> List[str]: # #region build_risks [TYPE Function] # @BRIEF Build risk list from computed diffs and target catalog state. -# @PRE source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures. -# @PRE target_client is authenticated/usable for database list retrieval. -# @POST Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks. -# @SIDE_EFFECT Calls target Superset API for databases metadata and emits logs. -# @DATA_CONTRACT ( -# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]], -# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]], -# @DATA_CONTRACT Dict[str, Dict[str, List[Dict[str, Any]]]], -# @DATA_CONTRACT SupersetClient -# @DATA_CONTRACT ) -> List[Dict[str, Any]] # @RELATION DEPENDS_ON -> [index_by_uuid] # @RELATION DEPENDS_ON -> [extract_owner_identifiers] +# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures. +# @PRE: target_client is authenticated/usable for database list retrieval. +# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks. +# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs. +# @DATA_CONTRACT: ( +# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]], +# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]], +# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]], +# @DATA_CONTRACT: SupersetClient +# @DATA_CONTRACT: ) -> List[Dict[str, Any]] def build_risks( source_objects: Dict[str, List[Dict[str, Any]]], target_objects: Dict[str, List[Dict[str, Any]]], @@ -106,7 +103,7 @@ def build_risks( target_client: SupersetClient, ) -> List[Dict[str, Any]]: with belief_scope("risk_assessor.build_risks"): - log.reason("Building migration risks from diff payload") + logger.reason("Building migration risks from diff payload") risks: List[Dict[str, Any]] = [] for object_type in ("dashboards", "charts", "datasets"): for item in diff[object_type]["update"]: @@ -171,7 +168,7 @@ def build_risks( "message": f"Owner mismatch for dashboard {item.get('title') or item['uuid']}", } ) - log.reflect("Risk list assembled", payload={"risk_count": len(risks)}) + logger.reflect("Risk list assembled", extra={"risk_count": len(risks)}) return risks @@ -180,20 +177,20 @@ def build_risks( # #region score_risks [TYPE Function] # @BRIEF Aggregate risk list into score and level. -# @PRE risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight. -# @POST Returns dict with score in [0,100], derived level, and original items. -# @SIDE_EFFECT Emits reasoning/reflective logs only. -# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Any] +# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight. +# @POST: Returns dict with score in [0,100], derived level, and original items. +# @SIDE_EFFECT: Emits reasoning/reflective logs only. +# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any] def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]: with belief_scope("risk_assessor.score_risks"): - log.reason("Scoring risk items", payload={"risk_items_count": len(risk_items)}) + logger.reason("Scoring risk items", extra={"risk_items_count": len(risk_items)}) weights = {"high": 25, "medium": 10, "low": 5} score = min( 100, sum(weights.get(item.get("severity", "low"), 5) for item in risk_items) ) level = "low" if score < 25 else "medium" if score < 60 else "high" result = {"score": score, "level": level, "items": risk_items} - log.reflect("Risk score computed", payload={"score": score, "level": level}) + logger.reflect("Risk score computed", extra={"score": score, "level": level}) return result diff --git a/backend/src/core/migration_engine.py b/backend/src/core/migration_engine.py index d2024dda..fb3af0c7 100644 --- a/backend/src/core/migration_engine.py +++ b/backend/src/core/migration_engine.py @@ -1,18 +1,17 @@ # #region MigrationEngineModule [C:5] [TYPE Module] [SEMANTICS migration, engine, zip, yaml, transformation, cross-filter, id-mapping] +# # @BRIEF Transforms Superset export ZIP archives while preserving archive integrity and patching mapped identifiers. -# @LAYER Domain -# @PRE Input archives are readable Superset exports and optional mapping collaborators expose remote id lookup APIs. -# @POST Migration engine contracts preserve ZIP integrity while exposing transformation entrypoints for import pipelines. -# @SIDE_EFFECT Reads and writes temporary archive contents during transformation workflows and emits structured belief-state logs. -# @DATA_CONTRACT Input[zip_path, output_path, db_mapping, target_env_id?, fix_cross_filters?] -> Output[Transformed Superset archive] -# @INVARIANT ZIP structure and non-targeted metadata must remain valid after transformation. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [LoggerModule] # @RELATION DEPENDS_ON -> [IdMappingService] # @RELATION DEPENDS_ON -> [ResourceType] # @RELATION DEPENDS_ON -> [yaml] -# +# @PRE: Input archives are readable Superset exports and optional mapping collaborators expose remote id lookup APIs. +# @POST: Migration engine contracts preserve ZIP integrity while exposing transformation entrypoints for import pipelines. +# @SIDE_EFFECT: Reads and writes temporary archive contents during transformation workflows and emits structured belief-state logs. +# @DATA_CONTRACT: Input[zip_path, output_path, db_mapping, target_env_id?, fix_cross_filters?] -> Output[Transformed Superset archive] +# @INVARIANT: ZIP structure and non-targeted metadata must remain valid after transformation. -# [SECTION: IMPORTS] import zipfile import yaml import os @@ -24,19 +23,18 @@ from typing import Dict, Optional, List from .logger import logger, belief_scope from src.core.mapping_service import IdMappingService from src.models.mapping import ResourceType -# [/SECTION] # #region MigrationEngine [TYPE Class] # @BRIEF Engine for transforming Superset export ZIPs. # @RELATION CONTAINS -> [__init__, transform_zip, _transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata] class MigrationEngine: - # #region __init__ [TYPE Function] - # @BRIEF Initializes migration orchestration dependencies for ZIP/YAML metadata transformations. - # @PRE mapping_service is None or implements batch remote ID lookup for ResourceType.CHART. - # @POST self.mapping_service is assigned and available for optional cross-filter patching flows. - # @SIDE_EFFECT Mutates in-memory engine state by storing dependency reference. - # @DATA_CONTRACT Input[Optional[IdMappingService]] -> Output[MigrationEngine] + # [DEF:__init__:Function] + # @PURPOSE: Initializes migration orchestration dependencies for ZIP/YAML metadata transformations. + # @PRE: mapping_service is None or implements batch remote ID lookup for ResourceType.CHART. + # @POST: self.mapping_service is assigned and available for optional cross-filter patching flows. + # @SIDE_EFFECT: Mutates in-memory engine state by storing dependency reference. + # @DATA_CONTRACT: Input[Optional[IdMappingService]] -> Output[MigrationEngine] # @PARAM: mapping_service (Optional[IdMappingService]) - Used for resolving target environment integer IDs. def __init__(self, mapping_service: Optional[IdMappingService] = None): with belief_scope("MigrationEngine.__init__"): @@ -44,11 +42,11 @@ class MigrationEngine: self.mapping_service = mapping_service logger.reflect("MigrationEngine initialized") - # #endregion __init__ + # [/DEF:__init__:Function] - # #region transform_zip [TYPE Function] - # @BRIEF Extracts ZIP, replaces database UUIDs in YAMLs, patches cross-filters, and re-packages. - # @RELATION DEPENDS_ON -> [_transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata] + # [DEF:transform_zip:Function] + # @PURPOSE: Extracts ZIP, replaces database UUIDs in YAMLs, patches cross-filters, and re-packages. + # @RELATION: DEPENDS_ON -> [_transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata] # @PARAM: zip_path (str) - Path to the source ZIP file. # @PARAM: output_path (str) - Path where the transformed ZIP will be saved. # @PARAM: db_mapping (Dict[str, str]) - Mapping of source UUID to target UUID. @@ -144,10 +142,10 @@ class MigrationEngine: logger.explore(f"Error transforming ZIP: {e}") return False - # #endregion transform_zip + # [/DEF:transform_zip:Function] - # #region _transform_yaml [TYPE Function] - # @BRIEF Replaces database_uuid in a single YAML file. + # [DEF:_transform_yaml:Function] + # @PURPOSE: Replaces database_uuid in a single YAML file. # @PARAM: file_path (Path) - Path to the YAML file. # @PARAM: db_mapping (Dict[str, str]) - UUID mapping dictionary. # @PRE: file_path exists, is readable YAML, and db_mapping contains source->target UUID pairs. @@ -174,14 +172,14 @@ class MigrationEngine: yaml.dump(data, f) logger.reflect(f"Database UUID patched in {file_path.name}") - # #endregion _transform_yaml + # [/DEF:_transform_yaml:Function] - # #region _extract_chart_uuids_from_archive [TYPE Function] - # @BRIEF Scans extracted chart YAML files and builds a source chart ID to UUID lookup map. - # @PRE temp_dir exists and points to extracted archive root with optional chart YAML resources. - # @POST Returns a best-effort Dict[int, str] containing only parseable chart id/uuid pairs. - # @SIDE_EFFECT Reads chart YAML files from filesystem; suppresses per-file parsing failures. - # @DATA_CONTRACT Input[Path] -> Output[Dict[int,str]] + # [DEF:_extract_chart_uuids_from_archive:Function] + # @PURPOSE: Scans extracted chart YAML files and builds a source chart ID to UUID lookup map. + # @PRE: temp_dir exists and points to extracted archive root with optional chart YAML resources. + # @POST: Returns a best-effort Dict[int, str] containing only parseable chart id/uuid pairs. + # @SIDE_EFFECT: Reads chart YAML files from filesystem; suppresses per-file parsing failures. + # @DATA_CONTRACT: Input[Path] -> Output[Dict[int,str]] # @PARAM: temp_dir (Path) - Root dir of unpacked archive. # @RETURN: Dict[int, str] - Mapping of source Integer ID to UUID. def _extract_chart_uuids_from_archive(self, temp_dir: Path) -> Dict[int, str]: @@ -204,14 +202,14 @@ class MigrationEngine: pass return mapping - # #endregion _extract_chart_uuids_from_archive + # [/DEF:_extract_chart_uuids_from_archive:Function] - # #region _patch_dashboard_metadata [TYPE Function] - # @BRIEF Rewrites dashboard json_metadata chart/dataset integer identifiers using target environment mappings. - # @PRE file_path points to dashboard YAML with json_metadata; target_env_id is non-empty; source_map contains source id->uuid. - # @POST json_metadata is re-serialized with mapped integer IDs when remote mappings are available; otherwise file remains unchanged. - # @SIDE_EFFECT Reads/writes YAML file, performs mapping lookup via mapping_service, emits logs for recoverable/terminal failures. - # @DATA_CONTRACT Input[(Path file_path, str target_env_id, Dict[int,str] source_map)] -> Output[None] + # [DEF:_patch_dashboard_metadata:Function] + # @PURPOSE: Rewrites dashboard json_metadata chart/dataset integer identifiers using target environment mappings. + # @PRE: file_path points to dashboard YAML with json_metadata; target_env_id is non-empty; source_map contains source id->uuid. + # @POST: json_metadata is re-serialized with mapped integer IDs when remote mappings are available; otherwise file remains unchanged. + # @SIDE_EFFECT: Reads/writes YAML file, performs mapping lookup via mapping_service, emits logs for recoverable/terminal failures. + # @DATA_CONTRACT: Input[(Path file_path, str target_env_id, Dict[int,str] source_map)] -> Output[None] # @PARAM: file_path (Path) # @PARAM: target_env_id (str) # @PARAM: source_map (Dict[int, str]) @@ -295,8 +293,8 @@ class MigrationEngine: except Exception as e: logger.explore(f"Metadata patch failed for {file_path.name}: {e}") - # #endregion _patch_dashboard_metadata + # [/DEF:_patch_dashboard_metadata:Function] # #endregion MigrationEngine -# #endregion MigrationEngineModule +# #endregion MigrationEngineModule \ No newline at end of file diff --git a/backend/src/core/plugin_base.py b/backend/src/core/plugin_base.py index a21fda2c..4b432e69 100755 --- a/backend/src/core/plugin_base.py +++ b/backend/src/core/plugin_base.py @@ -6,9 +6,9 @@ from pydantic import BaseModel, Field # #region PluginBase [TYPE Class] [SEMANTICS plugin, interface, base, abstract] # @BRIEF Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution. -# @LAYER Core -# @INVARIANT All plugins MUST inherit from this class. +# @LAYER: Core # @RELATION Used by PluginLoader to identify valid plugins. +# @INVARIANT: All plugins MUST inherit from this class. class PluginBase(ABC): """ Base class for all plugins. @@ -17,74 +17,74 @@ class PluginBase(ABC): @property @abstractmethod - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the plugin. - # @PRE Plugin instance exists. - # @POST Returns string ID. - # @RETURN str - Plugin ID. + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string ID. + # @RETURN: str - Plugin ID. def id(self) -> str: """A unique identifier for the plugin.""" with belief_scope("id"): pass - # #endregion id + # [/DEF:id:Function] @property @abstractmethod - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the plugin. - # @PRE Plugin instance exists. - # @POST Returns string name. - # @RETURN str - Plugin name. + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string name. + # @RETURN: str - Plugin name. def name(self) -> str: """A human-readable name for the plugin.""" with belief_scope("name"): pass - # #endregion name + # [/DEF:name:Function] @property @abstractmethod - # #region description [TYPE Function] - # @BRIEF Returns a brief description of the plugin. - # @PRE Plugin instance exists. - # @POST Returns string description. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a brief description of the plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string description. + # @RETURN: str - Plugin description. def description(self) -> str: """A brief description of what the plugin does.""" with belief_scope("description"): pass - # #endregion description + # [/DEF:description:Function] @property @abstractmethod - # #region version [TYPE Function] - # @BRIEF Returns the version of the plugin. - # @PRE Plugin instance exists. - # @POST Returns string version. - # @RETURN str - Plugin version. + # [DEF:version:Function] + # @PURPOSE: Returns the version of the plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string version. + # @RETURN: str - Plugin version. def version(self) -> str: """The version of the plugin.""" with belief_scope("version"): pass - # #endregion version + # [/DEF:version:Function] @property - # #region required_permission [TYPE Function] - # @BRIEF Returns the required permission string to execute this plugin. - # @PRE Plugin instance exists. - # @POST Returns string permission. - # @RETURN str - Required permission (e.g., "plugin:backup:execute"). + # [DEF:required_permission:Function] + # @PURPOSE: Returns the required permission string to execute this plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string permission. + # @RETURN: str - Required permission (e.g., "plugin:backup:execute"). def required_permission(self) -> str: """The permission string required to execute this plugin.""" with belief_scope("required_permission"): return f"plugin:{self.id}:execute" - # #endregion required_permission + # [/DEF:required_permission:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the plugin's UI, if applicable. - # @PRE Plugin instance exists. - # @POST Returns string route or None. - # @RETURN Optional[str] - Frontend route. + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the plugin's UI, if applicable. + # @PRE: Plugin instance exists. + # @POST: Returns string route or None. + # @RETURN: Optional[str] - Frontend route. def ui_route(self) -> Optional[str]: """ The frontend route for the plugin's UI. @@ -92,14 +92,14 @@ class PluginBase(ABC): """ with belief_scope("ui_route"): return None - # #endregion ui_route + # [/DEF:ui_route:Function] @abstractmethod - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for the plugin's input parameters. - # @PRE Plugin instance exists. - # @POST Returns dict schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for the plugin's input parameters. + # @PRE: Plugin instance exists. + # @POST: Returns dict schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: """ Returns the JSON schema for the plugin's input parameters. @@ -107,11 +107,11 @@ class PluginBase(ABC): """ with belief_scope("get_schema"): pass - # #endregion get_schema + # [/DEF:get_schema:Function] @abstractmethod - # #region execute [TYPE Function] - # @BRIEF Executes the plugin's core logic. + # [DEF:execute:Function] + # @PURPOSE: Executes the plugin's core logic. # @PARAM: params (Dict[str, Any]) - Validated input parameters. # @PRE: params must be a dictionary. # @POST: Plugin execution is completed. @@ -123,12 +123,12 @@ class PluginBase(ABC): The `params` argument will be validated against the schema returned by `get_schema()`. """ pass - # #endregion execute + # [/DEF:execute:Function] # #endregion PluginBase # #region PluginConfig [TYPE Class] [SEMANTICS plugin, config, schema, pydantic] # @BRIEF A Pydantic model used to represent the validated configuration and metadata of a loaded plugin. This object is what gets exposed to the API layer. -# @LAYER Core +# @LAYER: Core # @RELATION Instantiated by PluginLoader after validating a PluginBase instance. class PluginConfig(BaseModel): """Pydantic model for plugin configuration.""" @@ -138,4 +138,4 @@ class PluginConfig(BaseModel): version: str = Field(..., description="Version of the plugin") ui_route: Optional[str] = Field(None, description="Frontend route for the plugin UI") input_schema: Dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema") -# #endregion PluginConfig +# #endregion PluginConfig \ No newline at end of file diff --git a/backend/src/core/plugin_loader.py b/backend/src/core/plugin_loader.py index bbd9f07a..a6f262ce 100755 --- a/backend/src/core/plugin_loader.py +++ b/backend/src/core/plugin_loader.py @@ -7,7 +7,7 @@ from .logger import belief_scope # #region PluginLoader [C:3] [TYPE Class] [SEMANTICS plugin, loader, dynamic, import] # @BRIEF Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface. -# @LAYER Core +# @LAYER: Core # @RELATION Depends on PluginBase. It is used by the main application to discover and manage available plugins. class PluginLoader: """ @@ -15,10 +15,10 @@ class PluginLoader: that inherit from PluginBase. """ - # #region __init__ [TYPE Function] - # @BRIEF Initializes the PluginLoader with a directory to scan. - # @PRE plugin_dir is a valid directory path. - # @POST Plugins are loaded and registered. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the PluginLoader with a directory to scan. + # @PRE: plugin_dir is a valid directory path. + # @POST: Plugins are loaded and registered. # @PARAM: plugin_dir (str) - The directory containing plugin modules. def __init__(self, plugin_dir: str): with belief_scope("__init__"): @@ -26,12 +26,12 @@ class PluginLoader: self._plugins: Dict[str, PluginBase] = {} self._plugin_configs: Dict[str, PluginConfig] = {} self._load_plugins() - # #endregion __init__ + # [/DEF:__init__:Function] - # #region _load_plugins [TYPE Function] - # @BRIEF Scans the plugin directory and loads all valid plugins. - # @PRE plugin_dir exists or can be created. - # @POST _load_module is called for each .py file. + # [DEF:_load_plugins:Function] + # @PURPOSE: Scans the plugin directory and loads all valid plugins. + # @PRE: plugin_dir exists or can be created. + # @POST: _load_module is called for each .py file. def _load_plugins(self): with belief_scope("_load_plugins"): """ @@ -61,12 +61,12 @@ class PluginLoader: if filename.endswith(".py") and filename != "__init__.py": module_name = filename[:-3] self._load_module(module_name, file_path) - # #endregion _load_plugins + # [/DEF:_load_plugins:Function] - # #region _load_module [TYPE Function] - # @BRIEF Loads a single Python module and discovers PluginBase implementations. - # @PRE module_name and file_path are valid. - # @POST Plugin classes are instantiated and registered. + # [DEF:_load_module:Function] + # @PURPOSE: Loads a single Python module and discovers PluginBase implementations. + # @PRE: module_name and file_path are valid. + # @POST: Plugin classes are instantiated and registered. # @PARAM: module_name (str) - The name of the module. # @PARAM: file_path (str) - The path to the module file. def _load_module(self, module_name: str, file_path: str): @@ -102,12 +102,12 @@ class PluginLoader: self._register_plugin(plugin_instance) except Exception as e: print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging - # #endregion _load_module + # [/DEF:_load_module:Function] - # #region _register_plugin [TYPE Function] - # @BRIEF Registers a PluginBase instance and its configuration. - # @PRE plugin_instance is a valid implementation of PluginBase. - # @POST Plugin is added to _plugins and _plugin_configs. + # [DEF:_register_plugin:Function] + # @PURPOSE: Registers a PluginBase instance and its configuration. + # @PRE: plugin_instance is a valid implementation of PluginBase. + # @POST: Plugin is added to _plugins and _plugin_configs. # @PARAM: plugin_instance (PluginBase) - The plugin instance to register. def _register_plugin(self, plugin_instance: PluginBase): with belief_scope("_register_plugin"): @@ -143,13 +143,13 @@ class PluginLoader: except Exception as e: from ..core.logger import logger logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}") - # #endregion _register_plugin + # [/DEF:_register_plugin:Function] - # #region get_plugin [TYPE Function] - # @BRIEF Retrieves a loaded plugin instance by its ID. - # @PRE plugin_id is a string. - # @POST Returns plugin instance or None. + # [DEF:get_plugin:Function] + # @PURPOSE: Retrieves a loaded plugin instance by its ID. + # @PRE: plugin_id is a string. + # @POST: Returns plugin instance or None. # @PARAM: plugin_id (str) - The unique identifier of the plugin. # @RETURN: Optional[PluginBase] - The plugin instance if found, otherwise None. def get_plugin(self, plugin_id: str) -> Optional[PluginBase]: @@ -158,25 +158,25 @@ class PluginLoader: Returns a loaded plugin instance by its ID. """ return self._plugins.get(plugin_id) - # #endregion get_plugin + # [/DEF:get_plugin:Function] - # #region get_all_plugin_configs [TYPE Function] - # @BRIEF Returns a list of all registered plugin configurations. - # @PRE None. - # @POST Returns list of all PluginConfig objects. - # @RETURN List[PluginConfig] - A list of plugin configurations. + # [DEF:get_all_plugin_configs:Function] + # @PURPOSE: Returns a list of all registered plugin configurations. + # @PRE: None. + # @POST: Returns list of all PluginConfig objects. + # @RETURN: List[PluginConfig] - A list of plugin configurations. def get_all_plugin_configs(self) -> List[PluginConfig]: with belief_scope("get_all_plugin_configs"): """ Returns a list of all loaded plugin configurations. """ return list(self._plugin_configs.values()) - # #endregion get_all_plugin_configs + # [/DEF:get_all_plugin_configs:Function] - # #region has_plugin [TYPE Function] - # @BRIEF Checks if a plugin with the given ID is registered. - # @PRE plugin_id is a string. - # @POST Returns True if plugin exists. + # [DEF:has_plugin:Function] + # @PURPOSE: Checks if a plugin with the given ID is registered. + # @PRE: plugin_id is a string. + # @POST: Returns True if plugin exists. # @PARAM: plugin_id (str) - The unique identifier of the plugin. # @RETURN: bool - True if the plugin is registered, False otherwise. def has_plugin(self, plugin_id: str) -> bool: @@ -185,6 +185,6 @@ class PluginLoader: Checks if a plugin with the given ID is loaded. """ return plugin_id in self._plugins - # #endregion has_plugin + # [/DEF:has_plugin:Function] # #endregion PluginLoader diff --git a/backend/src/core/scheduler.py b/backend/src/core/scheduler.py index 05cf3602..47be20a3 100644 --- a/backend/src/core/scheduler.py +++ b/backend/src/core/scheduler.py @@ -1,30 +1,24 @@ # #region SchedulerModule [C:3] [TYPE Module] [SEMANTICS scheduler, apscheduler, cron, backup] # @BRIEF Manages scheduled tasks using APScheduler. -# @LAYER Core -# @RELATION DEPENDS_ON -> [TaskManager] +# @LAYER: Core +# @RELATION DEPENDS_ON -> TaskManager -# [SECTION: IMPORTS] from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger from .logger import logger, belief_scope from .config_manager import ConfigManager -from .cot_logger import MarkerLogger, get_trace_id, set_trace_id - -log = MarkerLogger("SchedulerService") -log_tsc = MarkerLogger("ThrottledSchedulerConfigurator") import asyncio from datetime import datetime, time, timedelta, date -# [/SECTION] # #region SchedulerService [C:3] [TYPE Class] [SEMANTICS scheduler, service, apscheduler] # @BRIEF Provides a service to manage scheduled backup tasks. # @RELATION DEPENDS_ON -> [ThrottledSchedulerConfigurator] class SchedulerService: - # #region __init__ [TYPE Function] - # @BRIEF Initializes the scheduler service with task and config managers. - # @PRE task_manager and config_manager must be provided. - # @POST Scheduler instance is created but not started. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the scheduler service with task and config managers. + # @PRE: task_manager and config_manager must be provided. + # @POST: Scheduler instance is created but not started. def __init__(self, task_manager, config_manager: ConfigManager): with belief_scope("SchedulerService.__init__"): self.task_manager = task_manager @@ -32,37 +26,37 @@ class SchedulerService: self.scheduler = BackgroundScheduler() self.loop = asyncio.get_event_loop() - # #endregion __init__ + # [/DEF:__init__:Function] - # #region start [TYPE Function] - # @BRIEF Starts the background scheduler and loads initial schedules. - # @PRE Scheduler should be initialized. - # @POST Scheduler is running and schedules are loaded. + # [DEF:start:Function] + # @PURPOSE: Starts the background scheduler and loads initial schedules. + # @PRE: Scheduler should be initialized. + # @POST: Scheduler is running and schedules are loaded. def start(self): with belief_scope("SchedulerService.start"): if not self.scheduler.running: self.scheduler.start() - log.reflect("Scheduler started") + logger.info("Scheduler started.") self.load_schedules() - # #endregion start + # [/DEF:start:Function] - # #region stop [TYPE Function] - # @BRIEF Stops the background scheduler. - # @PRE Scheduler should be running. - # @POST Scheduler is shut down. + # [DEF:stop:Function] + # @PURPOSE: Stops the background scheduler. + # @PRE: Scheduler should be running. + # @POST: Scheduler is shut down. def stop(self): with belief_scope("SchedulerService.stop"): if self.scheduler.running: self.scheduler.shutdown() - log.reflect("Scheduler stopped") + logger.info("Scheduler stopped.") - # #endregion stop + # [/DEF:stop:Function] - # #region load_schedules [TYPE Function] - # @BRIEF Loads backup schedules from configuration and registers them. - # @PRE config_manager must have valid configuration. - # @POST All enabled backup jobs are added to the scheduler. + # [DEF:load_schedules:Function] + # @PURPOSE: Loads backup schedules from configuration and registers them. + # @PRE: config_manager must have valid configuration. + # @POST: All enabled backup jobs are added to the scheduler. def load_schedules(self): with belief_scope("SchedulerService.load_schedules"): # Clear existing jobs @@ -73,12 +67,12 @@ class SchedulerService: if env.backup_schedule and env.backup_schedule.enabled: self.add_backup_job(env.id, env.backup_schedule.cron_expression) - # #endregion load_schedules + # [/DEF:load_schedules:Function] - # #region add_backup_job [TYPE Function] - # @BRIEF Adds a scheduled backup job for an environment. - # @PRE env_id and cron_expression must be valid strings. - # @POST A new job is added to the scheduler or replaced if it already exists. + # [DEF:add_backup_job:Function] + # @PURPOSE: Adds a scheduled backup job for an environment. + # @PRE: env_id and cron_expression must be valid strings. + # @POST: A new job is added to the scheduler or replaced if it already exists. # @PARAM: env_id (str) - The ID of the environment. # @PARAM: cron_expression (str) - The cron expression for the schedule. def add_backup_job(self, env_id: str, cron_expression: str): @@ -95,24 +89,22 @@ class SchedulerService: args=[env_id], replace_existing=True, ) - log.reflect( - "Scheduled backup job added", - payload={"env_id": env_id, "cron": cron_expression}, + logger.info( + f"Scheduled backup job added for environment {env_id}: {cron_expression}" ) except Exception as e: - log.explore("Failed to add backup job", error=str(e), payload={"env_id": env_id, "cron": cron_expression}) logger.error(f"Failed to add backup job for environment {env_id}: {e}") - # #endregion add_backup_job + # [/DEF:add_backup_job:Function] - # #region _trigger_backup [TYPE Function] - # @BRIEF Triggered by the scheduler to start a backup task. - # @PRE env_id must be a valid environment ID. - # @POST A new backup task is created in the task manager if not already running. + # [DEF:_trigger_backup:Function] + # @PURPOSE: Triggered by the scheduler to start a backup task. + # @PRE: env_id must be a valid environment ID. + # @POST: A new backup task is created in the task manager if not already running. # @PARAM: env_id (str) - The ID of the environment. def _trigger_backup(self, env_id: str): with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"): - log.reason("Triggering scheduled backup", payload={"env_id": env_id}) + logger.info(f"Triggering scheduled backup for environment {env_id}") # Check if a backup is already running for this environment active_tasks = self.task_manager.get_tasks(limit=100) @@ -122,23 +114,21 @@ class SchedulerService: and task.status in ["PENDING", "RUNNING"] and task.params.get("environment_id") == env_id ): - log.explore("Backup already running, skipping scheduled run", error="Duplicate backup prevented", payload={"env_id": env_id}) + logger.warning( + f"Backup already running for environment {env_id}. Skipping scheduled run." + ) return # Run the backup task # We need to run this in the event loop since create_task is async - trace_id = get_trace_id() - - async def _backup_task(): - if trace_id: - set_trace_id(trace_id) - await self.task_manager.create_task( + asyncio.run_coroutine_threadsafe( + self.task_manager.create_task( "superset-backup", {"environment_id": env_id} - ) + ), + self.loop, + ) - asyncio.run_coroutine_threadsafe(_backup_task(), self.loop) - - # #endregion _trigger_backup + # [/DEF:_trigger_backup:Function] # #endregion SchedulerService @@ -146,18 +136,18 @@ class SchedulerService: # #region ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution] # @BRIEF Distributes validation tasks evenly within an execution window. -# @PRE Validation policies provide a finite dashboard list and a valid execution window. -# @POST Produces deterministic per-dashboard run timestamps within the configured window. -# @SIDE_EFFECT Emits warning logs for degenerate or near-zero scheduling windows. -# @DATA_CONTRACT Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]] -# @INVARIANT Returned schedule size always matches number of dashboard IDs. -# @RELATION DEPENDS_ON -> [SchedulerModule] +# @PRE: Validation policies provide a finite dashboard list and a valid execution window. +# @POST: Produces deterministic per-dashboard run timestamps within the configured window. +# @RELATION DEPENDS_ON -> SchedulerModule +# @INVARIANT: Returned schedule size always matches number of dashboard IDs. +# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows. +# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]] class ThrottledSchedulerConfigurator: - # #region calculate_schedule [TYPE Function] - # @BRIEF Calculates execution times for N tasks within a window. - # @PRE window_start, window_end (time), dashboard_ids (List), current_date (date). - # @POST Returns List[datetime] of scheduled times. - # @INVARIANT Tasks are distributed with near-even spacing. + # [DEF:calculate_schedule:Function] + # @PURPOSE: Calculates execution times for N tasks within a window. + # @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date). + # @POST: Returns List[datetime] of scheduled times. + # @INVARIANT: Tasks are distributed with near-even spacing. @staticmethod def calculate_schedule( window_start: time, window_end: time, dashboard_ids: list, current_date: date @@ -178,7 +168,9 @@ class ThrottledSchedulerConfigurator: # Minimum interval of 1 second to avoid division by zero or negative if total_seconds <= 0: - log_tsc.explore("Window size is zero or negative, falling back to start time", error=f"total_seconds={total_seconds}", payload={"n": n}) + logger.warning( + f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks." + ) return [start_dt] * n # If window is too small for even distribution (e.g. 10 tasks in 5 seconds), @@ -193,7 +185,9 @@ class ThrottledSchedulerConfigurator: # If interval is too small (e.g. < 1s), we might want a fallback, # but the spec says "handle too-small windows with explicit fallback/warning". if interval < 1: - log_tsc.explore("Window too small for task distribution", error=f"interval={interval:.2f}s", payload={"n": n}) + logger.warning( + f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated." + ) scheduled_times = [] for i in range(n): @@ -201,7 +195,7 @@ class ThrottledSchedulerConfigurator: return scheduled_times - # #endregion calculate_schedule + # [/DEF:calculate_schedule:Function] # #endregion ThrottledSchedulerConfigurator diff --git a/backend/src/core/superset_client/__init__.py b/backend/src/core/superset_client/__init__.py index 49c825c6..9b115495 100644 --- a/backend/src/core/superset_client/__init__.py +++ b/backend/src/core/superset_client/__init__.py @@ -1,13 +1,13 @@ # #region SupersetClientModule [C:3] [TYPE Module] [SEMANTICS superset, api, client, rest, http, dashboard, dataset, import, export] +# @LAYER: Infra # @BRIEF Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию. -# @LAYER Infra -# @INVARIANT All network operations must use the internal APIClient instance. # @RELATION DEPENDS_ON -> [ConfigModels] # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetAPIError] # @RELATION DEPENDS_ON -> [get_filename_from_headers] # @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewFiltersMixin] # +# @INVARIANT: All network operations must use the internal APIClient instance. # @PUBLIC_API: SupersetClient # # @RATIONALE: Decomposed from monolithic superset_client.py (2145 lines) into diff --git a/backend/src/core/superset_client/_base.py b/backend/src/core/superset_client/_base.py index eaf33407..44b3f424 100644 --- a/backend/src/core/superset_client/_base.py +++ b/backend/src/core/superset_client/_base.py @@ -1,6 +1,6 @@ # #region SupersetClientBase [C:3] [TYPE Module] [SEMANTICS superset, api, client, base, pagination, auth, import, export] +# @LAYER: Infra # @BRIEF Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers. -# @LAYER Infra # @RELATION DEPENDS_ON -> [ConfigModels] # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetAPIError] @@ -12,13 +12,12 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast from requests import Response -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope from ..utils.network import APIClient, SupersetAPIError from ..utils.fileio import get_filename_from_headers from ..config_models import Environment -log = MarkerLogger("SupersetClientBase") +app_logger = cast(Any, app_logger) # #region SupersetClientBase [C:3] [TYPE Class] @@ -27,15 +26,16 @@ log = MarkerLogger("SupersetClientBase") # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetAPIError] class SupersetClientBase: - # #region SupersetClientInit [C:3] [TYPE Function] - # @BRIEF Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент. - # @RELATION DEPENDS_ON -> [Environment] - # @RELATION DEPENDS_ON -> [APIClient] + # [DEF:SupersetClientInit:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент. + # @RELATION: DEPENDS_ON -> [Environment] + # @RELATION: DEPENDS_ON -> [APIClient] def __init__(self, env: Environment): with belief_scope("SupersetClientInit"): - log.reason( + app_logger.reason( "Initializing Superset client for environment", - payload={"environment": getattr(env, "id", None), "env_name": env.name}, + extra={"environment": getattr(env, "id", None), "env_name": env.name}, ) self.env = env # Construct auth payload expected by Superset API @@ -51,46 +51,49 @@ class SupersetClientBase: timeout=env.timeout, ) self.delete_before_reimport: bool = False - log.reflect( + app_logger.reflect( "Superset client initialized", - payload={"environment": getattr(self.env, "id", None)}, + extra={"environment": getattr(self.env, "id", None)}, ) - # #endregion SupersetClientInit + # [/DEF:SupersetClientInit:Function] - # #region SupersetClientAuthenticate [C:3] [TYPE Function] - # @BRIEF Authenticates the client using the configured credentials. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientAuthenticate:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Authenticates the client using the configured credentials. + # @RELATION: CALLS -> [APIClient] def authenticate(self) -> Dict[str, str]: with belief_scope("SupersetClientAuthenticate"): - log.reason( + app_logger.reason( "Authenticating Superset client", - payload={"environment": getattr(self.env, "id", None)}, + extra={"environment": getattr(self.env, "id", None)}, ) tokens = self.network.authenticate() - log.reflect( + app_logger.reflect( "Superset client authentication completed", - payload={ + extra={ "environment": getattr(self.env, "id", None), "token_keys": sorted(tokens.keys()), }, ) return tokens - # #endregion SupersetClientAuthenticate + # [/DEF:SupersetClientAuthenticate:Function] @property - # #region SupersetClientHeaders [C:1] [TYPE Function] - # @BRIEF Возвращает базовые HTTP-заголовки, используемые сетевым клиентом. + # [DEF:SupersetClientHeaders:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом. def headers(self) -> dict: with belief_scope("headers"): return self.network.headers - # #endregion SupersetClientHeaders + # [/DEF:SupersetClientHeaders:Function] # --- Pagination helpers --- - # #region SupersetClientValidateQueryParams [C:1] [TYPE Function] - # @BRIEF Ensures query parameters have default page and page_size. + # [DEF:SupersetClientValidateQueryParams:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Ensures query parameters have default page and page_size. def _validate_query_params(self, query: Optional[Dict]) -> Dict: with belief_scope("_validate_query_params"): # Superset list endpoints commonly cap page_size at 100. @@ -98,11 +101,12 @@ class SupersetClientBase: base_query = {"page": 0, "page_size": 100} return {**base_query, **(query or {})} - # #endregion SupersetClientValidateQueryParams + # [/DEF:SupersetClientValidateQueryParams:Function] - # #region SupersetClientFetchTotalObjectCount [C:1] [TYPE Function] - # @BRIEF Fetches the total number of items for a given endpoint. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientFetchTotalObjectCount:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Fetches the total number of items for a given endpoint. + # @RELATION: CALLS -> [APIClient] def _fetch_total_object_count(self, endpoint: str) -> int: with belief_scope("_fetch_total_object_count"): return self.network.fetch_paginated_count( @@ -111,30 +115,34 @@ class SupersetClientBase: count_field="count", ) - # #endregion SupersetClientFetchTotalObjectCount + # [/DEF:SupersetClientFetchTotalObjectCount:Function] - # #region SupersetClientFetchAllPages [C:1] [TYPE Function] - # @BRIEF Iterates through all pages to collect all data items. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientFetchAllPages:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Iterates through all pages to collect all data items. + # @RELATION: CALLS -> [APIClient] def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]: with belief_scope("_fetch_all_pages"): return self.network.fetch_paginated_data( endpoint=endpoint, pagination_options=pagination_options ) - # #endregion SupersetClientFetchAllPages + # [/DEF:SupersetClientFetchAllPages:Function] # --- Import/Export helpers --- - # #region SupersetClientDoImport [C:1] [TYPE Function] - # @BRIEF Performs the actual multipart upload for import. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientDoImport:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Performs the actual multipart upload for import. + # @RELATION: CALLS -> [APIClient] def _do_import(self, file_name: Union[str, Path]) -> Dict: with belief_scope("_do_import"): - log.reason("Uploading file for import", payload={"file_name": str(file_name)}) + app_logger.debug(f"[_do_import][State] Uploading file: {file_name}") file_path = Path(file_name) if not file_path.exists(): - log.explore("Import file does not exist", error="FileNotFound", payload={"file_name": str(file_name)}) + app_logger.error( + f"[_do_import][Failure] File does not exist: {file_name}" + ) raise FileNotFoundError(f"File does not exist: {file_name}") return self.network.upload_file( @@ -148,10 +156,11 @@ class SupersetClientBase: timeout=self.env.timeout * 2, ) - # #endregion SupersetClientDoImport + # [/DEF:SupersetClientDoImport:Function] - # #region SupersetClientValidateExportResponse [C:1] [TYPE Function] - # @BRIEF Validates that the export response is a non-empty ZIP archive. + # [DEF:SupersetClientValidateExportResponse:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Validates that the export response is a non-empty ZIP archive. def _validate_export_response(self, response: Response, dashboard_id: int) -> None: with belief_scope("_validate_export_response"): content_type = response.headers.get("Content-Type", "") @@ -162,10 +171,11 @@ class SupersetClientBase: if not response.content: raise SupersetAPIError("Получены пустые данные при экспорте") - # #endregion SupersetClientValidateExportResponse + # [/DEF:SupersetClientValidateExportResponse:Function] - # #region SupersetClientResolveExportFilename [C:1] [TYPE Function] - # @BRIEF Determines the filename for an exported dashboard. + # [DEF:SupersetClientResolveExportFilename:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Determines the filename for an exported dashboard. def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str: with belief_scope("_resolve_export_filename"): filename = get_filename_from_headers(dict(response.headers)) @@ -174,16 +184,17 @@ class SupersetClientBase: timestamp = datetime.now().strftime("%Y%m%dT%H%M%S") filename = f"dashboard_export_{dashboard_id}_{timestamp}.zip" - log.reflect( - "Export filename not found in headers, using generated name", - payload={"dashboard_id": dashboard_id, "generated_filename": filename}, + app_logger.warning( + "[_resolve_export_filename][Warning] Generated filename: %s", + filename, ) return filename - # #endregion SupersetClientResolveExportFilename + # [/DEF:SupersetClientResolveExportFilename:Function] - # #region SupersetClientValidateImportFile [C:1] [TYPE Function] - # @BRIEF Validates that the file to be imported is a valid ZIP with metadata.yaml. + # [DEF:SupersetClientValidateImportFile:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml. def _validate_import_file(self, zip_path: Union[str, Path]) -> None: with belief_scope("_validate_import_file"): path = Path(zip_path) @@ -197,11 +208,12 @@ class SupersetClientBase: f"Архив {zip_path} не содержит 'metadata.yaml'" ) - # #endregion SupersetClientValidateImportFile + # [/DEF:SupersetClientValidateImportFile:Function] - # #region SupersetClientResolveTargetIdForDelete [C:1] [TYPE Function] - # @BRIEF Resolves a dashboard ID from either an ID or a slug. - # @RELATION CALLS -> [SupersetClientGetDashboards] + # [DEF:SupersetClientResolveTargetIdForDelete:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Resolves a dashboard ID from either an ID or a slug. + # @RELATION: CALLS -> [SupersetClientGetDashboards] def _resolve_target_id_for_delete( self, dash_id: Optional[int], dash_slug: Optional[str] ) -> Optional[int]: @@ -209,7 +221,10 @@ class SupersetClientBase: if dash_id is not None: return dash_id if dash_slug is not None: - log.reason("Resolving dashboard ID by slug", payload={"slug": dash_slug}) + app_logger.debug( + "[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.", + dash_slug, + ) try: _, candidates = self.get_dashboards( query={ @@ -218,17 +233,25 @@ class SupersetClientBase: ) if candidates: target_id = candidates[0]["id"] - log.reflect("Resolved slug to dashboard ID", payload={"slug": dash_slug, "target_id": target_id}) + app_logger.debug( + "[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.", + target_id, + ) return target_id except Exception as e: - log.explore("Could not resolve slug to dashboard ID", error=str(e), payload={"slug": dash_slug}) + app_logger.warning( + "[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s", + dash_slug, + e, + ) return None - # #endregion SupersetClientResolveTargetIdForDelete + # [/DEF:SupersetClientResolveTargetIdForDelete:Function] - # #region SupersetClientGetAllResources [C:3] [TYPE Function] - # @BRIEF Fetches all resources of a given type with id, uuid, and name columns. - # @RELATION CALLS -> [SupersetClientFetchAllPages] + # [DEF:SupersetClientGetAllResources:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns. + # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_all_resources( self, resource_type: str, since_dttm: Optional["datetime"] = None ) -> List[Dict]: @@ -252,7 +275,10 @@ class SupersetClientBase: } config = column_map.get(resource_type) if not config: - log.explore("Unknown resource type requested", error=f"Invalid resource type: {resource_type}", payload={"resource_type": resource_type}) + app_logger.warning( + "[get_all_resources][Warning] Unknown resource type: %s", + resource_type, + ) return [] query = {"columns": config["columns"]} @@ -272,13 +298,15 @@ class SupersetClientBase: endpoint=config["endpoint"], pagination_options={"base_query": validated, "results_field": "result"}, ) - log.reflect( - "Resources fetched", - payload={"resource_type": resource_type, "count": len(data)}, + app_logger.info( + "[get_all_resources][Exit] Fetched %d %s resources.", + len(data), + resource_type, ) return data - # #endregion SupersetClientGetAllResources + # [/DEF:SupersetClientGetAllResources:Function] # #endregion SupersetClientBase +# #endregion SupersetClientBase diff --git a/backend/src/core/superset_client/_charts.py b/backend/src/core/superset_client/_charts.py index 058ed7dd..71e15330 100644 --- a/backend/src/core/superset_client/_charts.py +++ b/backend/src/core/superset_client/_charts.py @@ -1,6 +1,6 @@ # #region SupersetChartsMixin [C:3] [TYPE Module] [SEMANTICS superset, charts, list, get, layout] +# @LAYER: Infra # @BRIEF Chart domain mixin for SupersetClient — list, get, extract IDs from layout. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] import json @@ -16,19 +16,21 @@ app_logger = cast(Any, app_logger) # @BRIEF Mixin providing all chart-related Superset API operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetChartsMixin: - # #region SupersetClientGetChart [C:3] [TYPE Function] - # @BRIEF Fetches a single chart by ID. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetChart:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches a single chart by ID. + # @RELATION: CALLS -> [APIClient] def get_chart(self, chart_id: int) -> Dict: with belief_scope("SupersetClient.get_chart", f"id={chart_id}"): response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}") return cast(Dict, response) - # #endregion SupersetClientGetChart + # [/DEF:SupersetClientGetChart:Function] - # #region SupersetClientGetCharts [C:3] [TYPE Function] - # @BRIEF Fetches all charts with pagination support. - # @RELATION CALLS -> [SupersetClientFetchAllPages] + # [DEF:SupersetClientGetCharts:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches all charts with pagination support. + # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_charts"): validated_query = self._validate_query_params(query or {}) @@ -44,10 +46,11 @@ class SupersetChartsMixin: ) return len(paginated_data), paginated_data - # #endregion SupersetClientGetCharts + # [/DEF:SupersetClientGetCharts:Function] - # #region SupersetClientExtractChartIdsFromLayout [C:1] [TYPE Function] - # @BRIEF Traverses dashboard layout metadata and extracts chart IDs from common keys. + # [DEF:SupersetClientExtractChartIdsFromLayout:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys. def _extract_chart_ids_from_layout( self, payload: Any ) -> set: @@ -77,7 +80,8 @@ class SupersetChartsMixin: walk(payload) return found - # #endregion SupersetClientExtractChartIdsFromLayout + # [/DEF:SupersetClientExtractChartIdsFromLayout:Function] # #endregion SupersetChartsMixin +# #endregion SupersetChartsMixin diff --git a/backend/src/core/superset_client/_dashboards_crud.py b/backend/src/core/superset_client/_dashboards_crud.py index 9235b516..3a2da608 100644 --- a/backend/src/core/superset_client/_dashboards_crud.py +++ b/backend/src/core/superset_client/_dashboards_crud.py @@ -1,6 +1,6 @@ # #region SupersetDashboardsCrudMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, crud, detail, export, import, delete] +# @LAYER: Infra # @BRIEF Dashboard CRUD mixin for SupersetClient — detail, export, import, delete. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin] # @RELATION DEPENDS_ON -> [SupersetChartsMixin] @@ -11,10 +11,9 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast from requests import Response -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope -log = MarkerLogger("SupersetDashboardsCrud") +app_logger = cast(Any, app_logger) # #region SupersetDashboardsCrudMixin [C:3] [TYPE Class] @@ -23,10 +22,11 @@ log = MarkerLogger("SupersetDashboardsCrud") # @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin] # @RELATION DEPENDS_ON -> [SupersetChartsMixin] class SupersetDashboardsCrudMixin: - # #region SupersetClientGetDashboardDetail [C:3] [TYPE Function] - # @BRIEF Fetches detailed dashboard information including related charts and datasets. - # @RELATION CALLS -> [SupersetClientGetDashboard] - # @RELATION CALLS -> [SupersetClientGetChart] + # [DEF:SupersetClientGetDashboardDetail:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches detailed dashboard information including related charts and datasets. + # @RELATION: CALLS -> [SupersetClientGetDashboard] + # @RELATION: CALLS -> [SupersetClientGetChart] def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict: with belief_scope( "SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}" @@ -104,7 +104,9 @@ class SupersetDashboardsCrudMixin: or "Chart", }) except Exception as e: - log.explore("Failed to fetch dashboard charts", error=str(e), payload={"dashboard_ref": dashboard_ref}) + app_logger.warning( + "[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", e, + ) try: datasets_response = self.network.request( @@ -142,7 +144,9 @@ class SupersetDashboardsCrudMixin: "overview": fq_name, }) except Exception as e: - log.explore("Failed to fetch dashboard datasets", error=str(e), payload={"dashboard_ref": dashboard_ref}) + app_logger.warning( + "[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", e, + ) # Fallback: derive chart IDs from layout metadata if not charts: @@ -175,12 +179,9 @@ class SupersetDashboardsCrudMixin: self._extract_chart_ids_from_layout(raw_json_metadata) ) - log.reason( - "Extracted fallback chart IDs from layout", - payload={ - "chart_count": len(chart_ids_from_position), - "dashboard_ref": dashboard_ref, - }, + app_logger.info( + "[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)", + len(chart_ids_from_position), dashboard_ref, ) for chart_id in sorted(chart_ids_from_position): @@ -198,7 +199,10 @@ class SupersetDashboardsCrudMixin: or chart_data.get("viz_type") or "Chart", }) except Exception as e: - log.explore("Failed to resolve fallback chart", error=str(e), payload={"chart_id": chart_id}) + app_logger.warning( + "[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s", + chart_id, e, + ) # Backfill datasets from chart datasource IDs. dataset_ids_from_charts = { @@ -240,7 +244,10 @@ class SupersetDashboardsCrudMixin: "overview": fq_name, }) except Exception as e: - log.explore("Failed to resolve dataset from chart datasource", error=str(e), payload={"dataset_id": dataset_id}) + app_logger.warning( + "[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s", + dataset_id, e, + ) unique_charts = {chart["id"]: chart for chart in charts} unique_datasets = {dataset["id"]: dataset for dataset in datasets} @@ -262,15 +269,18 @@ class SupersetDashboardsCrudMixin: "dataset_count": len(unique_datasets), } - # #endregion SupersetClientGetDashboardDetail + # [/DEF:SupersetClientGetDashboardDetail:Function] - # #region SupersetClientExportDashboard [C:3] [TYPE Function] - # @BRIEF Экспортирует дашборд в виде ZIP-архива. - # @SIDE_EFFECT Performs network I/O to download archive. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientExportDashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Экспортирует дашборд в виде ZIP-архива. + # @SIDE_EFFECT: Performs network I/O to download archive. + # @RELATION: CALLS -> [APIClient] def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]: with belief_scope("export_dashboard"): - log.reason("Exporting dashboard", payload={"dashboard_id": dashboard_id}) + app_logger.info( + "[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id + ) response = self.network.request( method="GET", endpoint="/dashboard/export/", @@ -281,16 +291,20 @@ class SupersetDashboardsCrudMixin: response = cast(Response, response) self._validate_export_response(response, dashboard_id) filename = self._resolve_export_filename(response, dashboard_id) - log.reflect("Dashboard exported", payload={"dashboard_id": dashboard_id, "filename": filename}) + app_logger.info( + "[export_dashboard][Exit] Exported dashboard %s to %s.", + dashboard_id, filename, + ) return response.content, filename - # #endregion SupersetClientExportDashboard + # [/DEF:SupersetClientExportDashboard:Function] - # #region SupersetClientImportDashboard [C:3] [TYPE Function] - # @BRIEF Импортирует дашборд из ZIP-файла. - # @SIDE_EFFECT Performs network I/O to upload archive. - # @RELATION CALLS -> [SupersetClientDoImport] - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientImportDashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Импортирует дашборд из ZIP-файла. + # @SIDE_EFFECT: Performs network I/O to upload archive. + # @RELATION: CALLS -> [SupersetClientDoImport] + # @RELATION: CALLS -> [APIClient] def import_dashboard( self, file_name: Union[str, Path], @@ -305,41 +319,55 @@ class SupersetDashboardsCrudMixin: try: return self._do_import(file_path) except Exception as exc: - log.explore("First import attempt failed", error=str(exc), payload={"file_name": file_name}) + app_logger.error( + "[import_dashboard][Failure] First import attempt failed: %s", + exc, exc_info=True, + ) if not self.delete_before_reimport: raise target_id = self._resolve_target_id_for_delete(dash_id, dash_slug) if target_id is None: - log.explore("No ID available for delete-retry during import", error="Cannot delete-retry: neither dash_id nor dash_slug resolved", payload={"dash_id": dash_id, "dash_slug": dash_slug}) + app_logger.error( + "[import_dashboard][Failure] No ID available for delete-retry." + ) raise self.delete_dashboard(target_id) - log.reason( - "Deleted dashboard, retrying import", - payload={"target_id": target_id}, + app_logger.info( + "[import_dashboard][State] Deleted dashboard ID %s, retrying import.", + target_id, ) return self._do_import(file_path) - # #endregion SupersetClientImportDashboard + # [/DEF:SupersetClientImportDashboard:Function] - # #region SupersetClientDeleteDashboard [C:3] [TYPE Function] - # @BRIEF Удаляет дашборд по его ID или slug. - # @SIDE_EFFECT Deletes resource from upstream Superset environment. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientDeleteDashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Удаляет дашборд по его ID или slug. + # @SIDE_EFFECT: Deletes resource from upstream Superset environment. + # @RELATION: CALLS -> [APIClient] def delete_dashboard(self, dashboard_id: Union[int, str]) -> None: with belief_scope("delete_dashboard"): - log.reason("Deleting dashboard", payload={"dashboard_id": dashboard_id}) + app_logger.info( + "[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id + ) response = self.network.request( method="DELETE", endpoint=f"/dashboard/{dashboard_id}" ) response = cast(Dict, response) if response.get("result", True) is not False: - log.reflect("Dashboard deleted", payload={"dashboard_id": dashboard_id}) + app_logger.info( + "[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id + ) else: - log.explore("Unexpected response while deleting dashboard", error=f"Unexpected API response: {response}", payload={"dashboard_id": dashboard_id, "response": response}) + app_logger.warning( + "[delete_dashboard][Warning] Unexpected response while deleting %s: %s", + dashboard_id, response, + ) - # #endregion SupersetClientDeleteDashboard + # [/DEF:SupersetClientDeleteDashboard:Function] # #endregion SupersetDashboardsCrudMixin +# #endregion SupersetDashboardsCrudMixin diff --git a/backend/src/core/superset_client/_dashboards_filters.py b/backend/src/core/superset_client/_dashboards_filters.py index cdff3fab..b79dce19 100644 --- a/backend/src/core/superset_client/_dashboards_filters.py +++ b/backend/src/core/superset_client/_dashboards_filters.py @@ -1,24 +1,24 @@ # #region SupersetDashboardsFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, filters, permalink, native-filters] +# @LAYER: Infra # @BRIEF Dashboard native filter extraction mixin for SupersetClient. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] import json from typing import Any, Dict, Optional, Union, cast -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope -log = MarkerLogger("SupersetDashboardsFilters") +app_logger = cast(Any, app_logger) # #region SupersetDashboardsFiltersMixin [C:3] [TYPE Class] # @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetDashboardsFiltersMixin: - # #region SupersetClientGetDashboard [C:3] [TYPE Function] - # @BRIEF Fetches a single dashboard by ID or slug. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches a single dashboard by ID or slug. + # @RELATION: CALLS -> [APIClient] def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict: with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"): response = self.network.request( @@ -26,11 +26,12 @@ class SupersetDashboardsFiltersMixin: ) return cast(Dict, response) - # #endregion SupersetClientGetDashboard + # [/DEF:SupersetClientGetDashboard:Function] - # #region SupersetClientGetDashboardPermalinkState [C:2] [TYPE Function] - # @BRIEF Fetches stored dashboard permalink state by permalink key. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDashboardPermalinkState:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Fetches stored dashboard permalink state by permalink key. + # @RELATION: CALLS -> [APIClient] def get_dashboard_permalink_state(self, permalink_key: str) -> Dict: with belief_scope( "SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}" @@ -40,11 +41,12 @@ class SupersetDashboardsFiltersMixin: ) return cast(Dict, response) - # #endregion SupersetClientGetDashboardPermalinkState + # [/DEF:SupersetClientGetDashboardPermalinkState:Function] - # #region SupersetClientGetNativeFilterState [C:2] [TYPE Function] - # @BRIEF Fetches stored native filter state by filter state key. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetNativeFilterState:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Fetches stored native filter state by filter state key. + # @RELATION: CALLS -> [APIClient] def get_native_filter_state( self, dashboard_id: Union[int, str], filter_state_key: str ) -> Dict: @@ -58,11 +60,12 @@ class SupersetDashboardsFiltersMixin: ) return cast(Dict, response) - # #endregion SupersetClientGetNativeFilterState + # [/DEF:SupersetClientGetNativeFilterState:Function] - # #region SupersetClientExtractNativeFiltersFromPermalink [C:3] [TYPE Function] - # @BRIEF Extract native filters dataMask from a permalink key. - # @RELATION CALLS -> [SupersetClientGetDashboardPermalinkState] + # [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract native filters dataMask from a permalink key. + # @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState] def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict: with belief_scope( "SupersetClient.extract_native_filters_from_permalink", @@ -92,11 +95,12 @@ class SupersetDashboardsFiltersMixin: "permalink_key": permalink_key, } - # #endregion SupersetClientExtractNativeFiltersFromPermalink + # [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function] - # #region SupersetClientExtractNativeFiltersFromKey [C:3] [TYPE Function] - # @BRIEF Extract native filters from a native_filters_key URL parameter. - # @RELATION CALLS -> [SupersetClientGetNativeFilterState] + # [DEF:SupersetClientExtractNativeFiltersFromKey:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract native filters from a native_filters_key URL parameter. + # @RELATION: CALLS -> [SupersetClientGetNativeFilterState] def extract_native_filters_from_key( self, dashboard_id: Union[int, str], filter_state_key: str ) -> Dict: @@ -115,7 +119,10 @@ class SupersetDashboardsFiltersMixin: try: parsed_value = json.loads(value) except json.JSONDecodeError as e: - log.explore("Failed to parse filter state JSON", error=str(e)) + app_logger.warning( + "[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s", + e, + ) parsed_value = {} elif isinstance(value, dict): parsed_value = value @@ -147,12 +154,13 @@ class SupersetDashboardsFiltersMixin: "filter_state_key": filter_state_key, } - # #endregion SupersetClientExtractNativeFiltersFromKey + # [/DEF:SupersetClientExtractNativeFiltersFromKey:Function] - # #region SupersetClientParseDashboardUrlForFilters [C:3] [TYPE Function] - # @BRIEF Parse a Superset dashboard URL and extract native filter state if present. - # @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromPermalink] - # @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromKey] + # [DEF:SupersetClientParseDashboardUrlForFilters:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present. + # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink] + # @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey] def parse_dashboard_url_for_filters(self, url: str) -> Dict: with belief_scope( "SupersetClient.parse_dashboard_url_for_filters", f"url={url}" @@ -215,7 +223,11 @@ class SupersetDashboardsFiltersMixin: if raw_id is not None: resolved_id = int(raw_id) except Exception as e: - log.explore("Failed to resolve dashboard slug to ID", error=str(e), payload={"slug": dashboard_ref}) + app_logger.warning( + "[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s", + dashboard_ref, + e, + ) if resolved_id is not None: filter_data = self.extract_native_filters_from_key( @@ -226,7 +238,9 @@ class SupersetDashboardsFiltersMixin: result["filters"] = filter_data return result else: - log.explore("Could not resolve dashboard_id from URL for native_filters_key", error="Dashboard ID could not be resolved from URL", payload={"dashboard_ref": dashboard_ref}) + app_logger.warning( + "[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key" + ) # Check for native_filters in query params (direct filter values) native_filters = query_params.get("native_filters", [None])[0] @@ -237,11 +251,15 @@ class SupersetDashboardsFiltersMixin: result["filters"] = {"dataMask": parsed_filters} return result except json.JSONDecodeError as e: - log.explore("Failed to parse native_filters JSON", error=str(e)) + app_logger.warning( + "[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s", + e, + ) return result - # #endregion SupersetClientParseDashboardUrlForFilters + # [/DEF:SupersetClientParseDashboardUrlForFilters:Function] # #endregion SupersetDashboardsFiltersMixin +# #endregion SupersetDashboardsFiltersMixin diff --git a/backend/src/core/superset_client/_dashboards_list.py b/backend/src/core/superset_client/_dashboards_list.py index 487d7f6c..02090676 100644 --- a/backend/src/core/superset_client/_dashboards_list.py +++ b/backend/src/core/superset_client/_dashboards_list.py @@ -1,16 +1,15 @@ # #region SupersetDashboardsListMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, list, pagination, summary] +# @LAYER: Infra # @BRIEF Dashboard listing mixin for SupersetClient — paginated list, summary projection. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin] import json from typing import Any, Dict, List, Optional, Tuple, cast -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope -log = MarkerLogger("SupersetDashboardsList") +app_logger = cast(Any, app_logger) # #region SupersetDashboardsListMixin [C:3] [TYPE Class] @@ -18,12 +17,13 @@ log = MarkerLogger("SupersetDashboardsList") # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin] class SupersetDashboardsListMixin: - # #region SupersetClientGetDashboards [C:3] [TYPE Function] - # @BRIEF Получает полный список дашбордов, автоматически обрабатывая пагинацию. - # @RELATION CALLS -> [SupersetClientFetchAllPages] + # [DEF:SupersetClientGetDashboards:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию. + # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_dashboards"): - log.reason("Fetching dashboards", payload={"has_query": query is not None}) + app_logger.info("[get_dashboards][Enter] Fetching dashboards.") validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: validated_query["columns"] = [ @@ -39,14 +39,15 @@ class SupersetDashboardsListMixin: }, ) total_count = len(paginated_data) - log.reflect("Dashboards fetched", payload={"total_count": total_count}) + app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count) return total_count, paginated_data - # #endregion SupersetClientGetDashboards + # [/DEF:SupersetClientGetDashboards:Function] - # #region SupersetClientGetDashboardsPage [C:3] [TYPE Function] - # @BRIEF Fetches a single dashboards page from Superset without iterating all pages. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDashboardsPage:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages. + # @RELATION: CALLS -> [APIClient] def get_dashboards_page( self, query: Optional[Dict] = None ) -> Tuple[int, List[Dict]]: @@ -70,11 +71,12 @@ class SupersetDashboardsListMixin: total_count = response_json.get("count", len(result)) return total_count, result - # #endregion SupersetClientGetDashboardsPage + # [/DEF:SupersetClientGetDashboardsPage:Function] - # #region SupersetClientGetDashboardsSummary [C:3] [TYPE Function] - # @BRIEF Fetches dashboard metadata optimized for the grid. - # @RELATION CALLS -> [SupersetClientGetDashboards] + # [DEF:SupersetClientGetDashboardsSummary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches dashboard metadata optimized for the grid. + # @RELATION: CALLS -> [SupersetClientGetDashboards] def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]: with belief_scope("SupersetClient.get_dashboards_summary"): query: Dict[str, Any] = {} @@ -122,37 +124,28 @@ class SupersetDashboardsListMixin: }) if index < max_debug_samples: - log.reflect( - "Dashboard actor projection sample", - payload={ - "env": getattr(self.env, "id", None), - "dashboard_id": dash.get("id"), - "raw_owners": raw_owners, - "raw_owner_usernames": raw_owner_usernames, - "raw_created_by": raw_created_by, - "raw_changed_by": raw_changed_by, - "raw_changed_by_name": raw_changed_by_name, - "projected_owners": owners, - "projected_created_by": projected_created_by, - "projected_modified_by": projected_modified_by, - }, + app_logger.reflect( + "[REFLECT] Dashboard actor projection sample " + f"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, " + f"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, " + f"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, " + f"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, " + f"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})" ) - log.reflect( - "Dashboard actor projection summary", - payload={ - "env": getattr(self.env, "id", None), - "dashboards": len(result), - "sampled": min(len(result), max_debug_samples), - }, + app_logger.reflect( + "[REFLECT] Dashboard actor projection summary " + f"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, " + f"sampled={min(len(result), max_debug_samples)})" ) return result - # #endregion SupersetClientGetDashboardsSummary + # [/DEF:SupersetClientGetDashboardsSummary:Function] - # #region SupersetClientGetDashboardsSummaryPage [C:3] [TYPE Function] - # @BRIEF Fetches one page of dashboard metadata optimized for the grid. - # @RELATION CALLS -> [SupersetClientGetDashboardsPage] + # [DEF:SupersetClientGetDashboardsSummaryPage:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches one page of dashboard metadata optimized for the grid. + # @RELATION: CALLS -> [SupersetClientGetDashboardsPage] def get_dashboards_summary_page( self, page: int, @@ -204,7 +197,8 @@ class SupersetDashboardsListMixin: return total_count, result - # #endregion SupersetClientGetDashboardsSummaryPage + # [/DEF:SupersetClientGetDashboardsSummaryPage:Function] # #endregion SupersetDashboardsListMixin +# #endregion SupersetDashboardsListMixin diff --git a/backend/src/core/superset_client/_databases.py b/backend/src/core/superset_client/_databases.py index 6f0c5b2e..07a1fb94 100644 --- a/backend/src/core/superset_client/_databases.py +++ b/backend/src/core/superset_client/_databases.py @@ -1,26 +1,26 @@ # #region SupersetDatabasesMixin [C:3] [TYPE Module] [SEMANTICS superset, databases, list, get, summary, uuid] +# @LAYER: Infra # @BRIEF Database domain mixin for SupersetClient — list, get, summary, by_uuid. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] from typing import Any, Dict, List, Optional, Tuple, cast -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope -log = MarkerLogger("SupersetDatabases") +app_logger = cast(Any, app_logger) # #region SupersetDatabasesMixin [C:3] [TYPE Class] # @BRIEF Mixin providing all database-related Superset API operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetDatabasesMixin: - # #region SupersetClientGetDatabases [C:3] [TYPE Function] - # @BRIEF Получает полный список баз данных. - # @RELATION CALLS -> [SupersetClientFetchAllPages] + # [DEF:SupersetClientGetDatabases:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Получает полный список баз данных. + # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_databases"): - log.reason("Fetching databases", payload={"has_query": query is not None}) + app_logger.info("[get_databases][Enter] Fetching databases.") validated_query = self._validate_query_params(query or {}) if "columns" not in validated_query: validated_query["columns"] = [] @@ -33,32 +33,34 @@ class SupersetDatabasesMixin: }, ) total_count = len(paginated_data) - log.reflect("Databases fetched", payload={"total_count": total_count}) + app_logger.info("[get_databases][Exit] Found %d databases.", total_count) return total_count, paginated_data - # #endregion SupersetClientGetDatabases + # [/DEF:SupersetClientGetDatabases:Function] - # #region SupersetClientGetDatabase [C:3] [TYPE Function] - # @BRIEF Получает информацию о конкретной базе данных по её ID. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDatabase:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Получает информацию о конкретной базе данных по её ID. + # @RELATION: CALLS -> [APIClient] def get_database(self, database_id: int) -> Dict: with belief_scope("get_database"): - log.reason("Fetching database", payload={"database_id": database_id}) + app_logger.info("[get_database][Enter] Fetching database %s.", database_id) response = self.network.request( method="GET", endpoint=f"/database/{database_id}" ) response = cast(Dict, response) - log.reflect("Database fetched", payload={"database_id": database_id}) + app_logger.info("[get_database][Exit] Got database %s.", database_id) return response - # #endregion SupersetClientGetDatabase + # [/DEF:SupersetClientGetDatabase:Function] - # #region SupersetClientGetDatabasesSummary [C:3] [TYPE Function] - # @BRIEF Fetch a summary of databases including uuid, name, and engine. - # @RELATION CALLS -> [SupersetClientGetDatabases] + # [DEF:SupersetClientGetDatabasesSummary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch a summary of databases including uuid, name, and engine. + # @RELATION: CALLS -> [SupersetClientGetDatabases] def get_databases_summary(self) -> List[Dict]: with belief_scope("SupersetClient.get_databases_summary"): - query = {"columns": ["id", "uuid", "database_name", "backend"]} + query = {"columns": ["uuid", "database_name", "backend"]} _, databases = self.get_databases(query=query) # Map 'backend' to 'engine' for consistency with contracts @@ -67,18 +69,20 @@ class SupersetDatabasesMixin: return databases - # #endregion SupersetClientGetDatabasesSummary + # [/DEF:SupersetClientGetDatabasesSummary:Function] - # #region SupersetClientGetDatabaseByUuid [C:3] [TYPE Function] - # @BRIEF Find a database by its UUID. - # @RELATION CALLS -> [SupersetClientGetDatabases] + # [DEF:SupersetClientGetDatabaseByUuid:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Find a database by its UUID. + # @RELATION: CALLS -> [SupersetClientGetDatabases] def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]: with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"): query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]} _, databases = self.get_databases(query=query) return databases[0] if databases else None - # #endregion SupersetClientGetDatabaseByUuid + # [/DEF:SupersetClientGetDatabaseByUuid:Function] # #endregion SupersetDatabasesMixin +# #endregion SupersetDatabasesMixin diff --git a/backend/src/core/superset_client/_datasets.py b/backend/src/core/superset_client/_datasets.py index 26d0c87e..b2ffc8fd 100644 --- a/backend/src/core/superset_client/_datasets.py +++ b/backend/src/core/superset_client/_datasets.py @@ -1,27 +1,27 @@ # #region SupersetDatasetsMixin [C:3] [TYPE Module] [SEMANTICS superset, datasets, list, get, detail, update] +# @LAYER: Infra # @BRIEF Dataset domain mixin for SupersetClient — list, get, detail, update. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] import json from typing import Any, Dict, List, Optional, Tuple, cast -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope -log = MarkerLogger("SupersetDatasets") +app_logger = cast(Any, app_logger) # #region SupersetDatasetsMixin [C:3] [TYPE Class] # @BRIEF Mixin providing basic dataset CRUD operations (list, get, detail, update). # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetDatasetsMixin: - # #region SupersetClientGetDatasets [C:3] [TYPE Function] - # @BRIEF Получает полный список датасетов, автоматически обрабатывая пагинацию. - # @RELATION CALLS -> [SupersetClientFetchAllPages] + # [DEF:SupersetClientGetDatasets:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию. + # @RELATION: CALLS -> [SupersetClientFetchAllPages] def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]: with belief_scope("get_datasets"): - log.reason("Fetching datasets") + app_logger.info("[get_datasets][Enter] Fetching datasets.") validated_query = self._validate_query_params(query) paginated_data = self._fetch_all_pages( @@ -32,14 +32,15 @@ class SupersetDatasetsMixin: }, ) total_count = len(paginated_data) - log.reflect("Datasets fetched", payload={"total_count": total_count}) + app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count) return total_count, paginated_data - # #endregion SupersetClientGetDatasets + # [/DEF:SupersetClientGetDatasets:Function] - # #region SupersetClientGetDatasetsSummary [C:3] [TYPE Function] - # @BRIEF Fetches dataset metadata optimized for the Dataset Hub grid. - # @RELATION CALLS -> [SupersetClientGetDatasets] + # [DEF:SupersetClientGetDatasetsSummary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid. + # @RELATION: CALLS -> [SupersetClientGetDatasets] def get_datasets_summary(self) -> List[Dict]: with belief_scope("SupersetClient.get_datasets_summary"): query = {"columns": ["id", "table_name", "schema", "database"]} @@ -59,12 +60,13 @@ class SupersetDatasetsMixin: ) return result - # #endregion SupersetClientGetDatasetsSummary + # [/DEF:SupersetClientGetDatasetsSummary:Function] - # #region SupersetClientGetDatasetDetail [C:3] [TYPE Function] - # @BRIEF Fetches detailed dataset information including columns and linked dashboards. - # @RELATION CALLS -> [SupersetClientGetDataset] - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDatasetDetail:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetches detailed dataset information including columns and linked dashboards. + # @RELATION: CALLS -> [SupersetClientGetDataset] + # @RELATION: CALLS -> [APIClient] def get_dataset_detail(self, dataset_id: int) -> Dict: with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"): @@ -139,7 +141,9 @@ class SupersetDatasetsMixin: {"id": dash_id, "title": f"Dashboard {dash_id}", "slug": None} ) except Exception as e: - log.explore("Failed to fetch related dashboards", error=str(e), payload={"dataset_id": dataset_id}) + app_logger.warning( + f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}" + ) linked_dashboards = [] database_obj = dataset.get("database") @@ -164,40 +168,37 @@ class SupersetDatasetsMixin: "changed_on": dataset.get("changed_on"), } - log.reflect( - "Dataset detail fetched", - payload={ - "dataset_id": dataset_id, - "column_count": len(column_info), - "linked_dashboard_count": len(linked_dashboards), - }, + app_logger.info( + f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards" ) return result - # #endregion SupersetClientGetDatasetDetail + # [/DEF:SupersetClientGetDatasetDetail:Function] - # #region SupersetClientGetDataset [C:3] [TYPE Function] - # @BRIEF Получает информацию о конкретном датасете по его ID. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientGetDataset:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Получает информацию о конкретном датасете по его ID. + # @RELATION: CALLS -> [APIClient] def get_dataset(self, dataset_id: int) -> Dict: with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"): - log.reason("Fetching dataset", payload={"dataset_id": dataset_id}) + app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id) response = self.network.request( method="GET", endpoint=f"/dataset/{dataset_id}" ) response = cast(Dict, response) - log.reflect("Dataset fetched", payload={"dataset_id": dataset_id}) + app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id) return response - # #endregion SupersetClientGetDataset + # [/DEF:SupersetClientGetDataset:Function] - # #region SupersetClientUpdateDataset [C:3] [TYPE Function] - # @BRIEF Обновляет данные датасета по его ID. - # @SIDE_EFFECT Modifies resource in upstream Superset environment. - # @RELATION CALLS -> [APIClient] + # [DEF:SupersetClientUpdateDataset:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Обновляет данные датасета по его ID. + # @SIDE_EFFECT: Modifies resource in upstream Superset environment. + # @RELATION: CALLS -> [APIClient] def update_dataset(self, dataset_id: int, data: Dict) -> Dict: with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"): - log.reason("Updating dataset", payload={"dataset_id": dataset_id}) + app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id) response = self.network.request( method="PUT", endpoint=f"/dataset/{dataset_id}", @@ -205,10 +206,11 @@ class SupersetDatasetsMixin: headers={"Content-Type": "application/json"}, ) response = cast(Dict, response) - log.reflect("Dataset updated", payload={"dataset_id": dataset_id}) + app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id) return response - # #endregion SupersetClientUpdateDataset + # [/DEF:SupersetClientUpdateDataset:Function] # #endregion SupersetDatasetsMixin +# #endregion SupersetDatasetsMixin diff --git a/backend/src/core/superset_client/_datasets_preview.py b/backend/src/core/superset_client/_datasets_preview.py index 14623fb9..0d8b616c 100644 --- a/backend/src/core/superset_client/_datasets_preview.py +++ b/backend/src/core/superset_client/_datasets_preview.py @@ -1,6 +1,6 @@ # #region SupersetDatasetsPreviewMixin [C:4] [TYPE Module] [SEMANTICS superset, datasets, preview, sql, compilation, filters] +# @LAYER: Infra # @BRIEF Dataset preview compilation mixin for SupersetClient — build query context, compile SQL. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDatasetsMixin] @@ -8,20 +8,17 @@ import json from copy import deepcopy from typing import Any, Dict, List, Optional -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope from ..utils.network import SupersetAPIError - -log = MarkerLogger("SupersetDatasetsPreview") - # #region SupersetDatasetsPreviewMixin [C:4] [TYPE Class] # @BRIEF Mixin providing dataset preview compilation and query context building. class SupersetDatasetsPreviewMixin: - # #region SupersetClientCompileDatasetPreview [C:4] [TYPE Function] - # @BRIEF Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. + # [DEF:SupersetClientCompileDatasetPreview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output. def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: with belief_scope('SupersetClientCompileDatasetPreview'): - log.reason('Compiling dataset preview SQL', payload={"dataset_id": dataset_id}) + app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview') dataset_response = self.get_dataset(dataset_id) dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {} query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or []) @@ -49,7 +46,7 @@ class SupersetDatasetsPreviewMixin: elif isinstance(request_body, dict): request_payload_keys = sorted(request_body.keys()) strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys} - log.reason('Attempting dataset preview compilation strategy', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])}) + app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])}) try: response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None) normalized = self._extract_compiled_sql_from_preview_response(response) @@ -59,20 +56,22 @@ class SupersetDatasetsPreviewMixin: normalized['endpoint_kind'] = endpoint_kind normalized['dataset_id'] = dataset_id normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}] - log.reflect('Dataset preview compilation returned normalized SQL payload', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')}) + app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')}) + app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview') return normalized except Exception as exc: failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)} strategy_attempts.append(failure_diagnostics) - log.explore('Dataset preview compilation strategy failed', payload={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}, error=str(exc)) + app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}) raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})') - # #endregion SupersetClientCompileDatasetPreview + # [/DEF:SupersetClientCompileDatasetPreview:Function] - # #region SupersetClientBuildDatasetPreviewLegacyFormData [C:4] [TYPE Function] - # @BRIEF Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. + # [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic. def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]: with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'): - log.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') + app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters) query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {}) legacy_form_data = deepcopy(query_context.get('form_data', {})) @@ -94,13 +93,14 @@ class SupersetDatasetsPreviewMixin: time_range = query_object.get('time_range') if time_range: legacy_form_data['time_range'] = time_range - log.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', payload={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})}) - log.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') + app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})}) + app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData') return legacy_form_data - # #endregion SupersetClientBuildDatasetPreviewLegacyFormData + # [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function] - # #region SupersetClientBuildDatasetPreviewQueryContext [C:4] [TYPE Function] - # @BRIEF Build a reduced-scope chart-data query context for deterministic dataset preview compilation. + # [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation. def build_dataset_preview_query_context( self, dataset_id: int, @@ -109,9 +109,9 @@ class SupersetDatasetsPreviewMixin: effective_filters: List[Dict[str, Any]], ) -> Dict[str, Any]: with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"): - log.reason( + app_logger.reason( "Building Superset dataset preview query context", - payload={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])}, + extra={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])}, ) normalized_template_params = deepcopy(template_params or {}) normalized_filter_payload = ( @@ -148,7 +148,10 @@ class SupersetDatasetsPreviewMixin: for key, value in parsed_dataset_template_params.items(): normalized_template_params.setdefault(str(key), value) except json.JSONDecodeError: - log.explore("Dataset template_params could not be parsed while building preview query context", error="Failed to parse template_params JSON", payload={"dataset_id": dataset_id}) + app_logger.explore( + "Dataset template_params could not be parsed while building preview query context", + extra={"dataset_id": dataset_id}, + ) extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data) if normalized_filters: @@ -204,9 +207,9 @@ class SupersetDatasetsPreviewMixin: "result_type": result_type, "force": True, } - log.reflect( + app_logger.reflect( "Built Superset dataset preview query context", - payload={ + extra={ "dataset_id": dataset_id, "datasource": datasource_payload, "normalized_effective_filters": normalized_filters, @@ -219,6 +222,7 @@ class SupersetDatasetsPreviewMixin: ) return payload - # #endregion SupersetClientBuildDatasetPreviewQueryContext + # [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function] # #endregion SupersetDatasetsPreviewMixin +# #endregion SupersetDatasetsPreviewMixin diff --git a/backend/src/core/superset_client/_datasets_preview_filters.py b/backend/src/core/superset_client/_datasets_preview_filters.py index 2b630d98..31cb0185 100644 --- a/backend/src/core/superset_client/_datasets_preview_filters.py +++ b/backend/src/core/superset_client/_datasets_preview_filters.py @@ -1,25 +1,24 @@ # #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, datasets, preview, filters, normalization, sql, extraction] +# @LAYER: Infra # @BRIEF Filter normalization and SQL extraction helpers for dataset preview compilation. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin] from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple -from ..cot_logger import MarkerLogger -from ..logger import belief_scope +from ..logger import logger as app_logger, belief_scope from ..utils.network import SupersetAPIError -log = MarkerLogger("SupersetDatasetsPreviewFilters") # #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class] # @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations. # @RELATION DEPENDS_ON -> [SupersetClientBase] # @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin] class SupersetDatasetsPreviewFiltersMixin: - # #region SupersetClientNormalizeEffectiveFiltersForQueryContext [C:3] [TYPE Function] - # @BRIEF Convert execution mappings into Superset chart-data filter objects. + # [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Convert execution mappings into Superset chart-data filter objects. def _normalize_effective_filters_for_query_context( self, effective_filters: List[Dict[str, Any]], @@ -98,9 +97,9 @@ class SupersetDatasetsPreviewFiltersMixin: "outgoing_clauses": deepcopy(outgoing_clauses), } ) - log.reason( + app_logger.reason( "Normalized effective preview filter for Superset query context", - payload={ + extra={ "filter_name": display_name, "used_preserved_clauses": used_preserved_clauses, "outgoing_clauses": outgoing_clauses, @@ -118,10 +117,11 @@ class SupersetDatasetsPreviewFiltersMixin: "diagnostics": diagnostics, } - # #endregion SupersetClientNormalizeEffectiveFiltersForQueryContext + # [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function] - # #region SupersetClientExtractCompiledSqlFromPreviewResponse [C:3] [TYPE Function] - # @BRIEF Normalize compiled SQL from either chart-data or legacy form_data preview responses. + # [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses. def _extract_compiled_sql_from_preview_response( self, response: Any ) -> Dict[str, Any]: @@ -191,7 +191,8 @@ class SupersetDatasetsPreviewFiltersMixin: f"(diagnostics={response_diagnostics!r})" ) - # #endregion SupersetClientExtractCompiledSqlFromPreviewResponse + # [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function] # #endregion SupersetDatasetsPreviewFiltersMixin +# #endregion SupersetDatasetsPreviewFiltersMixin diff --git a/backend/src/core/superset_client/_user_projection.py b/backend/src/core/superset_client/_user_projection.py index 540a6e00..e8f43bf9 100644 --- a/backend/src/core/superset_client/_user_projection.py +++ b/backend/src/core/superset_client/_user_projection.py @@ -1,6 +1,6 @@ # #region SupersetUserProjection [C:2] [TYPE Module] [SEMANTICS superset, users, owners, projection, normalization] +# @LAYER: Infra # @BRIEF User/owner payload normalization helpers for Superset client responses. -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetClientBase] from typing import Any, Dict, List, Optional, Union @@ -10,8 +10,9 @@ from typing import Any, Dict, List, Optional, Union # @BRIEF Mixin providing user/owner payload normalization for Superset API responses. # @RELATION DEPENDS_ON -> [SupersetClientBase] class SupersetUserProjectionMixin: - # #region SupersetClientExtractOwnerLabels [C:1] [TYPE Function] - # @BRIEF Normalize dashboard owners payload to stable display labels. + # [DEF:SupersetClientExtractOwnerLabels:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Normalize dashboard owners payload to stable display labels. def _extract_owner_labels(self, owners_payload: Any) -> List[str]: if owners_payload is None: return [] @@ -33,10 +34,11 @@ class SupersetUserProjectionMixin: normalized.append(label) return normalized - # #endregion SupersetClientExtractOwnerLabels + # [/DEF:SupersetClientExtractOwnerLabels:Function] - # #region SupersetClientExtractUserDisplay [C:1] [TYPE Function] - # @BRIEF Normalize user payload to a stable display name. + # [DEF:SupersetClientExtractUserDisplay:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Normalize user payload to a stable display name. def _extract_user_display( self, preferred_value: Optional[str], user_payload: Optional[Dict] ) -> Optional[str]: @@ -63,10 +65,11 @@ class SupersetUserProjectionMixin: return email return None - # #endregion SupersetClientExtractUserDisplay + # [/DEF:SupersetClientExtractUserDisplay:Function] - # #region SupersetClientSanitizeUserText [C:1] [TYPE Function] - # @BRIEF Convert scalar value to non-empty user-facing text. + # [DEF:SupersetClientSanitizeUserText:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Convert scalar value to non-empty user-facing text. def _sanitize_user_text(self, value: Optional[Union[str, int]]) -> Optional[str]: if value is None: return None @@ -75,7 +78,8 @@ class SupersetUserProjectionMixin: return None return normalized - # #endregion SupersetClientSanitizeUserText + # [/DEF:SupersetClientSanitizeUserText:Function] # #endregion SupersetUserProjectionMixin +# #endregion SupersetUserProjection diff --git a/backend/src/core/superset_profile_lookup.py b/backend/src/core/superset_profile_lookup.py index a66e5631..4a4ea3e5 100644 --- a/backend/src/core/superset_profile_lookup.py +++ b/backend/src/core/superset_profile_lookup.py @@ -1,20 +1,18 @@ # #region SupersetProfileLookup [C:3] [TYPE Module] [SEMANTICS superset, users, lookup, profile, pagination, normalization] +# # @BRIEF Provides environment-scoped Superset account lookup adapter with stable normalized output. -# @LAYER Core -# @INVARIANT Adapter never leaks raw upstream payload shape to API consumers. +# @LAYER: Core # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetAPIError] # @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter] # -# +# @INVARIANT: Adapter never leaks raw upstream payload shape to API consumers. -# [SECTION: IMPORTS] import json from typing import Any, Dict, List, Optional from .logger import logger, belief_scope from .utils.network import APIClient, AuthenticationError, SupersetAPIError -# [/SECTION] # #region SupersetAccountLookupAdapter [C:3] [TYPE Class] @@ -22,20 +20,20 @@ from .utils.network import APIClient, AuthenticationError, SupersetAPIError # @RELATION DEPENDS_ON -> [APIClient] # @RELATION DEPENDS_ON -> [SupersetProfileLookup] class SupersetAccountLookupAdapter: - # #region __init__ [TYPE Function] - # @BRIEF Initializes lookup adapter with authenticated API client and environment context. - # @PRE network_client supports request(method, endpoint, params=...). - # @POST Adapter is ready to perform users lookup requests. + # [DEF:__init__:Function] + # @PURPOSE: Initializes lookup adapter with authenticated API client and environment context. + # @PRE: network_client supports request(method, endpoint, params=...). + # @POST: Adapter is ready to perform users lookup requests. def __init__(self, network_client: APIClient, environment_id: str): self.network_client = network_client self.environment_id = str(environment_id or "") - # #endregion __init__ + # [/DEF:__init__:Function] - # #region get_users_page [TYPE Function] - # @BRIEF Fetch one users page from Superset with passthrough search/sort parameters. - # @PRE page_index >= 0 and page_size >= 1. - # @POST Returns deterministic payload with normalized items and total count. - # @RETURN Dict[str, Any] + # [DEF:get_users_page:Function] + # @PURPOSE: Fetch one users page from Superset with passthrough search/sort parameters. + # @PRE: page_index >= 0 and page_size >= 1. + # @POST: Returns deterministic payload with normalized items and total count. + # @RETURN: Dict[str, Any] def get_users_page( self, search: Optional[str] = None, @@ -133,13 +131,13 @@ class SupersetAccountLookupAdapter: ) raise selected_error raise SupersetAPIError("Superset users lookup failed without explicit error") - # #endregion get_users_page + # [/DEF:get_users_page:Function] - # #region _normalize_lookup_payload [TYPE Function] - # @BRIEF Convert Superset users response variants into stable candidates payload. - # @PRE response can be dict/list in any supported upstream shape. - # @POST Output contains canonical keys: status, environment_id, page_index, page_size, total, items. - # @RETURN Dict[str, Any] + # [DEF:_normalize_lookup_payload:Function] + # @PURPOSE: Convert Superset users response variants into stable candidates payload. + # @PRE: response can be dict/list in any supported upstream shape. + # @POST: Output contains canonical keys: status, environment_id, page_index, page_size, total, items. + # @RETURN: Dict[str, Any] def _normalize_lookup_payload( self, response: Any, @@ -194,13 +192,13 @@ class SupersetAccountLookupAdapter: "total": max(int(total), len(normalized_items)), "items": normalized_items, } - # #endregion _normalize_lookup_payload + # [/DEF:_normalize_lookup_payload:Function] - # #region normalize_user_payload [TYPE Function] - # @BRIEF Project raw Superset user object to canonical candidate shape. - # @PRE raw_user may have heterogenous key names between Superset versions. - # @POST Returns normalized candidate keys (environment_id, username, display_name, email, is_active). - # @RETURN Dict[str, Any] + # [DEF:normalize_user_payload:Function] + # @PURPOSE: Project raw Superset user object to canonical candidate shape. + # @PRE: raw_user may have heterogenous key names between Superset versions. + # @POST: Returns normalized candidate keys (environment_id, username, display_name, email, is_active). + # @RETURN: Dict[str, Any] def normalize_user_payload(self, raw_user: Any) -> Dict[str, Any]: if not isinstance(raw_user, dict): raw_user = {} @@ -232,7 +230,7 @@ class SupersetAccountLookupAdapter: "email": email, "is_active": is_active, } - # #endregion normalize_user_payload + # [/DEF:normalize_user_payload:Function] # #endregion SupersetAccountLookupAdapter -# #endregion SupersetProfileLookup +# #endregion SupersetProfileLookup \ No newline at end of file diff --git a/backend/src/core/task_manager/__tests__/test_context.py b/backend/src/core/task_manager/__tests__/test_context.py index 41848231..f2a5c816 100644 --- a/backend/src/core/task_manager/__tests__/test_context.py +++ b/backend/src/core/task_manager/__tests__/test_context.py @@ -1,17 +1,19 @@ -# #region TestContext [C:3] [TYPE Module] [SEMANTICS tests, task-context, background-tasks, sub-context] -# @BRIEF Verify TaskContext preserves optional background task scheduler across sub-context creation. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestContext:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, task-context, background-tasks, sub-context +# @PURPOSE: Verify TaskContext preserves optional background task scheduler across sub-context creation. from unittest.mock import MagicMock from src.core.task_manager.context import TaskContext -# #region test_task_context_preserves_background_tasks_across_sub_context [TYPE Function] -# @BRIEF Plugins must be able to access background_tasks from both root and sub-context loggers. -# @PRE TaskContext is initialized with a BackgroundTasks-like object. -# @POST background_tasks remains available on root and derived sub-contexts. -# @RELATION BINDS_TO -> [TestContext] +# [DEF:test_task_context_preserves_background_tasks_across_sub_context:Function] +# @RELATION: BINDS_TO -> TestContext +# @PURPOSE: Plugins must be able to access background_tasks from both root and sub-context loggers. +# @PRE: TaskContext is initialized with a BackgroundTasks-like object. +# @POST: background_tasks remains available on root and derived sub-contexts. def test_task_context_preserves_background_tasks_across_sub_context(): background_tasks = MagicMock() context = TaskContext( @@ -25,5 +27,5 @@ def test_task_context_preserves_background_tasks_across_sub_context(): assert context.background_tasks is background_tasks assert sub_context.background_tasks is background_tasks -# #endregion test_task_context_preserves_background_tasks_across_sub_context -# #endregion TestContext +# [/DEF:test_task_context_preserves_background_tasks_across_sub_context:Function] +# [/DEF:TestContext:Module] diff --git a/backend/src/core/task_manager/__tests__/test_task_logger.py b/backend/src/core/task_manager/__tests__/test_task_logger.py index aa22fb71..63528b45 100644 --- a/backend/src/core/task_manager/__tests__/test_task_logger.py +++ b/backend/src/core/task_manager/__tests__/test_task_logger.py @@ -1,7 +1,7 @@ -# #region __tests__/test_task_logger [TYPE Module] -# @BRIEF Contract testing for TaskLogger -# @RELATION VERIFIES -> [../task_logger.py] -# #endregion __tests__/test_task_logger +# [DEF:__tests__/test_task_logger:Module] +# @RELATION: VERIFIES -> ../task_logger.py +# @PURPOSE: Contract testing for TaskLogger +# [/DEF:__tests__/test_task_logger:Module] import pytest from unittest.mock import MagicMock @@ -17,20 +17,20 @@ def task_logger(mock_add_log): return TaskLogger(task_id="test_123", add_log_fn=mock_add_log, source="test_plugin") # @TEST_CONTRACT: TaskLoggerModel -> Invariants -# #region test_task_logger_initialization [TYPE Function] -# @BRIEF Verify TaskLogger initializes with correct task_id and state. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_task_logger_initialization:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger initializes with correct task_id and state. def test_task_logger_initialization(task_logger): """Verify TaskLogger is bound to specific task_id and source.""" assert task_logger._task_id == "test_123" assert task_logger._default_source == "test_plugin" # @TEST_CONTRACT: invariants -> "All specific log methods (info, error) delegate to _log" -# #endregion test_task_logger_initialization +# [/DEF:test_task_logger_initialization:Function] -# #region test_log_methods_delegation [TYPE Function] -# @BRIEF Verify TaskLogger delegates log method calls to the underlying persistence service. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_log_methods_delegation:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger delegates log method calls to the underlying persistence service. def test_log_methods_delegation(task_logger, mock_add_log): """Verify info, error, warning, debug delegate to internal _log.""" task_logger.info("info message", metadata={"k": "v"}) @@ -70,11 +70,11 @@ def test_log_methods_delegation(task_logger, mock_add_log): ) # @TEST_CONTRACT: invariants -> "with_source creates a new logger with the same task_id" -# #endregion test_log_methods_delegation +# [/DEF:test_log_methods_delegation:Function] -# #region test_with_source [TYPE Function] -# @BRIEF Verify TaskLogger.with_source returns a new logger with the correct source attribution. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_with_source:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger.with_source returns a new logger with the correct source attribution. def test_with_source(task_logger): """Verify with_source returns a new instance with updated default source.""" new_logger = task_logger.with_source("new_source") @@ -84,33 +84,33 @@ def test_with_source(task_logger): assert new_logger is not task_logger # @TEST_EDGE: missing_task_id -> raises TypeError -# #endregion test_with_source +# [/DEF:test_with_source:Function] -# #region test_missing_task_id [TYPE Function] -# @BRIEF Verify TaskLogger raises or handles missing task_id gracefully. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_missing_task_id:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger raises or handles missing task_id gracefully. def test_missing_task_id(): with pytest.raises(TypeError): TaskLogger(add_log_fn=lambda x: x) # @TEST_EDGE: invalid_add_log_fn -> raises TypeError # (Python doesn't strictly enforce this at init, but let's verify it fails on call if not callable) -# #endregion test_missing_task_id +# [/DEF:test_missing_task_id:Function] -# #region test_invalid_add_log_fn [TYPE Function] -# @BRIEF Verify TaskLogger raises ValueError for invalid add_log_fn parameter. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_invalid_add_log_fn:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger raises ValueError for invalid add_log_fn parameter. def test_invalid_add_log_fn(): logger = TaskLogger(task_id="msg", add_log_fn=None) with pytest.raises(TypeError): logger.info("test") # @TEST_INVARIANT: consistent_delegation -# #endregion test_invalid_add_log_fn +# [/DEF:test_invalid_add_log_fn:Function] -# #region test_progress_log [TYPE Function] -# @BRIEF Verify TaskLogger correctly logs progress updates with percentage and message. -# @RELATION BINDS_TO -> [__tests__/test_task_logger] +# [DEF:test_progress_log:Function] +# @RELATION: BINDS_TO -> __tests__/test_task_logger +# @PURPOSE: Verify TaskLogger correctly logs progress updates with percentage and message. def test_progress_log(task_logger, mock_add_log): """Verify progress method correctly formats metadata.""" task_logger.progress("Step 1", 45.5) @@ -128,4 +128,4 @@ def test_progress_log(task_logger, mock_add_log): task_logger.progress("Step low", -10) assert mock_add_log.call_args[1]["metadata"]["progress"] == 0 -# #endregion test_progress_log +# [/DEF:test_progress_log:Function] diff --git a/backend/src/core/task_manager/cleanup.py b/backend/src/core/task_manager/cleanup.py index 57308829..63d749c5 100644 --- a/backend/src/core/task_manager/cleanup.py +++ b/backend/src/core/task_manager/cleanup.py @@ -1,6 +1,6 @@ # #region TaskCleanupModule [C:3] [TYPE Module] [SEMANTICS task, cleanup, retention, logs] # @BRIEF Implements task cleanup and retention policies, including associated logs. -# @LAYER Core +# @LAYER: Core # @RELATION DEPENDS_ON -> [TaskPersistenceService] # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] # @RELATION DEPENDS_ON -> [ConfigManager] @@ -16,10 +16,10 @@ from ..config_manager import ConfigManager # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] # @RELATION DEPENDS_ON -> [ConfigManager] class TaskCleanupService: - # #region __init__ [TYPE Function] - # @BRIEF Initializes the cleanup service with dependencies. - # @PRE persistence_service and config_manager are valid. - # @POST Cleanup service is ready. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the cleanup service with dependencies. + # @PRE: persistence_service and config_manager are valid. + # @POST: Cleanup service is ready. def __init__( self, persistence_service: TaskPersistenceService, @@ -29,12 +29,12 @@ class TaskCleanupService: self.persistence_service = persistence_service self.log_persistence_service = log_persistence_service self.config_manager = config_manager - # #endregion __init__ + # [/DEF:__init__:Function] - # #region run_cleanup [TYPE Function] - # @BRIEF Deletes tasks older than the configured retention period and their logs. - # @PRE Config manager has valid settings. - # @POST Old tasks and their logs are deleted from persistence. + # [DEF:run_cleanup:Function] + # @PURPOSE: Deletes tasks older than the configured retention period and their logs. + # @PRE: Config manager has valid settings. + # @POST: Old tasks and their logs are deleted from persistence. def run_cleanup(self): with belief_scope("TaskCleanupService.run_cleanup"): settings = self.config_manager.get_config().settings @@ -54,12 +54,12 @@ class TaskCleanupService: self.persistence_service.delete_tasks(to_delete) logger.info(f"Deleted {len(to_delete)} tasks and their logs exceeding limit of {settings.task_retention_limit}") - # #endregion run_cleanup + # [/DEF:run_cleanup:Function] - # #region delete_task_with_logs [TYPE Function] - # @BRIEF Delete a single task and all its associated logs. - # @PRE task_id is a valid task ID. - # @POST Task and all its logs are deleted. + # [DEF:delete_task_with_logs:Function] + # @PURPOSE: Delete a single task and all its associated logs. + # @PRE: task_id is a valid task ID. + # @POST: Task and all its logs are deleted. # @PARAM: task_id (str) - The task ID to delete. def delete_task_with_logs(self, task_id: str) -> None: """Delete a single task and all its associated logs.""" @@ -71,7 +71,7 @@ class TaskCleanupService: self.persistence_service.delete_tasks([task_id]) logger.info(f"Deleted task {task_id} and its associated logs") - # #endregion delete_task_with_logs + # [/DEF:delete_task_with_logs:Function] # #endregion TaskCleanupService -# #endregion TaskCleanupModule +# #endregion TaskCleanupModule \ No newline at end of file diff --git a/backend/src/core/task_manager/context.py b/backend/src/core/task_manager/context.py index 5ef8f101..4ba2453f 100644 --- a/backend/src/core/task_manager/context.py +++ b/backend/src/core/task_manager/context.py @@ -1,30 +1,27 @@ # #region TaskContextModule [C:5] [TYPE Module] [SEMANTICS task, context, plugin, execution, logger] # @BRIEF Provides execution context passed to plugins during task execution. -# @LAYER Core -# @PRE Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries. -# @POST Plugins receive context instances with stable logger and parameter accessors. -# @SIDE_EFFECT Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts. -# @DATA_CONTRACT Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] -# @INVARIANT Each TaskContext is bound to a single task execution. +# @LAYER: Core # @RELATION DEPENDS_ON -> [TaskLoggerModule] # @RELATION DEPENDS_ON -> [TaskManager] +# @INVARIANT: Each TaskContext is bound to a single task execution. +# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries. +# @POST: Plugins receive context instances with stable logger and parameter accessors. +# @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts. +# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] -# [SECTION: IMPORTS] -# [SECTION: IMPORTS] from typing import Dict, Any, Callable, Optional from .task_logger import TaskLogger from ..logger import belief_scope -# [/SECTION] # #region TaskContext [C:5] [TYPE Class] [SEMANTICS context, task, execution, plugin] # @BRIEF A container passed to plugin.execute() providing the logger and other task-specific utilities. -# @PRE Constructor receives non-empty task_id, callable add_log_fn, and params mapping. -# @POST Instance exposes immutable task identity with logger, params, and optional background task access. -# @SIDE_EFFECT Emits structured task logs through TaskLogger on plugin interactions. -# @DATA_CONTRACT Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] -# @INVARIANT logger is always a valid TaskLogger instance. +# @INVARIANT: logger is always a valid TaskLogger instance. +# @PRE: Constructor receives non-empty task_id, callable add_log_fn, and params mapping. +# @POST: Instance exposes immutable task identity with logger, params, and optional background task access. # @RELATION DEPENDS_ON -> [TaskLogger] +# @SIDE_EFFECT: Emits structured task logs through TaskLogger on plugin interactions. +# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext] # @UX_STATE: Idle -> Active -> Complete # # @TEST_CONTRACT: TaskContextContract -> @@ -52,10 +49,10 @@ class TaskContext: # ... plugin logic """ - # #region __init__ [TYPE Function] - # @BRIEF Initialize the TaskContext with task-specific resources. - # @PRE task_id is a valid task identifier, add_log_fn is callable. - # @POST TaskContext is ready to be passed to plugin.execute(). + # [DEF:__init__:Function] + # @PURPOSE: Initialize the TaskContext with task-specific resources. + # @PRE: task_id is a valid task identifier, add_log_fn is callable. + # @POST: TaskContext is ready to be passed to plugin.execute(). # @PARAM: task_id (str) - The ID of the task. # @PARAM: add_log_fn (Callable) - Function to add log to TaskManager. # @PARAM: params (Dict) - Task parameters. @@ -76,59 +73,59 @@ class TaskContext: task_id=task_id, add_log_fn=add_log_fn, source=default_source ) - # #endregion __init__ + # [/DEF:__init__:Function] - # #region task_id [TYPE Function] - # @BRIEF Get the task ID. - # @PRE TaskContext must be initialized. - # @POST Returns the task ID string. - # @RETURN str - The task ID. + # [DEF:task_id:Function] + # @PURPOSE: Get the task ID. + # @PRE: TaskContext must be initialized. + # @POST: Returns the task ID string. + # @RETURN: str - The task ID. @property def task_id(self) -> str: with belief_scope("task_id"): return self._task_id - # #endregion task_id + # [/DEF:task_id:Function] - # #region logger [TYPE Function] - # @BRIEF Get the TaskLogger instance for this context. - # @PRE TaskContext must be initialized. - # @POST Returns the TaskLogger instance. - # @RETURN TaskLogger - The logger instance. + # [DEF:logger:Function] + # @PURPOSE: Get the TaskLogger instance for this context. + # @PRE: TaskContext must be initialized. + # @POST: Returns the TaskLogger instance. + # @RETURN: TaskLogger - The logger instance. @property def logger(self) -> TaskLogger: with belief_scope("logger"): return self._logger - # #endregion logger + # [/DEF:logger:Function] - # #region params [TYPE Function] - # @BRIEF Get the task parameters. - # @PRE TaskContext must be initialized. - # @POST Returns the parameters dictionary. - # @RETURN Dict[str, Any] - The task parameters. + # [DEF:params:Function] + # @PURPOSE: Get the task parameters. + # @PRE: TaskContext must be initialized. + # @POST: Returns the parameters dictionary. + # @RETURN: Dict[str, Any] - The task parameters. @property def params(self) -> Dict[str, Any]: with belief_scope("params"): return self._params - # #endregion params + # [/DEF:params:Function] - # #region background_tasks [TYPE Function] - # @BRIEF Expose optional background task scheduler for plugins that dispatch deferred side effects. - # @PRE TaskContext must be initialized. - # @POST Returns BackgroundTasks-like object or None. + # [DEF:background_tasks:Function] + # @PURPOSE: Expose optional background task scheduler for plugins that dispatch deferred side effects. + # @PRE: TaskContext must be initialized. + # @POST: Returns BackgroundTasks-like object or None. @property def background_tasks(self) -> Optional[Any]: with belief_scope("background_tasks"): return self._background_tasks - # #endregion background_tasks + # [/DEF:background_tasks:Function] - # #region get_param [TYPE Function] - # @BRIEF Get a specific parameter value with optional default. - # @PRE TaskContext must be initialized. - # @POST Returns parameter value or default. + # [DEF:get_param:Function] + # @PURPOSE: Get a specific parameter value with optional default. + # @PRE: TaskContext must be initialized. + # @POST: Returns parameter value or default. # @PARAM: key (str) - Parameter key. # @PARAM: default (Any) - Default value if key not found. # @RETURN: Any - Parameter value or default. @@ -136,12 +133,12 @@ class TaskContext: with belief_scope("get_param"): return self._params.get(key, default) - # #endregion get_param + # [/DEF:get_param:Function] - # #region create_sub_context [TYPE Function] - # @BRIEF Create a sub-context with a different default source. - # @PRE source is a non-empty string. - # @POST Returns new TaskContext with different logger source. + # [DEF:create_sub_context:Function] + # @PURPOSE: Create a sub-context with a different default source. + # @PRE: source is a non-empty string. + # @POST: Returns new TaskContext with different logger source. # @PARAM: source (str) - New default source for logging. # @RETURN: TaskContext - New context with different source. def create_sub_context(self, source: str) -> "TaskContext": @@ -155,7 +152,7 @@ class TaskContext: background_tasks=self._background_tasks, ) - # #endregion create_sub_context + # [/DEF:create_sub_context:Function] # #endregion TaskContext diff --git a/backend/src/core/task_manager/manager.py b/backend/src/core/task_manager/manager.py index c46225f0..746b0f84 100644 --- a/backend/src/core/task_manager/manager.py +++ b/backend/src/core/task_manager/manager.py @@ -1,11 +1,10 @@ # #region TaskManagerModule [C:5] [TYPE Module] [SEMANTICS task, manager, lifecycle, execution, state] # @BRIEF Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously. -# @LAYER Core -# @PRE Plugin loader and database sessions are initialized. -# @POST Orchestrates task execution and persistence. -# @SIDE_EFFECT Spawns worker threads and flushes logs to DB. -# @DATA_CONTRACT Input[plugin_id, params] -> Model[Task, LogEntry] -# @INVARIANT Task IDs are unique. +# @LAYER: Core +# @PRE: Plugin loader and database sessions are initialized. +# @POST: Orchestrates task execution and persistence. +# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB. +# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry] # @RELATION DEPENDS_ON -> [PluginLoader] # @RELATION DEPENDS_ON -> [TaskPersistenceService] # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] @@ -13,6 +12,7 @@ # @RELATION DEPENDS_ON -> [TaskGraph] # @RELATION DEPENDS_ON -> [JobLifecycle] # @RELATION DEPENDS_ON -> [EventBus] +# @INVARIANT: Task IDs are unique. # @TEST_CONTRACT: TaskManagerRuntime -> { # required_fields: {plugin_loader: PluginLoader}, # optional_fields: {}, @@ -25,7 +25,6 @@ # @TEST_EDGE: external_failure -> {"db_unavailable": true} # @TEST_INVARIANT: logger_compliance -> verifies: [valid_module] -# [SECTION: IMPORTS] import asyncio import threading import inspect @@ -37,22 +36,11 @@ from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats from .persistence import TaskPersistenceService, TaskLogPersistenceService from .context import TaskContext from ..logger import logger, belief_scope, should_log_task_level -from ..cot_logger import MarkerLogger, get_trace_id, set_trace_id - -log = MarkerLogger("TaskManager") -# [/SECTION] # #region TaskManager [C:5] [TYPE Class] [SEMANTICS task, manager, lifecycle, execution, state] # @BRIEF Manages the lifecycle of tasks, including their creation, execution, and state tracking. -# @LAYER Core -# @PRE Plugin loader resolves plugin ids and persistence services are available for task state hydration. -# @POST In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state. -# @SIDE_EFFECT Spawns worker threads, flushes logs to database, and mutates task states. -# @DATA_CONTRACT Input[plugin_id, params] -> Output[Task] -# @INVARIANT Task IDs are unique within the registry. -# @INVARIANT Each task has exactly one status at any time. -# @INVARIANT Log entries are never deleted after being added to a task. +# @LAYER: Core # @RELATION DEPENDS_ON -> [TaskPersistenceService] # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] # @RELATION DEPENDS_ON -> [PluginLoader] @@ -60,6 +48,13 @@ log = MarkerLogger("TaskManager") # @RELATION DEPENDS_ON -> [TaskGraph] # @RELATION DEPENDS_ON -> [JobLifecycle] # @RELATION DEPENDS_ON -> [EventBus] +# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration. +# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state. +# @INVARIANT: Task IDs are unique within the registry. +# @INVARIANT: Each task has exactly one status at any time. +# @INVARIANT: Log entries are never deleted after being added to a task. +# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states. +# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task] class TaskManager: """ Manages the lifecycle of tasks, including their creation, execution, and state tracking. @@ -68,53 +63,57 @@ class TaskManager: # Log flush interval in seconds LOG_FLUSH_INTERVAL = 2.0 - # #region TaskGraph [C:5] [TYPE Block] - # @BRIEF Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration. - # @PRE Task ids are generated before insertion and persisted tasks can be reconstructed into Task models. - # @POST Each live task id resolves to at most one active Task node and optional pause future. - # @SIDE_EFFECT Mutates the in-memory task registry and loads persisted state during manager startup. - # @DATA_CONTRACT Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]] - # @INVARIANT Registry membership is keyed by unique task id and survives log streaming side channels. - # @RELATION DEPENDS_ON -> [Task] - # @RELATION DEPENDS_ON -> [TaskPersistenceService] - # #endregion TaskGraph + # [DEF:TaskGraph:Block] + # @COMPLEXITY: 5 + # @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration. + # @RELATION: [DEPENDS_ON] ->[Task] + # @RELATION: [DEPENDS_ON] ->[TaskPersistenceService] + # @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models. + # @POST: Each live task id resolves to at most one active Task node and optional pause future. + # @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup. + # @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]] + # @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels. + # [/DEF:TaskGraph:Block] - # #region EventBus [C:5] [TYPE Block] - # @BRIEF Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers. - # @PRE Task log records accept structured LogEntry payloads and subscribers consume asyncio queues. - # @POST Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order. - # @SIDE_EFFECT Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues. - # @DATA_CONTRACT Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events] - # @INVARIANT Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events. - # @RELATION DEPENDS_ON -> [LogEntry] - # @RELATION DEPENDS_ON -> [TaskLogPersistenceService] - # #endregion EventBus + # [DEF:EventBus:Block] + # @COMPLEXITY: 5 + # @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers. + # @RELATION: [DEPENDS_ON] ->[LogEntry] + # @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService] + # @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues. + # @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order. + # @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues. + # @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events] + # @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events. + # [/DEF:EventBus:Block] - # #region JobLifecycle [C:5] [TYPE Block] - # @BRIEF Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs. - # @PRE Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values. - # @POST Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state. - # @SIDE_EFFECT Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers. - # @DATA_CONTRACT Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result] - # @INVARIANT A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created. - # @RELATION DEPENDS_ON -> [TaskGraph] - # @RELATION DEPENDS_ON -> [EventBus] - # @RELATION DEPENDS_ON -> [TaskContext] - # @RELATION DEPENDS_ON -> [PluginLoader] - # #endregion JobLifecycle + # [DEF:JobLifecycle:Block] + # @COMPLEXITY: 5 + # @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs. + # @RELATION: [DEPENDS_ON] ->[TaskGraph] + # @RELATION: [DEPENDS_ON] ->[EventBus] + # @RELATION: [DEPENDS_ON] ->[TaskContext] + # @RELATION: [DEPENDS_ON] ->[PluginLoader] + # @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values. + # @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state. + # @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers. + # @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result] + # @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created. + # [/DEF:JobLifecycle:Block] - # #region __init__ [C:5] [TYPE Function] - # @BRIEF Initialize the TaskManager with dependencies. - # @PRE plugin_loader is initialized. - # @POST TaskManager is ready to accept tasks. - # @SIDE_EFFECT Starts background flusher thread and loads persisted task state into memory. - # @RELATION CALLS -> [TaskPersistenceService.load_tasks] - # @RELATION DEPENDS_ON -> [TaskGraph] - # @RELATION DEPENDS_ON -> [EventBus] + # [DEF:__init__:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Initialize the TaskManager with dependencies. + # @PRE: plugin_loader is initialized. + # @POST: TaskManager is ready to accept tasks. + # @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory. + # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks] + # @RELATION: [DEPENDS_ON] ->[TaskGraph] + # @RELATION: [DEPENDS_ON] ->[EventBus] # @PARAM: plugin_loader - The plugin loader instance. def __init__(self, plugin_loader): with belief_scope("TaskManager.__init__"): - log.reason("Initializing task manager runtime services") + logger.reason("Initializing task manager runtime services") self.plugin_loader = plugin_loader self.tasks: Dict[str, Task] = {} self.subscribers: Dict[str, List[asyncio.Queue]] = {} @@ -143,33 +142,35 @@ class TaskManager: # Load persisted tasks on startup self.load_persisted_tasks() - log.reflect( + logger.reflect( "Task manager runtime initialized", - payload={"task_count": len(self.tasks)}, + extra={"task_count": len(self.tasks)}, ) - # #endregion __init__ + # [/DEF:__init__:Function] - # #region _flusher_loop [C:3] [TYPE Function] - # @BRIEF Background thread that periodically flushes log buffer to database. - # @PRE TaskManager is initialized. - # @POST Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds. - # @RELATION CALLS -> [TaskManager._flush_logs] - # @RELATION DEPENDS_ON -> [EventBus] + # [DEF:_flusher_loop:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Background thread that periodically flushes log buffer to database. + # @PRE: TaskManager is initialized. + # @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds. + # @RELATION: [CALLS] ->[TaskManager._flush_logs] + # @RELATION: [DEPENDS_ON] ->[EventBus] def _flusher_loop(self): """Background thread that flushes log buffer to database.""" while not self._flusher_stop_event.is_set(): self._flush_logs() self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL) - # #endregion _flusher_loop + # [/DEF:_flusher_loop:Function] - # #region _flush_logs [C:3] [TYPE Function] - # @BRIEF Flush all buffered logs to the database. - # @PRE None. - # @POST All buffered logs are written to task_logs table. - # @RELATION CALLS -> [TaskLogPersistenceService.add_logs] - # @RELATION DEPENDS_ON -> [EventBus] + # [DEF:_flush_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Flush all buffered logs to the database. + # @PRE: None. + # @POST: All buffered logs are written to task_logs table. + # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs] + # @RELATION: [DEPENDS_ON] ->[EventBus] def _flush_logs(self): """Flush all buffered logs to the database.""" with self._log_buffer_lock: @@ -182,9 +183,8 @@ class TaskManager: if logs: try: self.log_persistence_service.add_logs(task_id, logs) - log.reflect("Flushed logs to database", payload={"task_id": task_id, "count": len(logs)}) + logger.debug(f"Flushed {len(logs)} logs for task {task_id}") except Exception as e: - log.explore("Failed to flush logs to database", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to flush logs for task {task_id}: {e}") # Re-add logs to buffer on failure with self._log_buffer_lock: @@ -192,12 +192,13 @@ class TaskManager: self._log_buffer[task_id] = [] self._log_buffer[task_id].extend(logs) - # #endregion _flush_logs + # [/DEF:_flush_logs:Function] - # #region _flush_task_logs [C:3] [TYPE Function] - # @BRIEF Flush logs for a specific task immediately. - # @PRE task_id exists. - # @POST Task's buffered logs are written to database. + # [DEF:_flush_task_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Flush logs for a specific task immediately. + # @PRE: task_id exists. + # @POST: Task's buffered logs are written to database. # @PARAM: task_id (str) - The task ID. # @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs] # @RELATION: [DEPENDS_ON] ->[EventBus] @@ -210,17 +211,16 @@ class TaskManager: if logs: try: self.log_persistence_service.add_logs(task_id, logs) - log.reflect("Flushed task logs immediately", payload={"task_id": task_id, "count": len(logs)}) except Exception as e: - log.explore("Failed to flush task logs immediately", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to flush logs for task {task_id}: {e}") - # #endregion _flush_task_logs + # [/DEF:_flush_task_logs:Function] - # #region create_task [C:3] [TYPE Function] - # @BRIEF Creates and queues a new task for execution. - # @PRE Plugin with plugin_id exists. Params are valid. - # @POST Task is created, added to registry, and scheduled for execution. + # [DEF:create_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Creates and queues a new task for execution. + # @PRE: Plugin with plugin_id exists. Params are valid. + # @POST: Task is created, added to registry, and scheduled for execution. # @PARAM: plugin_id (str) - The ID of the plugin to run. # @PARAM: params (Dict[str, Any]) - Parameters for the plugin. # @PARAM: user_id (Optional[str]) - ID of the user requesting the task. @@ -234,50 +234,52 @@ class TaskManager: ) -> Task: with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"): if not self.plugin_loader.has_plugin(plugin_id): - log.explore("Plugin not found", error=f"Plugin with ID '{plugin_id}' not found.") + logger.error(f"Plugin with ID '{plugin_id}' not found.") raise ValueError(f"Plugin with ID '{plugin_id}' not found.") self.plugin_loader.get_plugin(plugin_id) if not isinstance(params, dict): - log.explore("Invalid task params type", error="Task parameters must be a dictionary.") + logger.error("Task parameters must be a dictionary.") raise ValueError("Task parameters must be a dictionary.") - log.reason("Creating task node and scheduling execution", payload={"plugin_id": plugin_id}) + logger.reason("Creating task node and scheduling execution") task = Task(plugin_id=plugin_id, params=params, user_id=user_id) self.tasks[task.id] = task self.persistence_service.persist_task(task) - trace_id = get_trace_id() + logger.info(f"Task {task.id} created and scheduled for execution") self.loop.create_task( - self._run_task(task.id, trace_id=trace_id) + self._run_task(task.id) ) # Schedule task for execution - log.reflect( + logger.reflect( "Task creation persisted and execution scheduled", - payload={"task_id": task.id, "plugin_id": plugin_id}, + extra={"task_id": task.id, "plugin_id": plugin_id}, ) return task - # #endregion create_task + # [/DEF:create_task:Function] - # #region _run_task [C:3] [TYPE Function] - # @BRIEF Internal method to execute a task with TaskContext support. - # @PRE Task exists in registry. - # @POST Task is executed, status updated to SUCCESS or FAILED. + # [DEF:_run_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Internal method to execute a task with TaskContext support. + # @PRE: Task exists in registry. + # @POST: Task is executed, status updated to SUCCESS or FAILED. # @PARAM: task_id (str) - The ID of the task to run. # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] # @RELATION: [DEPENDS_ON] ->[JobLifecycle] # @RELATION: [DEPENDS_ON] ->[TaskContext] # @RELATION: [DEPENDS_ON] ->[EventBus] - async def _run_task(self, task_id: str, trace_id: str = None): - if trace_id: - set_trace_id(trace_id) + async def _run_task(self, task_id: str): with belief_scope("TaskManager._run_task", f"task_id={task_id}"): task = self.tasks[task_id] plugin = self.plugin_loader.get_plugin(task.plugin_id) - log.reason( + logger.reason( "Transitioning task to running state", - payload={"task_id": task_id, "plugin_id": task.plugin_id}, + extra={"task_id": task_id, "plugin_id": task.plugin_id}, + ) + logger.info( + f"Starting execution of task {task_id} for plugin '{plugin.name}'" ) task.status = TaskStatus.RUNNING task.started_at = datetime.utcnow() @@ -323,6 +325,7 @@ class TaskManager: self.executor, plugin.execute, params ) + logger.info(f"Task {task_id} completed successfully") task.status = TaskStatus.SUCCESS self._add_log( task_id, @@ -330,8 +333,8 @@ class TaskManager: f"Task completed successfully for plugin '{plugin.name}'", source="system", ) - log.reflect("Task completed successfully", payload={"task_id": task_id, "plugin_id": task.plugin_id}) except Exception as e: + logger.error(f"Task {task_id} failed: {e}") task.status = TaskStatus.FAILED self._add_log( task_id, @@ -340,23 +343,26 @@ class TaskManager: source="system", metadata={"error_type": type(e).__name__}, ) - log.explore("Task failed during execution", error=str(e), payload={"task_id": task_id, "plugin_id": task.plugin_id}) finally: task.finished_at = datetime.utcnow() # Flush any remaining buffered logs before persisting task self._flush_task_logs(task_id) self.persistence_service.persist_task(task) - log.reflect( - "Task execution finished", - payload={"task_id": task_id, "status": str(task.status)}, + logger.info( + f"Task {task_id} execution finished with status: {task.status}" + ) + logger.reflect( + "Task lifecycle reached persisted terminal state", + extra={"task_id": task_id, "status": str(task.status)}, ) - # #endregion _run_task + # [/DEF:_run_task:Function] - # #region resolve_task [C:3] [TYPE Function] - # @BRIEF Resumes a task that is awaiting mapping. - # @PRE Task exists and is in AWAITING_MAPPING state. - # @POST Task status updated to RUNNING, params updated, execution resumed. + # [DEF:resolve_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Resumes a task that is awaiting mapping. + # @PRE: Task exists and is in AWAITING_MAPPING state. + # @POST: Task status updated to RUNNING, params updated, execution resumed. # @PARAM: task_id (str) - The ID of the task. # @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait. # @THROWS: ValueError if task not found or not awaiting mapping. @@ -378,12 +384,13 @@ class TaskManager: if task_id in self.task_futures: self.task_futures[task_id].set_result(True) - # #endregion resolve_task + # [/DEF:resolve_task:Function] - # #region wait_for_resolution [C:3] [TYPE Function] - # @BRIEF Pauses execution and waits for a resolution signal. - # @PRE Task exists. - # @POST Execution pauses until future is set. + # [DEF:wait_for_resolution:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Pauses execution and waits for a resolution signal. + # @PRE: Task exists. + # @POST: Execution pauses until future is set. # @PARAM: task_id (str) - The ID of the task. # @RELATION: [CALLS] ->[TaskPersistenceService.persist_task] # @RELATION: [DEPENDS_ON] ->[JobLifecycle] @@ -403,12 +410,13 @@ class TaskManager: if task_id in self.task_futures: del self.task_futures[task_id] - # #endregion wait_for_resolution + # [/DEF:wait_for_resolution:Function] - # #region wait_for_input [C:3] [TYPE Function] - # @BRIEF Pauses execution and waits for user input. - # @PRE Task exists. - # @POST Execution pauses until future is set via resume_task_with_password. + # [DEF:wait_for_input:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Pauses execution and waits for user input. + # @PRE: Task exists. + # @POST: Execution pauses until future is set via resume_task_with_password. # @PARAM: task_id (str) - The ID of the task. # @RELATION: [DEPENDS_ON] ->[JobLifecycle] async def wait_for_input(self, task_id: str): @@ -426,12 +434,13 @@ class TaskManager: if task_id in self.task_futures: del self.task_futures[task_id] - # #endregion wait_for_input + # [/DEF:wait_for_input:Function] - # #region get_task [C:3] [TYPE Function] - # @BRIEF Retrieves a task by its ID. - # @PRE task_id is a string. - # @POST Returns Task object or None. + # [DEF:get_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Retrieves a task by its ID. + # @PRE: task_id is a string. + # @POST: Returns Task object or None. # @PARAM: task_id (str) - ID of the task. # @RETURN: Optional[Task] - The task or None. # @RELATION: [DEPENDS_ON] ->[TaskGraph] @@ -439,24 +448,26 @@ class TaskManager: with belief_scope("TaskManager.get_task", f"task_id={task_id}"): return self.tasks.get(task_id) - # #endregion get_task + # [/DEF:get_task:Function] - # #region get_all_tasks [C:3] [TYPE Function] - # @BRIEF Retrieves all registered tasks. - # @PRE None. - # @POST Returns list of all Task objects. - # @RETURN List[Task] - All tasks. - # @RELATION DEPENDS_ON -> [TaskGraph] + # [DEF:get_all_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Retrieves all registered tasks. + # @PRE: None. + # @POST: Returns list of all Task objects. + # @RETURN: List[Task] - All tasks. + # @RELATION: [DEPENDS_ON] ->[TaskGraph] def get_all_tasks(self) -> List[Task]: with belief_scope("TaskManager.get_all_tasks"): return list(self.tasks.values()) - # #endregion get_all_tasks + # [/DEF:get_all_tasks:Function] - # #region get_tasks [C:3] [TYPE Function] - # @BRIEF Retrieves tasks with pagination and optional status filter. - # @PRE limit and offset are non-negative integers. - # @POST Returns a list of tasks sorted by start_time descending. + # [DEF:get_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Retrieves tasks with pagination and optional status filter. + # @PRE: limit and offset are non-negative integers. + # @POST: Returns a list of tasks sorted by start_time descending. # @PARAM: limit (int) - Maximum number of tasks to return. # @PARAM: offset (int) - Number of tasks to skip. # @PARAM: status (Optional[TaskStatus]) - Filter by task status. @@ -496,12 +507,13 @@ class TaskManager: tasks.sort(key=sort_key, reverse=True) return tasks[offset : offset + limit] - # #endregion get_tasks + # [/DEF:get_tasks:Function] - # #region get_task_logs [C:3] [TYPE Function] - # @BRIEF Retrieves logs for a specific task (from memory for running, persistence for completed). - # @PRE task_id is a string. - # @POST Returns list of LogEntry or TaskLog objects. + # [DEF:get_task_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed). + # @PRE: task_id is a string. + # @POST: Returns list of LogEntry or TaskLog objects. # @PARAM: task_id (str) - ID of the task. # @PARAM: log_filter (Optional[LogFilter]) - Filter parameters. # @RETURN: List[LogEntry] - List of log entries. @@ -533,12 +545,13 @@ class TaskManager: # For running/pending tasks, return from memory return task.logs if task else [] - # #endregion get_task_logs + # [/DEF:get_task_logs:Function] - # #region get_task_log_stats [C:3] [TYPE Function] - # @BRIEF Get statistics about logs for a task. - # @PRE task_id is a valid task ID. - # @POST Returns LogStats with counts by level and source. + # [DEF:get_task_log_stats:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get statistics about logs for a task. + # @PRE: task_id is a valid task ID. + # @POST: Returns LogStats with counts by level and source. # @PARAM: task_id (str) - The task ID. # @RETURN: LogStats - Statistics about task logs. # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats] @@ -547,12 +560,13 @@ class TaskManager: with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"): return self.log_persistence_service.get_log_stats(task_id) - # #endregion get_task_log_stats + # [/DEF:get_task_log_stats:Function] - # #region get_task_log_sources [C:3] [TYPE Function] - # @BRIEF Get unique sources for a task's logs. - # @PRE task_id is a valid task ID. - # @POST Returns list of unique source strings. + # [DEF:get_task_log_sources:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get unique sources for a task's logs. + # @PRE: task_id is a valid task ID. + # @POST: Returns list of unique source strings. # @PARAM: task_id (str) - The task ID. # @RETURN: List[str] - Unique source names. # @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources] @@ -561,12 +575,13 @@ class TaskManager: with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"): return self.log_persistence_service.get_sources(task_id) - # #endregion get_task_log_sources + # [/DEF:get_task_log_sources:Function] - # #region _add_log [C:3] [TYPE Function] - # @BRIEF Adds a log entry to a task buffer and notifies subscribers. - # @PRE Task exists. - # @POST Log added to buffer and pushed to queues (if level meets task_log_level filter). + # [DEF:_add_log:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Adds a log entry to a task buffer and notifies subscribers. + # @PRE: Task exists. + # @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter). # @PARAM: task_id (str) - ID of the task. # @PARAM: level (str) - Log level. # @PARAM: message (str) - Log message. @@ -616,12 +631,13 @@ class TaskManager: for queue in self.subscribers[task_id]: self.loop.call_soon_threadsafe(queue.put_nowait, log_entry) - # #endregion _add_log + # [/DEF:_add_log:Function] - # #region subscribe_logs [C:3] [TYPE Function] - # @BRIEF Subscribes to real-time logs for a task. - # @PRE task_id is a string. - # @POST Returns an asyncio.Queue for log entries. + # [DEF:subscribe_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Subscribes to real-time logs for a task. + # @PRE: task_id is a string. + # @POST: Returns an asyncio.Queue for log entries. # @PARAM: task_id (str) - ID of the task. # @RETURN: asyncio.Queue - Queue for log entries. # @RELATION: [DEPENDS_ON] ->[EventBus] @@ -633,12 +649,13 @@ class TaskManager: self.subscribers[task_id].append(queue) return queue - # #endregion subscribe_logs + # [/DEF:subscribe_logs:Function] - # #region unsubscribe_logs [C:3] [TYPE Function] - # @BRIEF Unsubscribes from real-time logs for a task. - # @PRE task_id is a string, queue is asyncio.Queue. - # @POST Queue removed from subscribers. + # [DEF:unsubscribe_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Unsubscribes from real-time logs for a task. + # @PRE: task_id is a string, queue is asyncio.Queue. + # @POST: Queue removed from subscribers. # @PARAM: task_id (str) - ID of the task. # @PARAM: queue (asyncio.Queue) - Queue to remove. # @RELATION: [DEPENDS_ON] ->[EventBus] @@ -650,14 +667,15 @@ class TaskManager: if not self.subscribers[task_id]: del self.subscribers[task_id] - # #endregion unsubscribe_logs + # [/DEF:unsubscribe_logs:Function] - # #region load_persisted_tasks [C:3] [TYPE Function] - # @BRIEF Load persisted tasks using persistence service. - # @PRE None. - # @POST Persisted tasks loaded into self.tasks. - # @RELATION CALLS -> [TaskPersistenceService.load_tasks] - # @RELATION DEPENDS_ON -> [TaskGraph] + # [DEF:load_persisted_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Load persisted tasks using persistence service. + # @PRE: None. + # @POST: Persisted tasks loaded into self.tasks. + # @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks] + # @RELATION: [DEPENDS_ON] ->[TaskGraph] def load_persisted_tasks(self) -> None: with belief_scope("TaskManager.load_persisted_tasks"): loaded_tasks = self.persistence_service.load_tasks(limit=100) @@ -665,12 +683,13 @@ class TaskManager: if task.id not in self.tasks: self.tasks[task.id] = task - # #endregion load_persisted_tasks + # [/DEF:load_persisted_tasks:Function] - # #region await_input [C:3] [TYPE Function] - # @BRIEF Transition a task to AWAITING_INPUT state with input request. - # @PRE Task exists and is in RUNNING state. - # @POST Task status changed to AWAITING_INPUT, input_request set, persisted. + # [DEF:await_input:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Transition a task to AWAITING_INPUT state with input request. + # @PRE: Task exists and is in RUNNING state. + # @POST: Task status changed to AWAITING_INPUT, input_request set, persisted. # @PARAM: task_id (str) - ID of the task. # @PARAM: input_request (Dict) - Details about required input. # @THROWS: ValueError if task not found or not RUNNING. @@ -697,12 +716,13 @@ class TaskManager: metadata={"input_request": input_request}, ) - # #endregion await_input + # [/DEF:await_input:Function] - # #region resume_task_with_password [C:3] [TYPE Function] - # @BRIEF Resume a task that is awaiting input with provided passwords. - # @PRE Task exists and is in AWAITING_INPUT state. - # @POST Task status changed to RUNNING, passwords injected, task resumed. + # [DEF:resume_task_with_password:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Resume a task that is awaiting input with provided passwords. + # @PRE: Task exists and is in AWAITING_INPUT state. + # @POST: Task status changed to RUNNING, passwords injected, task resumed. # @PARAM: task_id (str) - ID of the task. # @PARAM: passwords (Dict[str, str]) - Mapping of database name to password. # @THROWS: ValueError if task not found, not awaiting input, or passwords invalid. @@ -740,12 +760,13 @@ class TaskManager: if task_id in self.task_futures: self.task_futures[task_id].set_result(True) - # #endregion resume_task_with_password + # [/DEF:resume_task_with_password:Function] - # #region clear_tasks [C:3] [TYPE Function] - # @BRIEF Clears tasks based on status filter (also deletes associated logs). - # @PRE status is Optional[TaskStatus]. - # @POST Tasks matching filter (or all non-active) cleared from registry and database. + # [DEF:clear_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Clears tasks based on status filter (also deletes associated logs). + # @PRE: status is Optional[TaskStatus]. + # @POST: Tasks matching filter (or all non-active) cleared from registry and database. # @PARAM: status (Optional[TaskStatus]) - Filter by task status. # @RETURN: int - Number of tasks cleared. # @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks] @@ -791,10 +812,11 @@ class TaskManager: if tasks_to_remove: self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove) - log.reflect("Tasks cleared from registry and database", payload={"count": len(tasks_to_remove)}) + logger.info(f"Cleared {len(tasks_to_remove)} tasks.") return len(tasks_to_remove) - # #endregion clear_tasks + # [/DEF:clear_tasks:Function] # #endregion TaskManager +# #endregion TaskManagerModule diff --git a/backend/src/core/task_manager/models.py b/backend/src/core/task_manager/models.py index 6a8c4f28..f13b6ee7 100644 --- a/backend/src/core/task_manager/models.py +++ b/backend/src/core/task_manager/models.py @@ -1,19 +1,17 @@ # #region TaskManagerModels [C:3] [TYPE Module] [SEMANTICS task, models, pydantic, enum, state] # @BRIEF Defines the data models and enumerations used by the Task Manager. -# @LAYER Core -# @INVARIANT Task IDs are immutable once created. +# @LAYER: Core # @RELATION USED_BY -> [TaskManager] # @RELATION USED_BY -> [TaskManagerPackage] +# @INVARIANT: Task IDs are immutable once created. # @CONSTRAINT: Must use Pydantic for data validation. -# [SECTION: IMPORTS] import uuid from datetime import datetime from enum import Enum from typing import Dict, Any, List, Optional from pydantic import BaseModel, Field -# [/SECTION] # #region TaskStatus [TYPE Enum] @@ -127,17 +125,17 @@ class Task(BaseModel): # Result payload can be dict/list/scalar depending on plugin and legacy records. result: Optional[Any] = None - # #region __init__ [TYPE Function] - # @BRIEF Initializes the Task model and validates input_request for AWAITING_INPUT status. - # @PRE If status is AWAITING_INPUT, input_request must be provided. - # @POST Task instance is created or ValueError is raised. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the Task model and validates input_request for AWAITING_INPUT status. + # @PRE: If status is AWAITING_INPUT, input_request must be provided. + # @POST: Task instance is created or ValueError is raised. # @PARAM: **data - Keyword arguments for model initialization. def __init__(self, **data): super().__init__(**data) if self.status == TaskStatus.AWAITING_INPUT and not self.input_request: raise ValueError("input_request is required when status is AWAITING_INPUT") - # #endregion __init__ + # [/DEF:__init__:Function] # #endregion Task diff --git a/backend/src/core/task_manager/persistence.py b/backend/src/core/task_manager/persistence.py index 3414f597..dd71823a 100644 --- a/backend/src/core/task_manager/persistence.py +++ b/backend/src/core/task_manager/persistence.py @@ -1,16 +1,15 @@ # #region TaskPersistenceModule [C:5] [TYPE Module] [SEMANTICS persistence, sqlite, sqlalchemy, task, storage] # @BRIEF Handles the persistence of tasks using SQLAlchemy and the tasks.db database. -# @LAYER Core -# @PRE Tasks database must be initialized with TaskRecord and TaskLogRecord schemas. -# @POST Provides reliable storage and retrieval for task metadata and logs. -# @SIDE_EFFECT Performs database I/O on tasks.db. -# @DATA_CONTRACT Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord] -# @INVARIANT Database schema must match the TaskRecord model structure. +# @LAYER: Core +# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas. +# @POST: Provides reliable storage and retrieval for task metadata and logs. +# @SIDE_EFFECT: Performs database I/O on tasks.db. +# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord] # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [TaskGraph] # @RELATION DEPENDS_ON -> [TasksSessionLocal] +# @INVARIANT: Database schema must match the TaskRecord model structure. -# [SECTION: IMPORTS] from datetime import datetime from typing import List, Optional import json @@ -22,25 +21,20 @@ from ...models.mapping import Environment from ..database import TasksSessionLocal from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats from ..logger import logger, belief_scope -from ..cot_logger import MarkerLogger - -log = MarkerLogger("TaskPersistence") -log_pl = MarkerLogger("TaskLogPersistence") -# [/SECTION] # #region TaskPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, sqlalchemy] # @BRIEF Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models. -# @PRE TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available. -# @POST Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows. -# @SIDE_EFFECT Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures. -# @DATA_CONTRACT Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]] -# @INVARIANT Persistence must handle potentially missing task fields natively. +# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available. +# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows. +# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures. +# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]] # @RELATION DEPENDS_ON -> [TasksSessionLocal] # @RELATION DEPENDS_ON -> [TaskRecord] # @RELATION DEPENDS_ON -> [Environment] # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [TaskGraph] +# @INVARIANT: Persistence must handle potentially missing task fields natively. # # @TEST_CONTRACT: TaskPersistenceContract -> # { @@ -56,10 +50,11 @@ log_pl = MarkerLogger("TaskLogPersistence") # @TEST_EDGE: load_corrupt_json_params -> handled gracefully # @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params] class TaskPersistenceService: - # #region _json_load_if_needed [C:1] [TYPE Function] - # @BRIEF Safely load JSON strings from DB if necessary - # @PRE value is an arbitrary database value - # @POST Returns parsed JSON object, list, string, or primitive + # [DEF:_json_load_if_needed:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Safely load JSON strings from DB if necessary + # @PRE: value is an arbitrary database value + # @POST: Returns parsed JSON object, list, string, or primitive @staticmethod def _json_load_if_needed(value): with belief_scope("TaskPersistenceService._json_load_if_needed"): @@ -77,12 +72,13 @@ class TaskPersistenceService: return value return value - # #endregion _json_load_if_needed + # [/DEF:_json_load_if_needed:Function] - # #region _parse_datetime [C:1] [TYPE Function] - # @BRIEF Safely parse a datetime string from the database - # @PRE value is an ISO string or datetime object - # @POST Returns datetime object or None + # [DEF:_parse_datetime:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Safely parse a datetime string from the database + # @PRE: value is an ISO string or datetime object + # @POST: Returns datetime object or None @staticmethod def _parse_datetime(value): with belief_scope("TaskPersistenceService._parse_datetime"): @@ -95,14 +91,15 @@ class TaskPersistenceService: return None return None - # #endregion _parse_datetime + # [/DEF:_parse_datetime:Function] - # #region _resolve_environment_id [C:3] [TYPE Function] - # @BRIEF Resolve environment id into existing environments.id value to satisfy FK constraints. - # @PRE Session is active - # @POST Returns existing environments.id or None when unresolved. - # @DATA_CONTRACT Input[env_id: Optional[str]] -> Output[Optional[str]] - # @RELATION DEPENDS_ON -> [Environment] + # [DEF:_resolve_environment_id:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints. + # @PRE: Session is active + # @POST: Returns existing environments.id or None when unresolved. + # @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]] + # @RELATION: [DEPENDS_ON] ->[Environment] @staticmethod def _resolve_environment_id( session: Session, env_id: Optional[str] @@ -144,30 +141,31 @@ class TaskPersistenceService: return None - # #endregion _resolve_environment_id + # [/DEF:_resolve_environment_id:Function] - # #region __init__ [C:3] [TYPE Function] - # @BRIEF Initializes the persistence service. - # @PRE None. - # @POST Service is ready. + # [DEF:__init__:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Initializes the persistence service. + # @PRE: None. + # @POST: Service is ready. def __init__(self): with belief_scope("TaskPersistenceService.__init__"): # We use TasksSessionLocal from database.py pass - # #endregion __init__ + # [/DEF:__init__:Function] - # #region persist_task [C:3] [TYPE Function] - # @BRIEF Persists or updates a single task in the database. - # @PRE isinstance(task, Task) - # @POST Task record created or updated in database. + # [DEF:persist_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persists or updates a single task in the database. + # @PRE: isinstance(task, Task) + # @POST: Task record created or updated in database. # @PARAM: task (Task) - The task object to persist. # @SIDE_EFFECT: Writes to task_records table in tasks.db # @DATA_CONTRACT: Input[Task] -> Model[TaskRecord] # @RELATION: [CALLS] ->[_resolve_environment_id] def persist_task(self, task: Task) -> None: with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"): - log.reason("Persisting task to database", payload={"task_id": task.id}) session: Session = TasksSessionLocal() try: record = ( @@ -204,8 +202,8 @@ class TaskPersistenceService: # Store logs as JSON, converting datetime to string record.logs = [] - for log_entry in task.logs: - log_dict = log_entry.dict() + for log in task.logs: + log_dict = log.dict() if isinstance(log_dict.get("timestamp"), datetime): log_dict["timestamp"] = log_dict["timestamp"].isoformat() # Also clean up any datetimes in context @@ -215,41 +213,39 @@ class TaskPersistenceService: # Extract error if failed if task.status == TaskStatus.FAILED: - for log_entry in reversed(task.logs): - if log_entry.level == "ERROR": - record.error = log_entry.message + for log in reversed(task.logs): + if log.level == "ERROR": + record.error = log.message break session.commit() - log.reflect("Task persisted successfully", payload={"task_id": task.id}) except Exception as e: session.rollback() - log.explore("Failed to persist task", error=str(e), payload={"task_id": task.id}) logger.error(f"Failed to persist task {task.id}: {e}") finally: session.close() - # #endregion persist_task + # [/DEF:persist_task:Function] - # #region persist_tasks [C:3] [TYPE Function] - # @BRIEF Persists multiple tasks. - # @PRE isinstance(tasks, list) - # @POST All tasks in list are persisted. + # [DEF:persist_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persists multiple tasks. + # @PRE: isinstance(tasks, list) + # @POST: All tasks in list are persisted. # @PARAM: tasks (List[Task]) - The list of tasks to persist. # @RELATION: [CALLS] ->[persist_task] def persist_tasks(self, tasks: List[Task]) -> None: with belief_scope("TaskPersistenceService.persist_tasks"): - log.reason("Persisting multiple tasks", payload={"count": len(tasks)}) for task in tasks: self.persist_task(task) - log.reflect("All tasks persisted", payload={"count": len(tasks)}) - # #endregion persist_tasks + # [/DEF:persist_tasks:Function] - # #region load_tasks [C:3] [TYPE Function] - # @BRIEF Loads tasks from the database. - # @PRE limit is an integer. - # @POST Returns list of Task objects. + # [DEF:load_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Loads tasks from the database. + # @PRE: limit is an integer. + # @POST: Returns list of Task objects. # @PARAM: limit (int) - Max tasks to load. # @PARAM: status (Optional[TaskStatus]) - Filter by status. # @RETURN: List[Task] - The loaded tasks. @@ -260,7 +256,6 @@ class TaskPersistenceService: self, limit: int = 100, status: Optional[TaskStatus] = None ) -> List[Task]: with belief_scope("TaskPersistenceService.load_tasks"): - log.reason("Loading tasks from database", payload={"limit": limit, "status": str(status) if status else None}) session: Session = TasksSessionLocal() try: query = session.query(TaskRecord) @@ -304,20 +299,19 @@ class TaskPersistenceService: ) loaded_tasks.append(task) except Exception as e: - log.explore("Failed to reconstruct task from record", error=str(e), payload={"record_id": record.id}) logger.error(f"Failed to reconstruct task {record.id}: {e}") - log.reflect("Tasks loaded from database", payload={"count": len(loaded_tasks)}) return loaded_tasks finally: session.close() - # #endregion load_tasks + # [/DEF:load_tasks:Function] - # #region delete_tasks [C:3] [TYPE Function] - # @BRIEF Deletes specific tasks from the database. - # @PRE task_ids is a list of strings. - # @POST Specified task records deleted from database. + # [DEF:delete_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Deletes specific tasks from the database. + # @PRE: task_ids is a list of strings. + # @POST: Specified task records deleted from database. # @PARAM: task_ids (List[str]) - List of task IDs to delete. # @SIDE_EFFECT: Deletes rows from task_records table. # @RELATION: [DEPENDS_ON] ->[TaskRecord] @@ -325,30 +319,35 @@ class TaskPersistenceService: if not task_ids: return with belief_scope("TaskPersistenceService.delete_tasks"): - log.reason("Deleting tasks from database", payload={"count": len(task_ids)}) session: Session = TasksSessionLocal() try: session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete( synchronize_session=False ) session.commit() - log.reflect("Tasks deleted from database", payload={"count": len(task_ids)}) except Exception as e: session.rollback() - log.explore("Failed to delete tasks", error=str(e)) logger.error(f"Failed to delete tasks: {e}") finally: session.close() - # #endregion delete_tasks + # [/DEF:delete_tasks:Function] # #endregion TaskPersistenceService -# @INVARIANT Log entries are batch-inserted for performance. + + +# #region TaskLogPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, log, sqlalchemy] +# @BRIEF Provides methods to store, query, summarize, and delete task log rows in the task_logs table. +# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries. +# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers. +# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures. +# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]] # @RELATION DEPENDS_ON -> [TaskLogRecord] # @RELATION DEPENDS_ON -> [TasksSessionLocal] # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [EventBus] +# @INVARIANT: Log entries are batch-inserted for performance. # # @TEST_CONTRACT: TaskLogPersistenceContract -> # { @@ -368,19 +367,21 @@ class TaskLogPersistenceService: Supports batch inserts, filtering, and statistics. """ - # #region __init__ [C:3] [TYPE Function] - # @BRIEF Initializes the TaskLogPersistenceService - # @PRE config is provided or defaults are used - # @POST Service is ready for log persistence + # [DEF:__init__:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Initializes the TaskLogPersistenceService + # @PRE: config is provided or defaults are used + # @POST: Service is ready for log persistence def __init__(self, config=None): pass - # #endregion __init__ + # [/DEF:__init__:Function] - # #region add_logs [C:3] [TYPE Function] - # @BRIEF Batch insert log entries for a task. - # @PRE logs is a list of LogEntry objects. - # @POST All logs inserted into task_logs table. + # [DEF:add_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Batch insert log entries for a task. + # @PRE: logs is a list of LogEntry objects. + # @POST: All logs inserted into task_logs table. # @PARAM: task_id (str) - The task ID. # @PARAM: logs (List[LogEntry]) - Log entries to insert. # @SIDE_EFFECT: Writes to task_logs table. @@ -390,36 +391,34 @@ class TaskLogPersistenceService: if not logs: return with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"): - log_pl.reason("Batch inserting log entries", payload={"task_id": task_id, "count": len(logs)}) session: Session = TasksSessionLocal() try: - for log_entry in logs: + for log in logs: record = TaskLogRecord( task_id=task_id, - timestamp=log_entry.timestamp, - level=log_entry.level, - source=log_entry.source or "system", - message=log_entry.message, - metadata_json=json.dumps(log_entry.metadata) - if log_entry.metadata + timestamp=log.timestamp, + level=log.level, + source=log.source or "system", + message=log.message, + metadata_json=json.dumps(log.metadata) + if log.metadata else None, ) session.add(record) session.commit() - log_pl.reflect("Log entries persisted", payload={"task_id": task_id, "count": len(logs)}) except Exception as e: session.rollback() - log_pl.explore("Failed to persist log entries", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to add logs for task {task_id}: {e}") finally: session.close() - # #endregion add_logs + # [/DEF:add_logs:Function] - # #region get_logs [C:3] [TYPE Function] - # @BRIEF Query logs for a task with filtering and pagination. - # @PRE task_id is a valid task ID. - # @POST Returns list of TaskLog objects matching filters. + # [DEF:get_logs:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Query logs for a task with filtering and pagination. + # @PRE: task_id is a valid task ID. + # @POST: Returns list of TaskLog objects matching filters. # @PARAM: task_id (str) - The task ID. # @PARAM: log_filter (LogFilter) - Filter parameters. # @RETURN: List[TaskLog] - Filtered log entries. @@ -429,7 +428,6 @@ class TaskLogPersistenceService: # @RELATION: [DEPENDS_ON] ->[TaskLog] def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]: with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"): - log_pl.reason("Querying task logs", payload={"task_id": task_id, "filter_level": log_filter.level}) session: Session = TasksSessionLocal() try: query = session.query(TaskLogRecord).filter( @@ -474,17 +472,17 @@ class TaskLogPersistenceService: ) ) - log_pl.reflect("Task logs retrieved", payload={"task_id": task_id, "count": len(logs)}) return logs finally: session.close() - # #endregion get_logs + # [/DEF:get_logs:Function] - # #region get_log_stats [C:3] [TYPE Function] - # @BRIEF Get statistics about logs for a task. - # @PRE task_id is a valid task ID. - # @POST Returns LogStats with counts by level and source. + # [DEF:get_log_stats:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get statistics about logs for a task. + # @PRE: task_id is a valid task ID. + # @POST: Returns LogStats with counts by level and source. # @PARAM: task_id (str) - The task ID. # @RETURN: LogStats - Statistics about task logs. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats] @@ -494,7 +492,6 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.get_log_stats", f"task_id={task_id}" ): - log_pl.reason("Aggregating log statistics", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: # Get total count @@ -526,19 +523,19 @@ class TaskLogPersistenceService: by_source = {source: count for source, count in source_counts} - log_pl.reflect("Log statistics computed", payload={"task_id": task_id, "total": total_count}) return LogStats( total_count=total_count, by_level=by_level, by_source=by_source ) finally: session.close() - # #endregion get_log_stats + # [/DEF:get_log_stats:Function] - # #region get_sources [C:3] [TYPE Function] - # @BRIEF Get unique sources for a task's logs. - # @PRE task_id is a valid task ID. - # @POST Returns list of unique source strings. + # [DEF:get_sources:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get unique sources for a task's logs. + # @PRE: task_id is a valid task ID. + # @POST: Returns list of unique source strings. # @PARAM: task_id (str) - The task ID. # @RETURN: List[str] - Unique source names. # @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]] @@ -547,7 +544,6 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.get_sources", f"task_id={task_id}" ): - log_pl.reason("Querying distinct log sources", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: from sqlalchemy import distinct @@ -557,18 +553,17 @@ class TaskLogPersistenceService: .filter(TaskLogRecord.task_id == task_id) .all() ) - result = [s[0] for s in sources] - log_pl.reflect("Log sources retrieved", payload={"task_id": task_id, "sources": result}) - return result + return [s[0] for s in sources] finally: session.close() - # #endregion get_sources + # [/DEF:get_sources:Function] - # #region delete_logs_for_task [C:3] [TYPE Function] - # @BRIEF Delete all logs for a specific task. - # @PRE task_id is a valid task ID. - # @POST All logs for the task are deleted. + # [DEF:delete_logs_for_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Delete all logs for a specific task. + # @PRE: task_id is a valid task ID. + # @POST: All logs for the task are deleted. # @PARAM: task_id (str) - The task ID. # @SIDE_EFFECT: Deletes from task_logs table. # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] @@ -576,27 +571,25 @@ class TaskLogPersistenceService: with belief_scope( "TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}" ): - log_pl.reason("Deleting logs for task", payload={"task_id": task_id}) session: Session = TasksSessionLocal() try: session.query(TaskLogRecord).filter( TaskLogRecord.task_id == task_id ).delete(synchronize_session=False) session.commit() - log_pl.reflect("Task logs deleted", payload={"task_id": task_id}) except Exception as e: session.rollback() - log_pl.explore("Failed to delete task logs", error=str(e), payload={"task_id": task_id}) logger.error(f"Failed to delete logs for task {task_id}: {e}") finally: session.close() - # #endregion delete_logs_for_task + # [/DEF:delete_logs_for_task:Function] - # #region delete_logs_for_tasks [C:3] [TYPE Function] - # @BRIEF Delete all logs for multiple tasks. - # @PRE task_ids is a list of task IDs. - # @POST All logs for the tasks are deleted. + # [DEF:delete_logs_for_tasks:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Delete all logs for multiple tasks. + # @PRE: task_ids is a list of task IDs. + # @POST: All logs for the tasks are deleted. # @PARAM: task_ids (List[str]) - List of task IDs. # @SIDE_EFFECT: Deletes rows from task_logs table. # @RELATION: [DEPENDS_ON] ->[TaskLogRecord] @@ -604,22 +597,20 @@ class TaskLogPersistenceService: if not task_ids: return with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"): - log_pl.reason("Deleting logs for multiple tasks", payload={"count": len(task_ids)}) session: Session = TasksSessionLocal() try: session.query(TaskLogRecord).filter( TaskLogRecord.task_id.in_(task_ids) ).delete(synchronize_session=False) session.commit() - log_pl.reflect("Logs deleted for multiple tasks", payload={"count": len(task_ids)}) except Exception as e: session.rollback() - log_pl.explore("Failed to delete logs for tasks", error=str(e)) logger.error(f"Failed to delete logs for tasks: {e}") finally: session.close() - # #endregion delete_logs_for_tasks + # [/DEF:delete_logs_for_tasks:Function] # #endregion TaskLogPersistenceService +# #endregion TaskPersistenceModule diff --git a/backend/src/core/task_manager/task_logger.py b/backend/src/core/task_manager/task_logger.py index d62109d0..c8e2a04e 100644 --- a/backend/src/core/task_manager/task_logger.py +++ b/backend/src/core/task_manager/task_logger.py @@ -1,21 +1,19 @@ # #region TaskLoggerModule [C:2] [TYPE Module] [SEMANTICS task, logger, context, plugin, attribution] # @BRIEF Provides a dedicated logger for tasks with automatic source attribution. -# @LAYER Core -# @INVARIANT Each TaskLogger instance is bound to a specific task_id and default source. +# @LAYER: Core # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [EventBus] +# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source. -# [SECTION: IMPORTS] from typing import Dict, Any, Optional, Callable -# [/SECTION] # #region TaskLogger [C:2] [TYPE Class] [SEMANTICS logger, task, source, attribution] # @BRIEF A wrapper around TaskManager._add_log that carries task_id and source context. -# @INVARIANT All log calls include the task_id and source. # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [EventBus] # @RELATION USED_BY -> [TaskContext] +# @INVARIANT: All log calls include the task_id and source. # @UX_STATE: Idle -> Logging -> (system records log) # # @TEST_CONTRACT: TaskLoggerContract -> @@ -45,10 +43,10 @@ class TaskLogger: api_logger.info("Fetching dashboards") """ - # #region __init__ [TYPE Function] - # @BRIEF Initialize the TaskLogger with task context. - # @PRE add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata). - # @POST TaskLogger is ready to log messages. + # [DEF:__init__:Function] + # @PURPOSE: Initialize the TaskLogger with task context. + # @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata). + # @POST: TaskLogger is ready to log messages. # @PARAM: task_id (str) - The ID of the task. # @PARAM: add_log_fn (Callable) - Function to add log to TaskManager. # @PARAM: source (str) - Default source for logs (default: "plugin"). @@ -57,12 +55,12 @@ class TaskLogger: self._add_log = add_log_fn self._default_source = source - # #endregion __init__ + # [/DEF:__init__:Function] - # #region with_source [TYPE Function] - # @BRIEF Create a sub-logger with a different default source. - # @PRE source is a non-empty string. - # @POST Returns new TaskLogger with the same task_id but different source. + # [DEF:with_source:Function] + # @PURPOSE: Create a sub-logger with a different default source. + # @PRE: source is a non-empty string. + # @POST: Returns new TaskLogger with the same task_id but different source. # @PARAM: source (str) - New default source. # @RETURN: TaskLogger - New logger instance. def with_source(self, source: str) -> "TaskLogger": @@ -71,12 +69,12 @@ class TaskLogger: task_id=self._task_id, add_log_fn=self._add_log, source=source ) - # #endregion with_source + # [/DEF:with_source:Function] - # #region _log [TYPE Function] - # @BRIEF Internal method to log a message at a given level. - # @PRE level is a valid log level string. - # @POST Log entry added via add_log_fn. + # [DEF:_log:Function] + # @PURPOSE: Internal method to log a message at a given level. + # @PRE: level is a valid log level string. + # @POST: Log entry added via add_log_fn. # @PARAM: level (str) - Log level (DEBUG, INFO, WARNING, ERROR). # @PARAM: message (str) - Log message. # @PARAM: source (Optional[str]) - Override source for this log entry. @@ -98,12 +96,12 @@ class TaskLogger: metadata=metadata, ) - # #endregion _log + # [/DEF:_log:Function] - # #region debug [TYPE Function] - # @BRIEF Log a DEBUG level message. - # @PRE message is a string. - # @POST Log entry added via internally with DEBUG level. + # [DEF:debug:Function] + # @PURPOSE: Log a DEBUG level message. + # @PRE: message is a string. + # @POST: Log entry added via internally with DEBUG level. # @PARAM: message (str) - Log message. # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. @@ -115,12 +113,12 @@ class TaskLogger: ) -> None: self._log("DEBUG", message, source, metadata) - # #endregion debug + # [/DEF:debug:Function] - # #region info [TYPE Function] - # @BRIEF Log an INFO level message. - # @PRE message is a string. - # @POST Log entry added internally with INFO level. + # [DEF:info:Function] + # @PURPOSE: Log an INFO level message. + # @PRE: message is a string. + # @POST: Log entry added internally with INFO level. # @PARAM: message (str) - Log message. # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. @@ -132,12 +130,12 @@ class TaskLogger: ) -> None: self._log("INFO", message, source, metadata) - # #endregion info + # [/DEF:info:Function] - # #region warning [TYPE Function] - # @BRIEF Log a WARNING level message. - # @PRE message is a string. - # @POST Log entry added internally with WARNING level. + # [DEF:warning:Function] + # @PURPOSE: Log a WARNING level message. + # @PRE: message is a string. + # @POST: Log entry added internally with WARNING level. # @PARAM: message (str) - Log message. # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. @@ -149,12 +147,12 @@ class TaskLogger: ) -> None: self._log("WARNING", message, source, metadata) - # #endregion warning + # [/DEF:warning:Function] - # #region error [TYPE Function] - # @BRIEF Log an ERROR level message. - # @PRE message is a string. - # @POST Log entry added internally with ERROR level. + # [DEF:error:Function] + # @PURPOSE: Log an ERROR level message. + # @PRE: message is a string. + # @POST: Log entry added internally with ERROR level. # @PARAM: message (str) - Log message. # @PARAM: source (Optional[str]) - Override source. # @PARAM: metadata (Optional[Dict]) - Additional data. @@ -166,12 +164,12 @@ class TaskLogger: ) -> None: self._log("ERROR", message, source, metadata) - # #endregion error + # [/DEF:error:Function] - # #region progress [TYPE Function] - # @BRIEF Log a progress update with percentage. - # @PRE percent is between 0 and 100. - # @POST Log entry with progress metadata added. + # [DEF:progress:Function] + # @PURPOSE: Log a progress update with percentage. + # @PRE: percent is between 0 and 100. + # @POST: Log entry with progress metadata added. # @PARAM: message (str) - Progress message. # @PARAM: percent (float) - Progress percentage (0-100). # @PARAM: source (Optional[str]) - Override source. @@ -182,7 +180,7 @@ class TaskLogger: metadata = {"progress": min(100, max(0, percent))} self._log("INFO", message, source, metadata) - # #endregion progress + # [/DEF:progress:Function] # #endregion TaskLogger diff --git a/backend/src/core/utils/async_network.py b/backend/src/core/utils/async_network.py index 193656cb..73350cb9 100644 --- a/backend/src/core/utils/async_network.py +++ b/backend/src/core/utils/async_network.py @@ -1,22 +1,20 @@ # #region AsyncNetworkModule [C:5] [TYPE Module] [SEMANTICS network, httpx, async, superset, authentication, cache] -# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login. -# @LAYER Infra -# @PRE Config payloads contain a Superset base URL and authentication fields needed for login. -# @POST Async network clients reuse cached auth tokens and expose stable async request/error translation flow. -# @SIDE_EFFECT Performs upstream HTTP I/O and mutates process-local auth cache entries. -# @DATA_CONTRACT Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions] -# @INVARIANT Async client reuses cached auth tokens per environment credentials and invalidates on 401. -# @RELATION DEPENDS_ON -> [SupersetAuthCache] # +# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login. +# @LAYER: Infra +# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login. +# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow. +# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries. +# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions] +# @RELATION DEPENDS_ON -> [SupersetAuthCache] +# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401. -# [SECTION: IMPORTS] from typing import Optional, Dict, Any, Union import asyncio import httpx from ..logger import logger as app_logger, belief_scope -from ..cot_logger import MarkerLogger from .network import ( AuthenticationError, DashboardNotFoundError, @@ -25,9 +23,7 @@ from .network import ( SupersetAPIError, SupersetAuthCache, ) -# [/SECTION] -log = MarkerLogger("AsyncNetwork") # #region AsyncAPIClient [C:3] [TYPE Class] # @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache. @@ -38,13 +34,14 @@ class AsyncAPIClient: DEFAULT_TIMEOUT = 30 _auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {} - # #region AsyncAPIClient.__init__ [C:3] [TYPE Function] - # @BRIEF Initialize async API client for one environment. - # @PRE config contains base_url and auth payload. - # @POST Client is ready for async request/authentication flow. - # @DATA_CONTRACT Input[config: Dict[str, Any]] -> self._auth_cache_key[str] - # @RELATION CALLS -> [AsyncAPIClient._normalize_base_url] - # @RELATION DEPENDS_ON -> [SupersetAuthCache] + # [DEF:AsyncAPIClient.__init__:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Initialize async API client for one environment. + # @PRE: config contains base_url and auth payload. + # @POST: Client is ready for async request/authentication flow. + # @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str] + # @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url] + # @RELATION: [DEPENDS_ON] ->[SupersetAuthCache] def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): self.base_url: str = self._normalize_base_url(config.get("base_url", "")) self.api_base_url: str = f"{self.base_url}/api/v1" @@ -63,21 +60,23 @@ class AsyncAPIClient: verify_ssl, ) - # #endregion AsyncAPIClient.__init__ + # [/DEF:AsyncAPIClient.__init__:Function] - # #region AsyncAPIClient._normalize_base_url [C:1] [TYPE Function] - # @BRIEF Normalize base URL for Superset API root construction. - # @POST Returns canonical base URL without trailing slash and duplicate /api/v1 suffix. + # [DEF:AsyncAPIClient._normalize_base_url:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Normalize base URL for Superset API root construction. + # @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix. def _normalize_base_url(self, raw_url: str) -> str: normalized = str(raw_url or "").strip().rstrip("/") if normalized.lower().endswith("/api/v1"): normalized = normalized[:-len("/api/v1")] return normalized.rstrip("/") - # #endregion AsyncAPIClient._normalize_base_url + # [/DEF:AsyncAPIClient._normalize_base_url:Function] - # #region AsyncAPIClient._build_api_url [C:1] [TYPE Function] - # @BRIEF Build full API URL from relative Superset endpoint. - # @POST Returns absolute URL for upstream request. + # [DEF:AsyncAPIClient._build_api_url:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Build full API URL from relative Superset endpoint. + # @POST: Returns absolute URL for upstream request. def _build_api_url(self, endpoint: str) -> str: normalized_endpoint = str(endpoint or "").strip() if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"): @@ -87,11 +86,12 @@ class AsyncAPIClient: if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1": return f"{self.base_url}{normalized_endpoint}" return f"{self.api_base_url}{normalized_endpoint}" - # #endregion AsyncAPIClient._build_api_url + # [/DEF:AsyncAPIClient._build_api_url:Function] - # #region AsyncAPIClient._get_auth_lock [C:1] [TYPE Function] - # @BRIEF Return per-cache-key async lock to serialize fresh login attempts. - # @POST Returns stable asyncio.Lock instance. + # [DEF:AsyncAPIClient._get_auth_lock:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts. + # @POST: Returns stable asyncio.Lock instance. @classmethod def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock: existing_lock = cls._auth_locks.get(cache_key) @@ -100,40 +100,24 @@ class AsyncAPIClient: created_lock = asyncio.Lock() cls._auth_locks[cache_key] = created_lock return created_lock - # #endregion AsyncAPIClient._get_auth_lock + # [/DEF:AsyncAPIClient._get_auth_lock:Function] - # #region AsyncAPIClient.authenticate [C:3] [TYPE Function] - # @BRIEF Authenticate against Superset and cache access/csrf tokens. - # @POST Client tokens are populated and reusable across requests. - # @SIDE_EFFECT Performs network requests to Superset authentication endpoints. - # @DATA_CONTRACT None -> Output[Dict[str, str]] - # @RELATION CALLS -> [SupersetAuthCache.get] - # @RELATION CALLS -> [SupersetAuthCache.set] - # @RELATION CALLS -> [AsyncAPIClient._get_auth_lock] + # [DEF:AsyncAPIClient.authenticate:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Authenticate against Superset and cache access/csrf tokens. + # @POST: Client tokens are populated and reusable across requests. + # @SIDE_EFFECT: Performs network requests to Superset authentication endpoints. + # @DATA_CONTRACT: None -> Output[Dict[str, str]] + # @RELATION: [CALLS] ->[SupersetAuthCache.get] + # @RELATION: [CALLS] ->[SupersetAuthCache.set] + # @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock] async def authenticate(self) -> Dict[str, str]: cached_tokens = SupersetAuthCache.get(self._auth_cache_key) - if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): self._tokens = cached_tokens - # Always fetch a fresh CSRF token to establish session cookie in this httpx client. - # The cached access_token may still be valid; we avoid a full POST login. - try: - csrf_url = f"{self.api_base_url}/security/csrf_token/" - csrf_response = await self._client.get( - csrf_url, - headers={"Authorization": f"Bearer {self._tokens['access_token']}"}, - ) - csrf_response.raise_for_status() - self._tokens["csrf_token"] = csrf_response.json()["result"] - SupersetAuthCache.set(self._auth_cache_key, self._tokens) - self._authenticated = True - log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url}) - return self._tokens - except Exception as csrf_err: - log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err)) - SupersetAuthCache.invalidate(self._auth_cache_key) - self._tokens = {} - self._authenticated = False + self._authenticated = True + app_logger.info("[async_authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url) + return self._tokens auth_lock = self._get_auth_lock(self._auth_cache_key) async with auth_lock: @@ -141,11 +125,11 @@ class AsyncAPIClient: if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): self._tokens = cached_tokens self._authenticated = True - log.reason("Reusing cached Superset auth tokens (after lock wait)", payload={"base_url": self.base_url}) + app_logger.info("[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s", self.base_url) return self._tokens with belief_scope("AsyncAPIClient.authenticate"): - log.reason("Performing full authentication", payload={"base_url": self.base_url}) + app_logger.info("[async_authenticate][Enter] Authenticating to %s", self.base_url) try: login_url = f"{self.api_base_url}/security/login" response = await self._client.post(login_url, json=self.auth) @@ -165,7 +149,7 @@ class AsyncAPIClient: } self._authenticated = True SupersetAuthCache.set(self._auth_cache_key, self._tokens) - log.reflect("Authenticated successfully") + app_logger.info("[async_authenticate][Exit] Authenticated successfully.") return self._tokens except httpx.HTTPStatusError as exc: SupersetAuthCache.invalidate(self._auth_cache_key) @@ -179,12 +163,13 @@ class AsyncAPIClient: except (httpx.HTTPError, KeyError) as exc: SupersetAuthCache.invalidate(self._auth_cache_key) raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc - # #endregion AsyncAPIClient.authenticate + # [/DEF:AsyncAPIClient.authenticate:Function] - # #region AsyncAPIClient.get_headers [C:3] [TYPE Function] - # @BRIEF Return authenticated Superset headers for async requests. - # @POST Headers include Authorization and CSRF tokens. - # @RELATION CALLS -> [AsyncAPIClient.authenticate] + # [DEF:AsyncAPIClient.get_headers:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Return authenticated Superset headers for async requests. + # @POST: Headers include Authorization and CSRF tokens. + # @RELATION: [CALLS] ->[AsyncAPIClient.authenticate] async def get_headers(self) -> Dict[str, str]: if not self._authenticated: await self.authenticate() @@ -194,15 +179,16 @@ class AsyncAPIClient: "Referer": self.base_url, "Content-Type": "application/json", } - # #endregion AsyncAPIClient.get_headers + # [/DEF:AsyncAPIClient.get_headers:Function] - # #region AsyncAPIClient.request [C:3] [TYPE Function] - # @BRIEF Perform one authenticated async Superset API request. - # @POST Returns JSON payload or raw httpx.Response when raw_response=true. - # @SIDE_EFFECT Performs network I/O. - # @RELATION CALLS -> [AsyncAPIClient.get_headers] - # @RELATION CALLS -> [AsyncAPIClient._handle_http_error] - # @RELATION CALLS -> [AsyncAPIClient._handle_network_error] + # [DEF:AsyncAPIClient.request:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Perform one authenticated async Superset API request. + # @POST: Returns JSON payload or raw httpx.Response when raw_response=true. + # @SIDE_EFFECT: Performs network I/O. + # @RELATION: [CALLS] ->[AsyncAPIClient.get_headers] + # @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error] + # @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error] async def request( self, method: str, @@ -230,18 +216,19 @@ class AsyncAPIClient: self._handle_http_error(exc, endpoint) except httpx.HTTPError as exc: self._handle_network_error(exc, full_url) - # #endregion AsyncAPIClient.request + # [/DEF:AsyncAPIClient.request:Function] - # #region AsyncAPIClient._handle_http_error [C:3] [TYPE Function] - # @BRIEF Translate upstream HTTP errors into stable domain exceptions. - # @POST Raises domain-specific exception for caller flow control. - # @DATA_CONTRACT Input[httpx.HTTPStatusError] -> Exception - # @RELATION CALLS -> [AsyncAPIClient._is_dashboard_endpoint] - # @RELATION DEPENDS_ON -> [DashboardNotFoundError] - # @RELATION DEPENDS_ON -> [SupersetAPIError] - # @RELATION DEPENDS_ON -> [PermissionDeniedError] - # @RELATION DEPENDS_ON -> [AuthenticationError] - # @RELATION DEPENDS_ON -> [NetworkError] + # [DEF:AsyncAPIClient._handle_http_error:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Translate upstream HTTP errors into stable domain exceptions. + # @POST: Raises domain-specific exception for caller flow control. + # @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception + # @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint] + # @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError] + # @RELATION: [DEPENDS_ON] ->[SupersetAPIError] + # @RELATION: [DEPENDS_ON] ->[PermissionDeniedError] + # @RELATION: [DEPENDS_ON] ->[AuthenticationError] + # @RELATION: [DEPENDS_ON] ->[NetworkError] def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None: with belief_scope("AsyncAPIClient._handle_http_error"): status_code = exc.response.status_code @@ -261,11 +248,12 @@ class AsyncAPIClient: if status_code == 401: raise AuthenticationError() from exc raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc - # #endregion AsyncAPIClient._handle_http_error + # [/DEF:AsyncAPIClient._handle_http_error:Function] - # #region AsyncAPIClient._is_dashboard_endpoint [C:2] [TYPE Function] - # @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation. - # @POST Returns true only for dashboard-specific endpoints. + # [DEF:AsyncAPIClient._is_dashboard_endpoint:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation. + # @POST: Returns true only for dashboard-specific endpoints. def _is_dashboard_endpoint(self, endpoint: str) -> bool: normalized_endpoint = str(endpoint or "").strip().lower() if not normalized_endpoint: @@ -278,13 +266,14 @@ class AsyncAPIClient: if normalized_endpoint.startswith("/api/v1/"): normalized_endpoint = normalized_endpoint[len("/api/v1"):] return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard" - # #endregion AsyncAPIClient._is_dashboard_endpoint + # [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function] - # #region AsyncAPIClient._handle_network_error [C:3] [TYPE Function] - # @BRIEF Translate generic httpx errors into NetworkError. - # @POST Raises NetworkError with URL context. - # @DATA_CONTRACT Input[httpx.HTTPError] -> NetworkError - # @RELATION DEPENDS_ON -> [NetworkError] + # [DEF:AsyncAPIClient._handle_network_error:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Translate generic httpx errors into NetworkError. + # @POST: Raises NetworkError with URL context. + # @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError + # @RELATION: [DEPENDS_ON] ->[NetworkError] def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None: with belief_scope("AsyncAPIClient._handle_network_error"): if isinstance(exc, httpx.TimeoutException): @@ -294,14 +283,17 @@ class AsyncAPIClient: else: message = f"Unknown network error: {exc}" raise NetworkError(message, url=url) from exc - # #endregion AsyncAPIClient._handle_network_error + # [/DEF:AsyncAPIClient._handle_network_error:Function] - # #region AsyncAPIClient.aclose [C:3] [TYPE Function] - # @BRIEF Close underlying httpx client. - # @POST Client resources are released. - # @SIDE_EFFECT Closes network connections. - # @RELATION DEPENDS_ON -> [AsyncAPIClient.__init__] + # [DEF:AsyncAPIClient.aclose:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Close underlying httpx client. + # @POST: Client resources are released. + # @SIDE_EFFECT: Closes network connections. + # @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__] async def aclose(self) -> None: await self._client.aclose() - # #endregion AsyncAPIClient.aclose + # [/DEF:AsyncAPIClient.aclose:Function] # #endregion AsyncAPIClient + +# #endregion AsyncNetworkModule diff --git a/backend/src/core/utils/dataset_mapper.py b/backend/src/core/utils/dataset_mapper.py index 56d456ee..206fe663 100644 --- a/backend/src/core/utils/dataset_mapper.py +++ b/backend/src/core/utils/dataset_mapper.py @@ -1,37 +1,32 @@ # #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, mapping, postgresql, xlsx, superset] -# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [backend.core.superset_client] -# @RELATION DEPENDS_ON -> [pandas] -# @RELATION DEPENDS_ON -> [psycopg2] # +# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> backend.core.superset_client +# @RELATION DEPENDS_ON -> pandas +# @RELATION DEPENDS_ON -> psycopg2 # @PUBLIC_API: DatasetMapper -# [SECTION: IMPORTS] import pandas as pd # type: ignore import psycopg2 # type: ignore from typing import Dict, Optional, Any -from ..logger import belief_scope -from ..cot_logger import MarkerLogger -# [/SECTION] - -log = MarkerLogger("DatasetMapper") +from ..logger import logger as app_logger, belief_scope # #region DatasetMapper [TYPE Class] # @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset. class DatasetMapper: - # #region __init__ [TYPE Function] - # @BRIEF Initializes the mapper. - # @POST Объект DatasetMapper инициализирован. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the mapper. + # @POST: Объект DatasetMapper инициализирован. def __init__(self): pass - # #endregion __init__ + # [/DEF:__init__:Function] - # #region get_postgres_comments [TYPE Function] - # @BRIEF Извлекает комментарии к колонкам из системного каталога PostgreSQL. - # @PRE db_config должен содержать валидные параметры подключения (host, port, user, password, dbname). - # @PRE table_name и table_schema должны быть строками. - # @POST Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД. + # [DEF:get_postgres_comments:Function] + # @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL. + # @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname). + # @PRE: table_name и table_schema должны быть строками. + # @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД. # @THROW: Exception - При ошибках подключения или выполнения запроса к БД. # @PARAM: db_config (Dict) - Конфигурация для подключения к БД. # @PARAM: table_name (str) - Имя таблицы. @@ -39,7 +34,7 @@ class DatasetMapper: # @RETURN: Dict[str, str] - Словарь с комментариями к колонкам. def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]: with belief_scope("Fetch comments from PostgreSQL"): - log.reason("Fetching comments from PostgreSQL", payload={"table_schema": table_schema, "table_name": table_name}) + app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name) query = f""" SELECT cols.column_name, @@ -85,42 +80,42 @@ class DatasetMapper: for row in cursor.fetchall(): if row[1]: comments[row[0]] = row[1] - log.reflect("PostgreSQL comments fetched", payload={"count": len(comments)}) + app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments)) except Exception as e: - log.explore("Failed to fetch PostgreSQL comments", payload={"table_schema": table_schema, "table_name": table_name}, error=str(e)) + app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True) raise return comments - # #endregion get_postgres_comments + # [/DEF:get_postgres_comments:Function] - # #region load_excel_mappings [TYPE Function] - # @BRIEF Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла. - # @PRE file_path должен указывать на существующий XLSX файл. - # @POST Возвращается словарь с меппингами из файла. + # [DEF:load_excel_mappings:Function] + # @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла. + # @PRE: file_path должен указывать на существующий XLSX файл. + # @POST: Возвращается словарь с меппингами из файла. # @THROW: Exception - При ошибках чтения файла или парсинга. # @PARAM: file_path (str) - Путь к XLSX файлу. # @RETURN: Dict[str, str] - Словарь с меппингами. def load_excel_mappings(self, file_path: str) -> Dict[str, str]: with belief_scope("Load mappings from Excel"): - log.reason("Loading mappings from Excel", payload={"file_path": file_path}) + app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path) try: df = pd.read_excel(file_path) mappings = df.set_index('column_name')['verbose_name'].to_dict() - log.reflect("Excel mappings loaded", payload={"count": len(mappings)}) + app_logger.info("[load_excel_mappings][Success] Loaded %d mappings.", len(mappings)) return mappings except Exception as e: - log.explore("Failed to load Excel mappings", payload={"file_path": file_path}, error=str(e)) + app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True) raise - # #endregion load_excel_mappings + # [/DEF:load_excel_mappings:Function] - # #region run_mapping [TYPE Function] - # @BRIEF Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset. - # @PRE superset_client должен быть авторизован. - # @PRE dataset_id должен быть существующим ID в Superset. - # @POST Если найдены изменения, датасет в Superset обновлен через API. - # @RELATION CALLS -> [self.get_postgres_comments] - # @RELATION CALLS -> [self.load_excel_mappings] - # @RELATION CALLS -> [superset_client.get_dataset] - # @RELATION CALLS -> [superset_client.update_dataset] + # [DEF:run_mapping:Function] + # @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset. + # @PRE: superset_client должен быть авторизован. + # @PRE: dataset_id должен быть существующим ID в Superset. + # @POST: Если найдены изменения, датасет в Superset обновлен через API. + # @RELATION: CALLS -> self.get_postgres_comments + # @RELATION: CALLS -> self.load_excel_mappings + # @RELATION: CALLS -> superset_client.get_dataset + # @RELATION: CALLS -> superset_client.update_dataset # @PARAM: superset_client (Any) - Клиент Superset. # @PARAM: dataset_id (int) - ID датасета для обновления. # @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both'). @@ -130,20 +125,18 @@ class DatasetMapper: # @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL. def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None): with belief_scope(f"Run dataset mapping for ID {dataset_id}"): - log.reason("Starting dataset mapping", payload={"dataset_id": dataset_id, "source": source}) + app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source) mappings: Dict[str, str] = {} try: if source in ['postgres', 'both']: - if not (postgres_config and table_name and table_schema): - raise ValueError("Postgres config is required.") + assert postgres_config and table_name and table_schema, "Postgres config is required." mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema)) if source in ['excel', 'both']: - if not excel_path: - raise ValueError("Excel path is required.") + assert excel_path, "Excel path is required." mappings.update(self.load_excel_mappings(excel_path)) if source not in ['postgres', 'excel', 'both']: - log.explore("Invalid mapping source", payload={"source": source}, error=f"Invalid source: {source}") + app_logger.error("[run_mapping][Failure] Invalid source: %s.", source) return dataset_response = superset_client.get_dataset(dataset_id) @@ -228,14 +221,14 @@ class DatasetMapper: payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None} superset_client.update_dataset(dataset_id, payload_for_update) - log.reflect("Dataset verbose_name updated", payload={"dataset_id": dataset_id}) + app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id) else: - log.reflect("No changes in verbose_name, skipping update", payload={"dataset_id": dataset_id}) + app_logger.info("[run_mapping][State] No changes in columns' verbose_name, skipping update.") - except (FileNotFoundError, Exception) as e: - log.explore("Dataset mapping failed", payload={"dataset_id": dataset_id, "source": source}, error=str(e)) + except (AssertionError, FileNotFoundError, Exception) as e: + app_logger.error("[run_mapping][Failure] %s", e, exc_info=True) return - # #endregion run_mapping + # [/DEF:run_mapping:Function] # #endregion DatasetMapper -# #endregion DatasetMapperModule +# #endregion DatasetMapperModule \ No newline at end of file diff --git a/backend/src/core/utils/fileio.py b/backend/src/core/utils/fileio.py index 0f12d87c..80b85a79 100644 --- a/backend/src/core/utils/fileio.py +++ b/backend/src/core/utils/fileio.py @@ -1,13 +1,11 @@ -# #region FileIO [TYPE Module] +# #region FileIO [TYPE Module] [SEMANTICS file, io, zip, yaml, temp, archive, utility] # # @TIER: STANDARD -# @SEMANTICS: file, io, zip, yaml, temp, archive, utility -# @PURPOSE: Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий. +# @BRIEF Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий. # @LAYER: Infra -# @RELATION: DEPENDS_ON -> [LoggerModule] +# @RELATION DEPENDS_ON -> [LoggerModule] # @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders -# [SECTION: IMPORTS] import os import re import zipfile @@ -20,10 +18,6 @@ import shutil import zlib from dataclasses import dataclass from ..logger import logger as app_logger, belief_scope -from ..cot_logger import MarkerLogger -# [/SECTION] - -log = MarkerLogger("FileIO") # #region InvalidZipFormatError [TYPE Class] # @BRIEF Exception raised when a file is not a valid ZIP archive. @@ -33,13 +27,9 @@ class InvalidZipFormatError(Exception): # #region create_temp_file [TYPE Function] # @BRIEF Контекстный менеджер для создания временного файла или директории с гарантированным удалением. -# @PRE suffix должен быть строкой, определяющей тип ресурса. -# @POST Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста. -# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл. -# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория. -# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb'). +# @PRE: suffix должен быть строкой, определяющей тип ресурса. +# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста. # @YIELDS: Path - Путь к временному ресурсу. -# @THROW: IOError - При ошибках создания ресурса. @contextmanager def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]: with belief_scope("Create temporary resource"): @@ -49,7 +39,7 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode if is_dir: with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir: resource_path = Path(temp_dir) - log.reason("Created temporary directory", payload={"path": str(resource_path)}) + app_logger.debug("[create_temp_file][State] Created temporary directory: %s", resource_path) yield resource_path else: fd, temp_path_str = tempfile.mkstemp(suffix=suffix) @@ -57,70 +47,63 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode os.close(fd) if content: resource_path.write_bytes(content) - log.reason("Created temporary file", payload={"path": str(resource_path)}) + app_logger.debug("[create_temp_file][State] Created temporary file: %s", resource_path) yield resource_path finally: if resource_path and resource_path.exists() and not dry_run: try: if resource_path.is_dir(): shutil.rmtree(resource_path) - log.reflect("Cleaned up temporary directory", payload={"path": str(resource_path)}) + app_logger.debug("[create_temp_file][Cleanup] Removed temporary directory: %s", resource_path) else: resource_path.unlink() - log.reflect("Cleaned up temporary file", payload={"path": str(resource_path)}) + app_logger.debug("[create_temp_file][Cleanup] Removed temporary file: %s", resource_path) except OSError as e: - log.explore("Error during temp resource cleanup", payload={"path": str(resource_path)}, error=str(e)) + app_logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e) # #endregion create_temp_file # #region remove_empty_directories [TYPE Function] # @BRIEF Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути. -# @PRE root_dir должен быть путем к существующей директории. -# @POST Все пустые поддиректории удалены, возвращено их количество. -# @PARAM: root_dir (str) - Путь к корневой директории для очистки. -# @RETURN: int - Количество удаленных директорий. +# @PRE: root_dir должен быть путем к существующей директории. +# @POST: Все пустые поддиректории удалены, возвращено их количество. def remove_empty_directories(root_dir: str) -> int: with belief_scope(f"Remove empty directories in {root_dir}"): + app_logger.info("[remove_empty_directories][Enter] Starting cleanup of empty directories in %s", root_dir) removed_count = 0 if not os.path.isdir(root_dir): - log.explore("Empty directory cleanup skipped - directory not found", payload={"root_dir": root_dir}, error=f"Directory not found: {root_dir}") + app_logger.error("[remove_empty_directories][Failure] Directory not found: %s", root_dir) return 0 for current_dir, _, _ in os.walk(root_dir, topdown=False): if not os.listdir(current_dir): try: os.rmdir(current_dir) removed_count += 1 - log.reason("Removed empty directory", payload={"directory": current_dir}) + app_logger.info("[remove_empty_directories][State] Removed empty directory: %s", current_dir) except OSError as e: - log.explore("Failed to remove empty directory", payload={"directory": current_dir}, error=str(e)) - log.reflect("Empty directory cleanup complete", payload={"removed_count": removed_count}) + app_logger.error("[remove_empty_directories][Failure] Failed to remove %s: %s", current_dir, e) + app_logger.info("[remove_empty_directories][Exit] Removed %d empty directories.", removed_count) return removed_count # #endregion remove_empty_directories # #region read_dashboard_from_disk [TYPE Function] # @BRIEF Читает бинарное содержимое файла с диска. -# @PRE file_path должен указывать на существующий файл. -# @POST Возвращает байты содержимого и имя файла. -# @PARAM: file_path (str) - Путь к файлу. -# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла). -# @THROW: FileNotFoundError - Если файл не найден. +# @PRE: file_path должен указывать на существующий файл. +# @POST: Возвращает байты содержимого и имя файла. def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]: with belief_scope(f"Read dashboard from {file_path}"): path = Path(file_path) assert path.is_file(), f"Файл дашборда не найден: {file_path}" - log.reason("Reading dashboard file from disk", payload={"file_path": file_path}) + app_logger.info("[read_dashboard_from_disk][Enter] Reading file: %s", file_path) content = path.read_bytes() if not content: - log.explore("Dashboard file is empty", payload={"file_path": file_path}, error="Empty dashboard file") + app_logger.warning("[read_dashboard_from_disk][Warning] File is empty: %s", file_path) return content, path.name # #endregion read_dashboard_from_disk # #region calculate_crc32 [TYPE Function] # @BRIEF Вычисляет контрольную сумму CRC32 для файла. -# @PRE file_path должен быть объектом Path к существующему файлу. -# @POST Возвращает 8-значную hex-строку CRC32. -# @PARAM: file_path (Path) - Путь к файлу. -# @RETURN: str - 8-значное шестнадцатеричное представление CRC32. -# @THROW: IOError - При ошибках чтения файла. +# @PRE: file_path должен быть объектом Path к существующему файлу. +# @POST: Возвращает 8-значную hex-строку CRC32. def calculate_crc32(file_path: Path) -> str: with belief_scope(f"Calculate CRC32 for {file_path}"): with open(file_path, 'rb') as f: @@ -128,7 +111,6 @@ def calculate_crc32(file_path: Path) -> str: return f"{crc32_value:08x}" # #endregion calculate_crc32 -# [SECTION: DATA_CLASSES] # #region RetentionPolicy [TYPE DataClass] # @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные). @dataclass @@ -137,35 +119,31 @@ class RetentionPolicy: weekly: int = 4 monthly: int = 12 # #endregion RetentionPolicy -# [/SECTION] # #region archive_exports [TYPE Function] # @BRIEF Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию. -# @PRE output_dir должен быть путем к существующей директории. -# @POST Старые или дублирующиеся архивы удалены согласно политике. -# @RELATION CALLS -> [apply_retention_policy] -# @RELATION CALLS -> [calculate_crc32] -# @PARAM: output_dir (str) - Директория с архивами. -# @PARAM: policy (RetentionPolicy) - Политика хранения. -# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32. +# @PRE: output_dir должен быть путем к существующей директории. +# @POST: Старые или дублирующиеся архивы удалены согласно политике. +# @RELATION CALLS -> apply_retention_policy +# @RELATION CALLS -> calculate_crc32 def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None: with belief_scope(f"Archive exports in {output_dir}"): output_path = Path(output_dir) if not output_path.is_dir(): - log.explore("Archive directory not found", payload={"output_dir": output_dir}, error=f"Archive directory not found: {output_dir}") + app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir) return - log.reason("Managing archive exports", payload={"output_dir": output_dir}) + app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir) # 1. Collect all zip files zip_files = list(output_path.glob("*.zip")) if not zip_files: - log.reason("No zip files found in archive directory", payload={"output_dir": output_dir}) + app_logger.info("[archive_exports][State] No zip files found in %s", output_dir) return # 2. Deduplication if deduplicate: - log.reason("Starting archive deduplication") + app_logger.info("[archive_exports][State] Starting deduplication...") checksums = {} files_to_remove = [] @@ -177,19 +155,19 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool crc = calculate_crc32(file_path) if crc in checksums: files_to_remove.append(file_path) - log.reason("Duplicate archive found (same checksum)", payload={"file": file_path.name, "original": checksums[crc].name}) + app_logger.debug("[archive_exports][State] Duplicate found: %s (same as %s)", file_path.name, checksums[crc].name) else: checksums[crc] = file_path except Exception as e: - log.explore("Failed to calculate CRC32 for archive", payload={"file": str(file_path)}, error=str(e)) + app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e) for f in files_to_remove: try: f.unlink() zip_files.remove(f) - log.reason("Removed duplicate archive", payload={"file": f.name}) + app_logger.info("[archive_exports][State] Removed duplicate: %s", f.name) except OSError as e: - log.explore("Failed to remove duplicate archive", payload={"file": str(f)}, error=str(e)) + app_logger.error("[archive_exports][Failure] Failed to remove duplicate %s: %s", f, e) # 3. Retention Policy files_with_dates = [] @@ -217,18 +195,15 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool if file_path not in files_to_keep: try: file_path.unlink() - log.reason("Removed archive by retention policy", payload={"file": file_path.name}) + app_logger.info("[archive_exports][State] Removed by retention policy: %s", file_path.name) except OSError as e: - log.explore("Failed to remove archive by retention policy", payload={"file": str(file_path)}, error=str(e)) + app_logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e) # #endregion archive_exports # #region apply_retention_policy [TYPE Function] # @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить. -# @PRE files_with_dates is a list of (Path, date) tuples. -# @POST Returns a set of files to keep. -# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами. -# @PARAM: policy (RetentionPolicy) - Политика хранения. -# @RETURN: set - Множество путей к файлам, которые должны быть сохранены. +# @PRE: files_with_dates is a list of (Path, date) tuples. +# @POST: Returns a set of files to keep. def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set: with belief_scope("Apply retention policy"): # Сортируем по дате (от новой к старой) @@ -253,54 +228,43 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re files_to_keep.update(daily_files) files_to_keep.update(weekly_files[:policy.weekly]) files_to_keep.update(monthly_files[:policy.monthly]) - log.reflect("Retention policy applied", payload={"files_to_keep": len(files_to_keep)}) + app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep)) return files_to_keep # #endregion apply_retention_policy # #region save_and_unpack_dashboard [TYPE Function] # @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его. -# @PRE zip_content должен быть байтами валидного ZIP-архива. -# @POST ZIP-файл сохранен, и если unpack=True, он распакован в output_dir. -# @PARAM: zip_content (bytes) - Содержимое ZIP-архива. -# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения. -# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив. -# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения. -# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой. -# @THROW: InvalidZipFormatError - При ошибке формата ZIP. +# @PRE: zip_content должен быть байтами валидного ZIP-архива. +# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir. def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]: with belief_scope("Save and unpack dashboard"): - log.reason("Processing dashboard ZIP save", payload={"unpack": unpack}) + app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack) try: output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) zip_name = sanitize_filename(original_filename) if original_filename else f"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip" zip_path = output_path / zip_name zip_path.write_bytes(zip_content) - log.reason("Dashboard ZIP saved to disk", payload={"zip_path": str(zip_path)}) + app_logger.info("[save_and_unpack_dashboard][State] Dashboard saved to: %s", zip_path) if unpack: with zipfile.ZipFile(zip_path, 'r') as zip_ref: zip_ref.extractall(output_path) - log.reason("Dashboard unpacked to directory", payload={"output_path": str(output_path)}) + app_logger.info("[save_and_unpack_dashboard][State] Dashboard unpacked to: %s", output_path) return zip_path, output_path return zip_path, None except zipfile.BadZipFile as e: - log.explore("Invalid ZIP archive format", payload={"output_dir": str(output_dir)}, error=str(e)) + app_logger.error("[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s", e) raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e # #endregion save_and_unpack_dashboard # #region update_yamls [TYPE Function] # @BRIEF Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex. -# @PRE path должен быть существующей директорией. -# @POST Все YAML файлы в директории обновлены согласно переданным параметрам. -# @RELATION CALLS -> [_update_yaml_file] -# @THROW: FileNotFoundError - Если `path` не существует. -# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены. -# @PARAM: path (str) - Путь к директории с YAML файлами. -# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска. -# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены. +# @PRE: path должен быть существующей директорией. +# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам. +# @RELATION CALLS -> _update_yaml_file def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None: with belief_scope("Update YAML configurations"): - log.reason("Starting YAML configuration update", payload={"path": path}) + app_logger.info("[update_yamls][Enter] Starting YAML configuration update.") dir_path = Path(path) assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией" @@ -312,12 +276,8 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = # #region _update_yaml_file [TYPE Function] # @BRIEF (Helper) Обновляет один YAML файл. -# @PRE file_path должен быть объектом Path к существующему YAML файлу. -# @POST Файл обновлен согласно переданным конфигурациям или регулярному выражению. -# @PARAM: file_path (Path) - Путь к файлу. -# @PARAM: db_configs (List[Dict]) - Конфигурации. -# @PARAM: regexp_pattern (Optional[str]) - Паттерн. -# @PARAM: replace_string (Optional[str]) - Замена. +# @PRE: file_path должен быть объектом Path к существующему YAML файлу. +# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению. def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None: with belief_scope(f"Update YAML file: {file_path}"): # Читаем содержимое файла @@ -325,7 +285,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ with open(file_path, 'r', encoding='utf-8') as f: content = f.read() except Exception as e: - log.explore("Failed to read YAML file", payload={"file_path": str(file_path)}, error=str(e)) + app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e) return # Если задан pattern и replace_string, применяем замену по регулярному выражению if regexp_pattern and replace_string: @@ -334,9 +294,9 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ if new_content != content: with open(file_path, 'w', encoding='utf-8') as f: f.write(new_content) - log.reason("Updated YAML file using regex pattern", payload={"file_path": str(file_path)}) + app_logger.info("[_update_yaml_file][State] Updated %s using regex pattern", file_path) except Exception as e: - log.explore("Error applying regex to YAML file", payload={"file_path": str(file_path)}, error=str(e)) + app_logger.error("[_update_yaml_file][Failure] Error applying regex to %s: %s", file_path, e) # Если заданы конфигурации, заменяем значения (поддержка old/new) if db_configs: try: @@ -357,37 +317,33 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_ # Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц) pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)' - # #region replacer [TYPE Function] - # @BRIEF Функция замены, сохраняющая кавычки если они были. - # @PRE match должен быть объектом совпадения регулярного выражения. - # @POST Возвращает строку с новым значением, сохраняя префикс и кавычки. + # [DEF:replacer:Function] + # @PURPOSE: Функция замены, сохраняющая кавычки если они были. + # @PRE: match должен быть объектом совпадения регулярного выражения. + # @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки. def replacer(match): prefix = match.group(1) quote_open = match.group(2) quote_close = match.group(4) return f"{prefix}{quote_open}{new_val}{quote_close}" - # #endregion replacer + # [/DEF:replacer:Function] modified_content = re.sub(pattern, replacer, modified_content) - log.reason("Replaced YAML config value", payload={"key": key, "file_path": str(file_path)}) + app_logger.info("[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s", old_val, new_val, key, file_path) # Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование with open(file_path, 'w', encoding='utf-8') as f: f.write(modified_content) except Exception as e: - log.explore("Error performing raw replacement in YAML", payload={"file_path": str(file_path)}, error=str(e)) + app_logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e) # #endregion _update_yaml_file # #region create_dashboard_export [TYPE Function] # @BRIEF Создает ZIP-архив из указанных исходных путей. -# @PRE source_paths должен содержать существующие пути. -# @POST ZIP-архив создан по пути zip_path. -# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива. -# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации. -# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения. -# @RETURN: bool - `True` при успехе, `False` при ошибке. +# @PRE: source_paths должен содержать существующие пути. +# @POST: ZIP-архив создан по пути zip_path. def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool: with belief_scope(f"Create dashboard export: {zip_path}"): - log.reason("Packing dashboard export", payload={"source_paths": [str(p) for p in source_paths], "zip_path": str(zip_path)}) + app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path) try: exclude_ext = [ext.lower() for ext in exclude_extensions or []] with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: @@ -398,19 +354,17 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union if item.is_file() and item.suffix.lower() not in exclude_ext: arcname = item.relative_to(src_path.parent) zipf.write(item, arcname) - log.reflect("Dashboard archive created", payload={"zip_path": str(zip_path)}) + app_logger.info("[create_dashboard_export][Exit] Archive created: %s", zip_path) return True except (IOError, zipfile.BadZipFile, AssertionError) as e: - log.explore("Dashboard archive creation failed", payload={"zip_path": str(zip_path)}, error=str(e)) + app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True) return False # #endregion create_dashboard_export # #region sanitize_filename [TYPE Function] # @BRIEF Очищает строку от символов, недопустимых в именах файлов. -# @PRE filename должен быть строкой. -# @POST Возвращает строку без спецсимволов. -# @PARAM: filename (str) - Исходное имя файла. -# @RETURN: str - Очищенная строка. +# @PRE: filename должен быть строкой. +# @POST: Возвращает строку без спецсимволов. def sanitize_filename(filename: str) -> str: with belief_scope(f"Sanitize filename: {filename}"): return re.sub(r'[\\/*?:"<>|]', "_", filename).strip() @@ -418,10 +372,8 @@ def sanitize_filename(filename: str) -> str: # #region get_filename_from_headers [TYPE Function] # @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'. -# @PRE headers должен быть словарем заголовков. -# @POST Возвращает имя файла или None, если заголовок отсутствует. -# @PARAM: headers (dict) - Словарь HTTP заголовков. -# @RETURN: Optional[str] - Имя файла or `None`. +# @PRE: headers должен быть словарем заголовков. +# @POST: Возвращает имя файла или None, если заголовок отсутствует. def get_filename_from_headers(headers: dict) -> Optional[str]: with belief_scope("Get filename from headers"): content_disposition = headers.get("Content-Disposition", "") @@ -432,16 +384,14 @@ def get_filename_from_headers(headers: dict) -> Optional[str]: # #region consolidate_archive_folders [TYPE Function] # @BRIEF Консолидирует директории архивов на основе общего слага в имени. -# @PRE root_directory должен быть объектом Path к существующей директории. -# @POST Директории с одинаковым префиксом объединены в одну. -# @THROW: TypeError, ValueError - Если `root_directory` невалиден. -# @PARAM: root_directory (Path) - Корневая директория для консолидации. +# @PRE: root_directory должен быть объектом Path к существующей директории. +# @POST: Директории с одинаковым префиксом объединены в одну. def consolidate_archive_folders(root_directory: Path) -> None: with belief_scope(f"Consolidate archives in {root_directory}"): assert isinstance(root_directory, Path), "root_directory must be a Path object." assert root_directory.is_dir(), "root_directory must be an existing directory." - log.reason("Consolidating archive folders", payload={"root_directory": str(root_directory)}) + app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory) # Собираем все директории с архивами archive_dirs = [] for item in root_directory.iterdir(): @@ -464,7 +414,7 @@ def consolidate_archive_folders(root_directory: Path) -> None: # Создаем целевую директорию target_dir = root_directory / slug target_dir.mkdir(exist_ok=True) - log.reason("Consolidating archive directories", payload={"slug": slug, "dir_count": len(dirs), "target": str(target_dir)}) + app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir) # Перемещаем содержимое for source_dir in dirs: if source_dir == target_dir: @@ -477,13 +427,13 @@ def consolidate_archive_folders(root_directory: Path) -> None: else: shutil.move(str(item), str(dest_item)) except Exception as e: - log.explore("Failed to move item during consolidation", payload={"source": str(item), "dest": str(dest_item)}, error=str(e)) + app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e) # Удаляем исходную директорию try: source_dir.rmdir() - log.reason("Removed source directory after consolidation", payload={"directory": str(source_dir)}) + app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir) except Exception as e: - log.explore("Failed to remove source directory after consolidation", payload={"directory": str(source_dir)}, error=str(e)) + app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e) # #endregion consolidate_archive_folders -# #endregion FileIO +# #endregion FileIO \ No newline at end of file diff --git a/backend/src/core/utils/matching.py b/backend/src/core/utils/matching.py index 0cb17273..9b35cec5 100644 --- a/backend/src/core/utils/matching.py +++ b/backend/src/core/utils/matching.py @@ -1,24 +1,18 @@ # #region FuzzyMatching [TYPE Module] [SEMANTICS fuzzy, matching, rapidfuzz, database, mapping] +# # @BRIEF Provides utility functions for fuzzy matching database names. -# @LAYER Core -# @INVARIANT Confidence scores are returned as floats between 0.0 and 1.0. -# @RELATION DEPENDS_ON -> [rapidfuzz] -# +# @LAYER: Core +# @RELATION DEPENDS_ON -> rapidfuzz # +# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0. -# [SECTION: IMPORTS] from rapidfuzz import fuzz, process from typing import List, Dict -# [/SECTION] # #region suggest_mappings [TYPE Function] # @BRIEF Suggests mappings between source and target databases using fuzzy matching. -# @PRE source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'. -# @POST Returns a list of suggested mappings with confidence scores. -# @PARAM: source_databases (List[Dict]) - Databases from the source environment. -# @PARAM: target_databases (List[Dict]) - Databases from the target environment. -# @PARAM: threshold (int) - Minimum confidence score (0-100). -# @RETURN: List[Dict] - Suggested mappings. +# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'. +# @POST: Returns a list of suggested mappings with confidence scores. def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]: """ Suggest mappings between source and target databases using fuzzy matching. diff --git a/backend/src/core/utils/network.py b/backend/src/core/utils/network.py index 60f620f7..e7910d8a 100644 --- a/backend/src/core/utils/network.py +++ b/backend/src/core/utils/network.py @@ -1,11 +1,10 @@ # #region NetworkModule [C:3] [TYPE Module] [SEMANTICS network, http, client, api, requests, session, authentication] -# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [LoggerModule] # +# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок. +# @LAYER: Infra +# @RELATION DEPENDS_ON -> [LoggerModule] # @PUBLIC_API: APIClient -# [SECTION: IMPORTS] from typing import Optional, Dict, Any, List, Union, cast, Tuple import json import io @@ -17,81 +16,81 @@ from requests.adapters import HTTPAdapter import urllib3 from urllib3.util.retry import Retry from ..logger import logger as app_logger, belief_scope -from ..cot_logger import MarkerLogger -# [/SECTION] - -log = MarkerLogger("Network") # #region SupersetAPIError [C:1] [TYPE Class] # @BRIEF Base exception for all Superset API related errors. class SupersetAPIError(Exception): - # #region __init__ [C:1] [TYPE Function] - # @BRIEF Initializes the exception with a message and context. - # @PRE message is a string, context is a dict. - # @POST Exception is initialized with context. + # [DEF:__init__:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Initializes the exception with a message and context. + # @PRE: message is a string, context is a dict. + # @POST: Exception is initialized with context. def __init__(self, message: str = "Superset API error", **context: Any): with belief_scope("SupersetAPIError.__init__"): self.context = context super().__init__(f"[API_FAILURE] {message} | Context: {self.context}") - # #endregion __init__ + # [/DEF:__init__:Function] # #endregion SupersetAPIError + # #region AuthenticationError [C:1] [TYPE Class] # @BRIEF Exception raised when authentication fails. class AuthenticationError(SupersetAPIError): - # #region __init__ [C:1] [TYPE Function] - # @BRIEF Initializes the authentication error. - # @PRE message is a string, context is a dict. - # @POST AuthenticationError is initialized. + # [DEF:__init__:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Initializes the authentication error. + # @PRE: message is a string, context is a dict. + # @POST: AuthenticationError is initialized. def __init__(self, message: str = "Authentication failed", **context: Any): with belief_scope("AuthenticationError.__init__"): super().__init__(message, type="authentication", **context) - # #endregion __init__ + # [/DEF:__init__:Function] # #endregion AuthenticationError + # #region PermissionDeniedError [TYPE Class] # @BRIEF Exception raised when access is denied. class PermissionDeniedError(AuthenticationError): - # #region __init__ [TYPE Function] - # @BRIEF Initializes the permission denied error. - # @PRE message is a string, context is a dict. - # @POST PermissionDeniedError is initialized. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the permission denied error. + # @PRE: message is a string, context is a dict. + # @POST: PermissionDeniedError is initialized. def __init__(self, message: str = "Permission denied", **context: Any): with belief_scope("PermissionDeniedError.__init__"): super().__init__(message, **context) - # #endregion __init__ + # [/DEF:__init__:Function] # #endregion PermissionDeniedError # #region DashboardNotFoundError [TYPE Class] # @BRIEF Exception raised when a dashboard cannot be found. class DashboardNotFoundError(SupersetAPIError): - # #region __init__ [TYPE Function] - # @BRIEF Initializes the not found error with resource ID. - # @PRE resource_id is provided. - # @POST DashboardNotFoundError is initialized. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the not found error with resource ID. + # @PRE: resource_id is provided. + # @POST: DashboardNotFoundError is initialized. def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any): with belief_scope("DashboardNotFoundError.__init__"): super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context) - # #endregion __init__ + # [/DEF:__init__:Function] # #endregion DashboardNotFoundError # #region NetworkError [TYPE Class] # @BRIEF Exception raised when a network level error occurs. class NetworkError(Exception): - # #region NetworkError.__init__ [TYPE Function] - # @BRIEF Initializes the network error. - # @PRE message is a string. - # @POST NetworkError is initialized. + # [DEF:NetworkError.__init__:Function] + # @PURPOSE: Initializes the network error. + # @PRE: message is a string. + # @POST: NetworkError is initialized. def __init__(self, message: str = "Network connection failed", **context: Any): with belief_scope("NetworkError.__init__"): self.context = context super().__init__(f"[NETWORK_FAILURE] {message} | Context: {self.context}") - # #endregion NetworkError.__init__ + # [/DEF:NetworkError.__init__:Function] # #endregion NetworkError # #region SupersetAuthCache [TYPE Class] # @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials. -# @PRE base_url and username are stable strings. -# @POST Cached entries expire automatically by TTL and can be reused across requests. +# @PRE: base_url and username are stable strings. +# @POST: Cached entries expire automatically by TTL and can be reused across requests. class SupersetAuthCache: TTL_SECONDS = 300 @@ -106,7 +105,7 @@ class SupersetAuthCache: return (str(base_url or "").strip(), username, bool(verify_ssl)) @classmethod - # #region SupersetAuthCache.get [TYPE Function] + # [DEF:SupersetAuthCache.get:Function] def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]: now = time.time() with cls._lock: @@ -125,10 +124,10 @@ class SupersetAuthCache: "access_token": str(tokens.get("access_token") or ""), "csrf_token": str(tokens.get("csrf_token") or ""), } - # #endregion SupersetAuthCache.get + # [/DEF:SupersetAuthCache.get:Function] @classmethod - # #region SupersetAuthCache.set [TYPE Function] + # [DEF:SupersetAuthCache.set:Function] def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None: normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1) with cls._lock: @@ -139,7 +138,7 @@ class SupersetAuthCache: }, "expires_at": time.time() + normalized_ttl, } - # #endregion SupersetAuthCache.set + # [/DEF:SupersetAuthCache.set:Function] @classmethod def invalidate(cls, key: Tuple[str, str, bool]) -> None: @@ -154,8 +153,8 @@ class SupersetAuthCache: class APIClient: DEFAULT_TIMEOUT = 30 - # #region APIClient.__init__ [TYPE Function] - # @BRIEF Инициализирует API клиент с конфигурацией, сессией и логгером. + # [DEF:APIClient.__init__:Function] + # @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером. # @PARAM: config (Dict[str, Any]) - Конфигурация. # @PARAM: verify_ssl (bool) - Проверять ли SSL. # @PARAM: timeout (int) - Таймаут запросов. @@ -163,7 +162,7 @@ class APIClient: # @POST: APIClient instance is initialized with a session. def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT): with belief_scope("__init__"): - log.reason("Initializing APIClient") + app_logger.info("[APIClient.__init__][Entry] Initializing APIClient.") self.base_url: str = self._normalize_base_url(config.get("base_url", "")) self.api_base_url: str = f"{self.base_url}/api/v1" self.auth = config.get("auth") @@ -176,14 +175,14 @@ class APIClient: verify_ssl, ) self._authenticated = False - log.reflect("APIClient initialized") - # #endregion APIClient.__init__ + app_logger.info("[APIClient.__init__][Exit] APIClient initialized.") + # [/DEF:APIClient.__init__:Function] - # #region _init_session [TYPE Function] - # @BRIEF Создает и настраивает `requests.Session` с retry-логикой. - # @PRE self.request_settings must be initialized. - # @POST Returns a configured requests.Session instance. - # @RETURN requests.Session - Настроенная сессия. + # [DEF:_init_session:Function] + # @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой. + # @PRE: self.request_settings must be initialized. + # @POST: Returns a configured requests.Session instance. + # @RETURN: requests.Session - Настроенная сессия. def _init_session(self) -> requests.Session: with belief_scope("_init_session"): session = requests.Session() @@ -216,32 +215,32 @@ class APIClient: if not self.request_settings["verify_ssl"]: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - log.reflect("SSL verification disabled for session", payload={"base_url": self.base_url}) + app_logger.warning("[_init_session][State] SSL verification disabled.") # When verify_ssl is false, we should also disable hostname verification session.verify = False else: session.verify = True return session - # #endregion _init_session + # [/DEF:_init_session:Function] - # #region _normalize_base_url [TYPE Function] - # @BRIEF Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix. - # @PRE raw_url can be empty. - # @POST Returns canonical base URL suitable for building API endpoints. - # @RETURN str + # [DEF:_normalize_base_url:Function] + # @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix. + # @PRE: raw_url can be empty. + # @POST: Returns canonical base URL suitable for building API endpoints. + # @RETURN: str def _normalize_base_url(self, raw_url: str) -> str: normalized = str(raw_url or "").strip().rstrip("/") if normalized.lower().endswith("/api/v1"): normalized = normalized[:-len("/api/v1")] return normalized.rstrip("/") - # #endregion _normalize_base_url + # [/DEF:_normalize_base_url:Function] - # #region _build_api_url [TYPE Function] - # @BRIEF Build absolute Superset API URL for endpoint using canonical /api/v1 base. - # @PRE endpoint is relative path or absolute URL. - # @POST Returns full URL without accidental duplicate slashes. - # @RETURN str + # [DEF:_build_api_url:Function] + # @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base. + # @PRE: endpoint is relative path or absolute URL. + # @POST: Returns full URL without accidental duplicate slashes. + # @RETURN: str def _build_api_url(self, endpoint: str) -> str: normalized_endpoint = str(endpoint or "").strip() if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"): @@ -251,97 +250,65 @@ class APIClient: if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1": return f"{self.base_url}{normalized_endpoint}" return f"{self.api_base_url}{normalized_endpoint}" - # #endregion _build_api_url + # [/DEF:_build_api_url:Function] - # #region APIClient.authenticate [TYPE Function] - # @BRIEF Выполняет аутентификацию в Superset API и получает access и CSRF токены. - # @PRE self.auth and self.base_url must be valid. - # @POST `self._tokens` заполнен, `self._authenticated` установлен в `True`. - # @RETURN Dict[str, str] - Словарь с токенами. + # [DEF:APIClient.authenticate:Function] + # @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены. + # @PRE: self.auth and self.base_url must be valid. + # @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`. + # @RETURN: Dict[str, str] - Словарь с токенами. # @THROW: AuthenticationError, NetworkError - при ошибках. # @RELATION: [CALLS] ->[SupersetAuthCache.get] # @RELATION: [CALLS] ->[SupersetAuthCache.set] - # @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie. - # When a new APIClient instance reuses cached tokens, its fresh requests.Session - # has no session cookie — causing superset CSRF check to fail with - # "CSRF session token is missing". The fix is to always fetch a fresh CSRF token - # (GET /security/csrf_token/) even on cache hit, which establishes a session cookie - # in the new session while avoiding a full POST login. - # @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing - # all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances. def authenticate(self) -> Dict[str, str]: with belief_scope("authenticate"): - log.reason("Authenticating to Superset", payload={"base_url": self.base_url}) + app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url) cached_tokens = SupersetAuthCache.get(self._auth_cache_key) - if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"): self._tokens = cached_tokens - # Always fetch a fresh CSRF token to establish session cookie in this session. - # The cached access_token may still be valid; we avoid a full POST login. - try: - csrf_url = f"{self.api_base_url}/security/csrf_token/" - csrf_response = self.session.get( - csrf_url, - headers={"Authorization": f"Bearer {self._tokens['access_token']}"}, - timeout=self.request_settings["timeout"], - ) - csrf_response.raise_for_status() - # Update CSRF token (may have rotated) - self._tokens["csrf_token"] = csrf_response.json()["result"] - SupersetAuthCache.set(self._auth_cache_key, self._tokens) - log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url}) - except Exception as csrf_err: - # CSRF refresh failed — likely access_token expired, do full re-auth - log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err)) - SupersetAuthCache.invalidate(self._auth_cache_key) - # Fall through to full login below - self._tokens = {} - self._authenticated = False - - if not self._authenticated: - try: - login_url = f"{self.api_base_url}/security/login" - # Log the payload keys and values (masking password) - masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()} - log.reason("Performing full Superset login", payload={"login_url": login_url, "auth_payload": masked_auth}) - - response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"]) - - if response.status_code != 200: - log.explore("Superset login returned non-200 status", payload={"status_code": response.status_code, "response": response.text}, error=f"HTTP {response.status_code}: {response.text[:200]}") - - response.raise_for_status() - access_token = response.json()["access_token"] - - csrf_url = f"{self.api_base_url}/security/csrf_token/" - csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"]) - csrf_response.raise_for_status() - - self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]} - self._authenticated = True - SupersetAuthCache.set(self._auth_cache_key, self._tokens) - log.reflect("Authenticated successfully") - return self._tokens - except requests.exceptions.HTTPError as e: - SupersetAuthCache.invalidate(self._auth_cache_key) - status_code = e.response.status_code if e.response is not None else None - if status_code in [502, 503, 504]: - raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e - raise AuthenticationError(f"Authentication failed: {e}") from e - except (requests.exceptions.RequestException, KeyError) as e: - SupersetAuthCache.invalidate(self._auth_cache_key) - raise NetworkError(f"Network or parsing error during authentication: {e}") from e - - self._authenticated = True - log.reflect("Authenticated successfully (from cached tokens)") - return self._tokens - # #endregion APIClient.authenticate + self._authenticated = True + app_logger.info("[authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url) + return self._tokens + try: + login_url = f"{self.api_base_url}/security/login" + # Log the payload keys and values (masking password) + masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()} + app_logger.info(f"[authenticate][Debug] Login URL: {login_url}") + app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}") + + response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"]) + + if response.status_code != 200: + app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}") + + response.raise_for_status() + access_token = response.json()["access_token"] + + csrf_url = f"{self.api_base_url}/security/csrf_token/" + csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"]) + csrf_response.raise_for_status() + + self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]} + self._authenticated = True + SupersetAuthCache.set(self._auth_cache_key, self._tokens) + app_logger.info("[authenticate][Exit] Authenticated successfully.") + return self._tokens + except requests.exceptions.HTTPError as e: + SupersetAuthCache.invalidate(self._auth_cache_key) + status_code = e.response.status_code if e.response is not None else None + if status_code in [502, 503, 504]: + raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e + raise AuthenticationError(f"Authentication failed: {e}") from e + except (requests.exceptions.RequestException, KeyError) as e: + SupersetAuthCache.invalidate(self._auth_cache_key) + raise NetworkError(f"Network or parsing error during authentication: {e}") from e + # [/DEF:APIClient.authenticate:Function] @property - # #region headers [TYPE Function] - # @BRIEF Возвращает HTTP-заголовки для аутентифицированных запросов. - # @PRE APIClient is initialized and authenticated or can be authenticated. - # @POST Returns headers including auth tokens. + # [DEF:headers:Function] + # @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов. + # @PRE: APIClient is initialized and authenticated or can be authenticated. + # @POST: Returns headers including auth tokens. def headers(self) -> Dict[str, str]: if not self._authenticated: self.authenticate() @@ -351,10 +318,10 @@ class APIClient: "Referer": self.base_url, "Content-Type": "application/json" } - # #endregion headers + # [/DEF:headers:Function] - # #region request [TYPE Function] - # @BRIEF Выполняет универсальный HTTP-запрос к API. + # [DEF:request:Function] + # @PURPOSE: Выполняет универсальный HTTP-запрос к API. # @PARAM: method (str) - HTTP метод. # @PARAM: endpoint (str) - API эндпоинт. # @PARAM: headers (Optional[Dict]) - Дополнительные заголовки. @@ -381,10 +348,10 @@ class APIClient: self._handle_http_error(e, endpoint) except requests.exceptions.RequestException as e: self._handle_network_error(e, full_url) - # #endregion request + # [/DEF:request:Function] - # #region _handle_http_error [TYPE Function] - # @BRIEF (Helper) Преобразует HTTP ошибки в кастомные исключения. + # [DEF:_handle_http_error:Function] + # @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения. # @PARAM: e (requests.exceptions.HTTPError) - Ошибка. # @PARAM: endpoint (str) - Эндпоинт. # @PRE: e must be a valid HTTPError with a response. @@ -408,12 +375,12 @@ class APIClient: if status_code == 401: raise AuthenticationError() from e raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e - # #endregion _handle_http_error + # [/DEF:_handle_http_error:Function] - # #region _is_dashboard_endpoint [TYPE Function] - # @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation. - # @PRE endpoint may be relative or absolute. - # @POST Returns true only for dashboard-specific endpoints. + # [DEF:_is_dashboard_endpoint:Function] + # @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation. + # @PRE: endpoint may be relative or absolute. + # @POST: Returns true only for dashboard-specific endpoints. def _is_dashboard_endpoint(self, endpoint: str) -> bool: normalized_endpoint = str(endpoint or "").strip().lower() if not normalized_endpoint: @@ -426,10 +393,10 @@ class APIClient: if normalized_endpoint.startswith("/api/v1/"): normalized_endpoint = normalized_endpoint[len("/api/v1"):] return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard" - # #endregion _is_dashboard_endpoint + # [/DEF:_is_dashboard_endpoint:Function] - # #region _handle_network_error [TYPE Function] - # @BRIEF (Helper) Преобразует сетевые ошибки в `NetworkError`. + # [DEF:_handle_network_error:Function] + # @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`. # @PARAM: e (requests.exceptions.RequestException) - Ошибка. # @PARAM: url (str) - URL. # @PRE: e must be a RequestException. @@ -443,10 +410,10 @@ class APIClient: else: msg = f"Unknown network error: {e}" raise NetworkError(msg, url=url) from e - # #endregion _handle_network_error + # [/DEF:_handle_network_error:Function] - # #region upload_file [TYPE Function] - # @BRIEF Загружает файл на сервер через multipart/form-data. + # [DEF:upload_file:Function] + # @PURPOSE: Загружает файл на сервер через multipart/form-data. # @PARAM: endpoint (str) - Эндпоинт. # @PARAM: file_info (Dict[str, Any]) - Информация о файле. # @PARAM: extra_data (Optional[Dict]) - Дополнительные данные. @@ -474,10 +441,10 @@ class APIClient: raise TypeError(f"Unsupported file_obj type: {type(file_obj)}") return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout) - # #endregion upload_file + # [/DEF:upload_file:Function] - # #region _perform_upload [TYPE Function] - # @BRIEF (Helper) Выполняет POST запрос с файлом. + # [DEF:_perform_upload:Function] + # @PURPOSE: (Helper) Выполняет POST запрос с файлом. # @PARAM: url (str) - URL. # @PARAM: files (Dict) - Файлы. # @PARAM: data (Optional[Dict]) - Данные. @@ -495,17 +462,17 @@ class APIClient: try: return response.json() except Exception as json_e: - log.explore("Upload response not valid JSON", payload={"preview": response.text[:200]}, error=str(json_e)) + app_logger.debug(f"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...") raise SupersetAPIError(f"API error during upload: Response is not valid JSON: {json_e}") from json_e return response.json() except requests.exceptions.HTTPError as e: raise SupersetAPIError(f"API error during upload: {e.response.text}") from e except requests.exceptions.RequestException as e: raise NetworkError(f"Network error during upload: {e}", url=url) from e - # #endregion _perform_upload + # [/DEF:_perform_upload:Function] - # #region fetch_paginated_count [TYPE Function] - # @BRIEF Получает общее количество элементов для пагинации. + # [DEF:fetch_paginated_count:Function] + # @PURPOSE: Получает общее количество элементов для пагинации. # @PARAM: endpoint (str) - Эндпоинт. # @PARAM: query_params (Dict) - Параметры запроса. # @PARAM: count_field (str) - Поле с количеством. @@ -516,10 +483,10 @@ class APIClient: with belief_scope("fetch_paginated_count"): response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)})) return response_json.get(count_field, 0) - # #endregion fetch_paginated_count + # [/DEF:fetch_paginated_count:Function] - # #region fetch_paginated_data [TYPE Function] - # @BRIEF Автоматически собирает данные со всех страниц пагинированного эндпоинта. + # [DEF:fetch_paginated_data:Function] + # @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта. # @PARAM: endpoint (str) - Эндпоинт. # @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации. # @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional. @@ -547,7 +514,7 @@ class APIClient: if total_count is None: total_count = response_json.get(count_field, len(first_page_results)) - log.reason("Total count resolved from first page", payload={"total_count": total_count}) + app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}") # Fetch remaining pages total_pages = (total_count + page_size - 1) // page_size @@ -557,7 +524,7 @@ class APIClient: results.extend(response_json.get(results_field, [])) return results - # #endregion fetch_paginated_data + # [/DEF:fetch_paginated_data:Function] # #endregion APIClient diff --git a/backend/src/core/utils/superset_compilation_adapter.py b/backend/src/core/utils/superset_compilation_adapter.py index 725bead8..d318fa0e 100644 --- a/backend/src/core/utils/superset_compilation_adapter.py +++ b/backend/src/core/utils/superset_compilation_adapter.py @@ -1,12 +1,12 @@ # #region SupersetCompilationAdapter [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth] # @BRIEF Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context. -# @LAYER Infra -# @PRE effective template params and dataset execution reference are available. -# @POST preview and launch calls return Superset-originated artifacts or explicit errors. -# @SIDE_EFFECT performs upstream Superset preview and SQL Lab calls. -# @INVARIANT The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only. +# @LAYER: Infra # @RELATION CALLS -> [SupersetClient] # @RELATION DEPENDS_ON -> [CompiledPreview] +# @PRE: effective template params and dataset execution reference are available. +# @POST: preview and launch calls return Superset-originated artifacts or explicit errors. +# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls. +# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only. from __future__ import annotations @@ -52,38 +52,41 @@ class SqlLabLaunchPayload: # #region SupersetCompilationAdapter [C:4] [TYPE Class] # @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication. -# @PRE environment is configured and Superset is reachable for the target session. -# @POST adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references. -# @SIDE_EFFECT issues network requests to Superset API surfaces. # @RELATION CALLS -> [SupersetClient] +# @PRE: environment is configured and Superset is reachable for the target session. +# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references. +# @SIDE_EFFECT: issues network requests to Superset API surfaces. class SupersetCompilationAdapter: - # #region SupersetCompilationAdapter.__init__ [C:2] [TYPE Function] - # @BRIEF Bind adapter to one Superset environment and client instance. + # [DEF:SupersetCompilationAdapter.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind adapter to one Superset environment and client instance. def __init__( self, environment: Environment, client: Optional[SupersetClient] = None ) -> None: self.environment = environment self.client = client or SupersetClient(environment) - # #endregion SupersetCompilationAdapter.__init__ + # [/DEF:SupersetCompilationAdapter.__init__:Function] - # #region SupersetCompilationAdapter._supports_client_method [C:2] [TYPE Function] - # @BRIEF Detect explicitly implemented client capabilities without treating loose mocks as real methods. + # [DEF:SupersetCompilationAdapter._supports_client_method:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Detect explicitly implemented client capabilities without treating loose mocks as real methods. def _supports_client_method(self, method_name: str) -> bool: client_dict = getattr(self.client, "__dict__", {}) if method_name in client_dict: return callable(client_dict[method_name]) return callable(getattr(type(self.client), method_name, None)) - # #endregion SupersetCompilationAdapter._supports_client_method + # [/DEF:SupersetCompilationAdapter._supports_client_method:Function] - # #region SupersetCompilationAdapter.compile_preview [C:4] [TYPE Function] - # @BRIEF Request Superset-side compiled SQL preview for the current effective inputs. - # @PRE dataset_id and effective inputs are available for the current session. - # @POST returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics. - # @SIDE_EFFECT performs upstream preview requests. - # @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[CompiledPreview] - # @RELATION CALLS -> [SupersetCompilationAdapter._request_superset_preview] + # [DEF:SupersetCompilationAdapter.compile_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview] + # @PRE: dataset_id and effective inputs are available for the current session. + # @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics. + # @SIDE_EFFECT: performs upstream preview requests. + # @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview] def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview: with belief_scope("SupersetCompilationAdapter.compile_preview"): if payload.dataset_id <= 0: @@ -169,25 +172,27 @@ class SupersetCompilationAdapter: ) return preview - # #endregion SupersetCompilationAdapter.compile_preview + # [/DEF:SupersetCompilationAdapter.compile_preview:Function] - # #region SupersetCompilationAdapter.mark_preview_stale [C:2] [TYPE Function] - # @BRIEF Invalidate previous preview after mapping or value changes. - # @PRE preview is a persisted preview artifact or current in-memory snapshot. - # @POST preview status becomes stale without fabricating a replacement artifact. + # [DEF:SupersetCompilationAdapter.mark_preview_stale:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Invalidate previous preview after mapping or value changes. + # @PRE: preview is a persisted preview artifact or current in-memory snapshot. + # @POST: preview status becomes stale without fabricating a replacement artifact. def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview: preview.preview_status = PreviewStatus.STALE return preview - # #endregion SupersetCompilationAdapter.mark_preview_stale + # [/DEF:SupersetCompilationAdapter.mark_preview_stale:Function] - # #region SupersetCompilationAdapter.create_sql_lab_session [C:4] [TYPE Function] - # @BRIEF Create the canonical audited execution session after all launch gates pass. - # @PRE compiled_sql is Superset-originated and launch gates are already satisfied. - # @POST returns one canonical SQL Lab session reference from Superset. - # @SIDE_EFFECT performs upstream SQL Lab execution/session creation. - # @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[str] - # @RELATION CALLS -> [SupersetCompilationAdapter._request_sql_lab_session] + # [DEF:SupersetCompilationAdapter.create_sql_lab_session:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Create the canonical audited execution session after all launch gates pass. + # @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session] + # @PRE: compiled_sql is Superset-originated and launch gates are already satisfied. + # @POST: returns one canonical SQL Lab session reference from Superset. + # @SIDE_EFFECT: performs upstream SQL Lab execution/session creation. + # @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str] def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str: with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"): compiled_sql = str(payload.compiled_sql or "").strip() @@ -239,15 +244,16 @@ class SupersetCompilationAdapter: ) return sql_lab_session_ref - # #endregion SupersetCompilationAdapter.create_sql_lab_session + # [/DEF:SupersetCompilationAdapter.create_sql_lab_session:Function] - # #region SupersetCompilationAdapter._request_superset_preview [C:4] [TYPE Function] - # @BRIEF Request preview compilation through explicit client support backed by real Superset endpoints only. - # @PRE payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt. - # @POST returns one normalized upstream compilation response including the chosen strategy metadata. - # @SIDE_EFFECT issues one or more Superset preview requests through the client fallback chain. - # @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[Dict[str,Any]] - # @RELATION CALLS -> [SupersetClient.compile_dataset_preview] + # [DEF:SupersetCompilationAdapter._request_superset_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only. + # @RELATION: [CALLS] ->[SupersetClient.compile_dataset_preview] + # @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt. + # @POST: returns one normalized upstream compilation response including the chosen strategy metadata. + # @SIDE_EFFECT: issues one or more Superset preview requests through the client fallback chain. + # @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]] def _request_superset_preview( self, payload: PreviewCompilationPayload ) -> Dict[str, Any]: @@ -384,15 +390,16 @@ class SupersetCompilationAdapter: ) raise RuntimeError(str(exc)) from exc - # #endregion SupersetCompilationAdapter._request_superset_preview + # [/DEF:SupersetCompilationAdapter._request_superset_preview:Function] - # #region SupersetCompilationAdapter._request_sql_lab_session [C:4] [TYPE Function] - # @BRIEF Probe supported SQL Lab execution surfaces and return the first successful response. - # @PRE payload carries non-empty Superset-originated SQL and a preview identifier for the current launch. - # @POST returns the first successful SQL Lab execution response from Superset. - # @SIDE_EFFECT issues Superset dataset lookup and SQL Lab execution requests. - # @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]] - # @RELATION CALLS -> [SupersetClient.get_dataset] + # [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response. + # @RELATION: [CALLS] ->[SupersetClient.get_dataset] + # @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch. + # @POST: returns the first successful SQL Lab execution response from Superset. + # @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests. + # @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]] def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]: dataset_raw = self.client.get_dataset(payload.dataset_id) dataset_record = ( @@ -444,11 +451,12 @@ class SupersetCompilationAdapter: "; ".join(errors) or "No Superset SQL Lab surface accepted the request" ) - # #endregion SupersetCompilationAdapter._request_sql_lab_session + # [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function] - # #region SupersetCompilationAdapter._normalize_preview_response [C:3] [TYPE Function] - # @BRIEF Normalize candidate Superset preview responses into one compiled-sql structure. - # @RELATION DEPENDS_ON -> [CompiledPreview] + # [DEF:SupersetCompilationAdapter._normalize_preview_response:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure. + # @RELATION: [DEPENDS_ON] ->[CompiledPreview] def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]: if not isinstance(response, dict): return None @@ -477,16 +485,19 @@ class SupersetCompilationAdapter: } return None - # #endregion SupersetCompilationAdapter._normalize_preview_response + # [/DEF:SupersetCompilationAdapter._normalize_preview_response:Function] - # #region SupersetCompilationAdapter._dump_json [C:1] [TYPE Function] - # @BRIEF Serialize Superset request payload deterministically for network transport. + # [DEF:SupersetCompilationAdapter._dump_json:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Serialize Superset request payload deterministically for network transport. def _dump_json(self, payload: Dict[str, Any]) -> str: import json return json.dumps(payload, sort_keys=True, default=str) - # #endregion SupersetCompilationAdapter._dump_json + # [/DEF:SupersetCompilationAdapter._dump_json:Function] # #endregion SupersetCompilationAdapter + +# #endregion SupersetCompilationAdapter diff --git a/backend/src/core/utils/superset_context_extractor/__init__.py b/backend/src/core/utils/superset_context_extractor/__init__.py index a356b648..70e7fc45 100644 --- a/backend/src/core/utils/superset_context_extractor/__init__.py +++ b/backend/src/core/utils/superset_context_extractor/__init__.py @@ -1,15 +1,15 @@ # #region SupersetContextExtractorPackage [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery] +# @LAYER: Infra # @BRIEF Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers. -# @LAYER Infra -# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. -# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. -# @SIDE_EFFECT Performs upstream Superset API reads. -# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. # @RELATION DEPENDS_ON -> [ImportedFilter] # @RELATION DEPENDS_ON -> [TemplateVariable] # @RELATION DEPENDS_ON -> [SupersetClient] -# @RATIONALE Decomposed from monolithic superset_context_extractor.py (1397 lines) into +# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. +# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. +# @SIDE_EFFECT: Performs upstream Superset API reads. +# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. # +# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into # domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class # preserves the original public API surface — all consumers continue to import # from `src.core.utils.superset_context_extractor` without changes. @@ -26,15 +26,15 @@ from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_ # #region SupersetContextExtractor [C:4] [TYPE Class] # @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. -# @PRE constructor receives a configured environment with a usable Superset base URL. -# @POST extractor instance is ready to parse links against one Superset environment. -# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient. # @RELATION DEPENDS_ON -> [Environment] # @RELATION INHERITS -> [SupersetContextExtractorBase] # @RELATION INHERITS -> [SupersetContextParsingMixin] # @RELATION INHERITS -> [SupersetContextRecoveryMixin] # @RELATION INHERITS -> [SupersetContextTemplatesMixin] # @RELATION INHERITS -> [SupersetContextFiltersExtractMixin] +# @PRE: constructor receives a configured environment with a usable Superset base URL. +# @POST: extractor instance is ready to parse links against one Superset environment. +# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient. class SupersetContextExtractor( SupersetContextFiltersExtractMixin, SupersetContextRecoveryMixin, diff --git a/backend/src/core/utils/superset_context_extractor/_base.py b/backend/src/core/utils/superset_context_extractor/_base.py index ce824bf8..672f1348 100644 --- a/backend/src/core/utils/superset_context_extractor/_base.py +++ b/backend/src/core/utils/superset_context_extractor/_base.py @@ -1,13 +1,13 @@ # #region SupersetContextExtractorBase [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery] +# @LAYER: Infra # @BRIEF Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers. -# @LAYER Infra -# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. -# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. -# @SIDE_EFFECT Performs upstream Superset API reads. -# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. # @RELATION DEPENDS_ON -> [ImportedFilter] # @RELATION DEPENDS_ON -> [TemplateVariable] # @RELATION CALLS -> [SupersetClient] +# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource. +# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary. +# @SIDE_EFFECT: Performs upstream Superset API reads. +# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context. # #endregion SupersetContextExtractorBase # #region _base_imports [TYPE Block] @@ -50,23 +50,25 @@ class SupersetParsedContext: # #region SupersetContextExtractorBase [C:4] [TYPE Class] # @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers. -# @PRE constructor receives a configured environment with a usable Superset base URL. -# @POST extractor instance is ready to parse links against one Superset environment. -# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient. # @RELATION DEPENDS_ON -> [Environment] +# @PRE: constructor receives a configured environment with a usable Superset base URL. +# @POST: extractor instance is ready to parse links against one Superset environment. +# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient. class SupersetContextExtractorBase: - # #region SupersetContextExtractorBase.__init__ [C:2] [TYPE Function] - # @BRIEF Bind extractor to one Superset environment and client instance. + # [DEF:SupersetContextExtractorBase.__init__:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind extractor to one Superset environment and client instance. def __init__( self, environment: Environment, client: Optional[SupersetClient] = None ) -> None: self.environment = environment self.client = client or SupersetClient(environment) - # #endregion SupersetContextExtractorBase.__init__ + # [/DEF:SupersetContextExtractorBase.__init__:Function] - # #region SupersetContextExtractorBase.build_recovery_summary [C:2] [TYPE Function] - # @BRIEF Summarize recovered, partial, and unresolved context for session state and UX. + # [DEF:SupersetContextExtractorBase.build_recovery_summary:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX. def build_recovery_summary( self, parsed_context: SupersetParsedContext ) -> Dict[str, Any]: @@ -80,10 +82,11 @@ class SupersetContextExtractorBase: "imported_filter_count": len(parsed_context.imported_filters), } - # #endregion SupersetContextExtractorBase.build_recovery_summary + # [/DEF:SupersetContextExtractorBase.build_recovery_summary:Function] - # #region SupersetContextExtractorBase._extract_numeric_identifier [C:2] [TYPE Function] - # @BRIEF Extract a numeric identifier from a REST-like Superset URL path. + # [DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path. def _extract_numeric_identifier( self, path_parts: List[str], resource_name: str ) -> Optional[int]: @@ -102,10 +105,11 @@ class SupersetContextExtractorBase: return None return int(candidate) - # #endregion SupersetContextExtractorBase._extract_numeric_identifier + # [/DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function] - # #region SupersetContextExtractorBase._extract_dashboard_reference [C:2] [TYPE Function] - # @BRIEF Extract a dashboard id-or-slug reference from a Superset URL path. + # [DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path. def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]: if "dashboard" not in path_parts: return None @@ -122,10 +126,11 @@ class SupersetContextExtractorBase: return None return candidate - # #endregion SupersetContextExtractorBase._extract_dashboard_reference + # [/DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function] - # #region SupersetContextExtractorBase._extract_dashboard_permalink_key [C:2] [TYPE Function] - # @BRIEF Extract a dashboard permalink key from a Superset URL path. + # [DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard permalink key from a Superset URL path. def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]: if "dashboard" not in path_parts: return None @@ -143,31 +148,34 @@ class SupersetContextExtractorBase: return None return permalink_key - # #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key + # [/DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function] - # #region SupersetContextExtractorBase._extract_dashboard_id_from_state [C:2] [TYPE Function] - # @BRIEF Extract a dashboard identifier from returned permalink state when present. + # [DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a dashboard identifier from returned permalink state when present. def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: return self._search_nested_numeric_key( payload=state, candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"}, ) - # #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state + # [/DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function] - # #region SupersetContextExtractorBase._extract_chart_id_from_state [C:2] [TYPE Function] - # @BRIEF Extract a chart identifier from returned permalink state when dashboard id is absent. + # [DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent. def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]: return self._search_nested_numeric_key( payload=state, candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"}, ) - # #endregion SupersetContextExtractorBase._extract_chart_id_from_state + # [/DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function] - # #region SupersetContextExtractorBase._search_nested_numeric_key [C:3] [TYPE Function] - # @BRIEF Recursively search nested dict/list payloads for the first numeric value under a candidate key set. - # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] + # [DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set. + # @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase] def _search_nested_numeric_key( self, payload: Any, candidate_keys: Set[str] ) -> Optional[int]: @@ -189,10 +197,11 @@ class SupersetContextExtractorBase: return found return None - # #endregion SupersetContextExtractorBase._search_nested_numeric_key + # [/DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function] - # #region SupersetContextExtractorBase._decode_query_state [C:2] [TYPE Function] - # @BRIEF Decode query-string structures used by Superset URL state transport. + # [DEF:SupersetContextExtractorBase._decode_query_state:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Decode query-string structures used by Superset URL state transport. def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]: query_state: Dict[str, Any] = {} for key, values in query_params.items(): @@ -212,7 +221,7 @@ class SupersetContextExtractorBase: query_state[key] = decoded_value return query_state - # #endregion SupersetContextExtractorBase._decode_query_state + # [/DEF:SupersetContextExtractorBase._decode_query_state:Function] # #endregion SupersetContextExtractorBase diff --git a/backend/src/core/utils/superset_context_extractor/_filters.py b/backend/src/core/utils/superset_context_extractor/_filters.py index 3b2c680e..6741c12e 100644 --- a/backend/src/core/utils/superset_context_extractor/_filters.py +++ b/backend/src/core/utils/superset_context_extractor/_filters.py @@ -1,6 +1,6 @@ # #region SupersetContextFiltersExtractMixin [C:3] [TYPE Module] [SEMANTICS superset, filters, extraction, query_state, dataMask, native_filters] +# @LAYER: Infra # @BRIEF Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data). -# @LAYER Infra # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] # #endregion SupersetContextFiltersExtractMixin @@ -19,8 +19,9 @@ app_logger = cast(Any, app_logger) # #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class] # @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor. class SupersetContextFiltersExtractMixin: - # #region SupersetContextFiltersExtractMixin._extract_imported_filters [C:2] [TYPE Function] - # @BRIEF Normalize imported filters from decoded query state without fabricating missing values. + # [DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values. def _extract_imported_filters( self, query_state: Dict[str, Any] ) -> List[Dict[str, Any]]: @@ -251,7 +252,7 @@ class SupersetContextFiltersExtractMixin: return imported_filters - # #endregion SupersetContextFiltersExtractMixin._extract_imported_filters + # [/DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function] # #endregion SupersetContextFiltersExtractMixin diff --git a/backend/src/core/utils/superset_context_extractor/_parsing.py b/backend/src/core/utils/superset_context_extractor/_parsing.py index cbff2f13..dd7fe2c0 100644 --- a/backend/src/core/utils/superset_context_extractor/_parsing.py +++ b/backend/src/core/utils/superset_context_extractor/_parsing.py @@ -1,6 +1,6 @@ # #region SupersetContextParsingMixin [C:4] [TYPE Module] [SEMANTICS superset, link_parsing, url, permalink, recovery] +# @LAYER: Infra # @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake. -# @LAYER Infra # @RELATION CALLS -> [SupersetClient] # @RELATION CALLS -> [SupersetContextExtractorBase] # #endregion SupersetContextParsingMixin @@ -21,13 +21,14 @@ logger = cast(Any, logger) # #region SupersetContextParsingMixin [C:4] [TYPE Class] # @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor. class SupersetContextParsingMixin: - # #region SupersetContextParsingMixin.parse_superset_link [C:4] [TYPE Function] - # @BRIEF Extract candidate identifiers and query state from supported Superset URLs. - # @PRE link is a non-empty Superset URL compatible with the configured environment. - # @POST returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed. - # @SIDE_EFFECT may issue Superset API reads to resolve dataset references from dashboard or chart URLs. - # @DATA_CONTRACT Input[link:str] -> Output[SupersetParsedContext] - # @RELATION CALLS -> [SupersetClient.get_dashboard_detail] + # [DEF:SupersetContextParsingMixin.parse_superset_link:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs. + # @RELATION: CALLS -> [SupersetClient.get_dashboard_detail] + # @PRE: link is a non-empty Superset URL compatible with the configured environment. + # @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed. + # @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs. + # @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext] def parse_superset_link(self, link: str) -> SupersetParsedContext: with belief_scope("SupersetContextExtractor.parse_superset_link"): normalized_link = str(link or "").strip() @@ -328,11 +329,12 @@ class SupersetContextParsingMixin: ) return result - # #endregion SupersetContextParsingMixin.parse_superset_link + # [/DEF:SupersetContextParsingMixin.parse_superset_link:Function] - # #region SupersetContextParsingMixin._recover_dataset_binding_from_dashboard [C:3] [TYPE Function] - # @BRIEF Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers. - # @RELATION CALLS -> [SupersetClient.get_dashboard_detail] + # [DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers. + # @RELATION: CALLS -> [SupersetClient.get_dashboard_detail] def _recover_dataset_binding_from_dashboard( self, dashboard_id: int, @@ -369,7 +371,7 @@ class SupersetContextParsingMixin: unresolved_references.append("dashboard_dataset_binding_missing") return None, unresolved_references - # #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard + # [/DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function] # #endregion SupersetContextParsingMixin diff --git a/backend/src/core/utils/superset_context_extractor/_pii.py b/backend/src/core/utils/superset_context_extractor/_pii.py index c285f2ab..57fab51e 100644 --- a/backend/src/core/utils/superset_context_extractor/_pii.py +++ b/backend/src/core/utils/superset_context_extractor/_pii.py @@ -1,6 +1,6 @@ # #region SupersetContextExtractorPII [C:3] [TYPE Module] [SEMANTICS superset, pii, masking, sanitization, privacy] +# @LAYER: Infra # @BRIEF PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers. -# @LAYER Infra # #endregion SupersetContextExtractorPII # #region _pii_imports [TYPE Block] diff --git a/backend/src/core/utils/superset_context_extractor/_recovery.py b/backend/src/core/utils/superset_context_extractor/_recovery.py index 18fbac87..90c48c65 100644 --- a/backend/src/core/utils/superset_context_extractor/_recovery.py +++ b/backend/src/core/utils/superset_context_extractor/_recovery.py @@ -1,6 +1,6 @@ # #region SupersetContextRecoveryMixin [C:4] [TYPE Module] [SEMANTICS superset, filters, recovery, imported_filters] +# @LAYER: Infra # @BRIEF Recover imported filters from Superset parsed context and dashboard metadata. -# @LAYER Infra # @RELATION CALLS -> [SupersetClient] # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] # #endregion SupersetContextRecoveryMixin @@ -22,13 +22,14 @@ logger = cast(Any, logger) # #region SupersetContextRecoveryMixin [C:4] [TYPE Class] # @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor. class SupersetContextRecoveryMixin: - # #region SupersetContextRecoveryMixin.recover_imported_filters [C:4] [TYPE Function] - # @BRIEF Build imported filter entries from URL state and Superset-side saved context. - # @PRE parsed_context comes from a successful Superset link parse for one environment. - # @POST returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements. - # @SIDE_EFFECT may issue Superset reads for dashboard metadata enrichment. - # @DATA_CONTRACT Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] - # @RELATION CALLS -> [SupersetClient.get_dashboard] + # [DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Build imported filter entries from URL state and Superset-side saved context. + # @RELATION: CALLS -> [SupersetClient.get_dashboard] + # @PRE: parsed_context comes from a successful Superset link parse for one environment. + # @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements. + # @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment. + # @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]] def recover_imported_filters( self, parsed_context: SupersetParsedContext ) -> List[Dict[str, Any]]: @@ -259,10 +260,11 @@ class SupersetContextRecoveryMixin: ) return recovered_filters - # #endregion SupersetContextRecoveryMixin.recover_imported_filters + # [/DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function] - # #region SupersetContextRecoveryMixin._normalize_imported_filter_payload [C:2] [TYPE Function] - # @BRIEF Normalize one imported-filter payload with explicit provenance and confirmation state. + # [DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state. def _normalize_imported_filter_payload( self, payload: Dict[str, Any], @@ -303,7 +305,7 @@ class SupersetContextRecoveryMixin: "notes": str(payload.get("notes") or default_note), } - # #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload + # [/DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function] # #endregion SupersetContextRecoveryMixin diff --git a/backend/src/core/utils/superset_context_extractor/_templates.py b/backend/src/core/utils/superset_context_extractor/_templates.py index 37b1ddbc..f359e729 100644 --- a/backend/src/core/utils/superset_context_extractor/_templates.py +++ b/backend/src/core/utils/superset_context_extractor/_templates.py @@ -1,6 +1,6 @@ # #region SupersetContextTemplatesMixin [C:3] [TYPE Module] [SEMANTICS superset, template_variables, jinja, discovery, deterministic] +# @LAYER: Infra # @BRIEF Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution. -# @LAYER Infra # @RELATION DEPENDS_ON -> [TemplateVariable] # @RELATION DEPENDS_ON -> [SupersetContextExtractorBase] # #endregion SupersetContextTemplatesMixin @@ -20,12 +20,13 @@ logger = cast(Any, logger) # #region SupersetContextTemplatesMixin [C:3] [TYPE Class] # @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor. class SupersetContextTemplatesMixin: - # #region SupersetContextTemplatesMixin.discover_template_variables [C:4] [TYPE Function] - # @BRIEF Detect runtime variables and Jinja references from dataset query-bearing fields. - # @PRE dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. - # @POST returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. - # @SIDE_EFFECT none. - # @DATA_CONTRACT Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] + # [DEF:SupersetContextTemplatesMixin.discover_template_variables:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields. + # @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available. + # @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values. + # @SIDE_EFFECT: none. + # @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]] def discover_template_variables( self, dataset_payload: Dict[str, Any] ) -> List[Dict[str, Any]]: @@ -117,11 +118,12 @@ class SupersetContextTemplatesMixin: ) return discovered - # #endregion SupersetContextTemplatesMixin.discover_template_variables + # [/DEF:SupersetContextTemplatesMixin.discover_template_variables:Function] - # #region SupersetContextTemplatesMixin._collect_query_bearing_expressions [C:3] [TYPE Function] - # @BRIEF Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. - # @RELATION DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables] + # [DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery. + # @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables] def _collect_query_bearing_expressions( self, dataset_payload: Dict[str, Any] ) -> List[str]: @@ -160,10 +162,11 @@ class SupersetContextTemplatesMixin: return expressions - # #endregion SupersetContextTemplatesMixin._collect_query_bearing_expressions + # [/DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function] - # #region SupersetContextTemplatesMixin._append_template_variable [C:2] [TYPE Function] - # @BRIEF Append one deduplicated template-variable descriptor. + # [DEF:SupersetContextTemplatesMixin._append_template_variable:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Append one deduplicated template-variable descriptor. def _append_template_variable( self, discovered: List[Dict[str, Any]], @@ -192,10 +195,11 @@ class SupersetContextTemplatesMixin: } ) - # #endregion SupersetContextTemplatesMixin._append_template_variable + # [/DEF:SupersetContextTemplatesMixin._append_template_variable:Function] - # #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [C:2] [TYPE Function] - # @BRIEF Extract a deterministic primary identifier from a Jinja expression without executing it. + # [DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it. def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]: matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip()) if matched is None: @@ -214,10 +218,11 @@ class SupersetContextTemplatesMixin: return None return candidate - # #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier + # [/DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function] - # #region SupersetContextTemplatesMixin._normalize_default_literal [C:2] [TYPE Function] - # @BRIEF Normalize literal default fragments from template helper calls into JSON-safe values. + # [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values. def _normalize_default_literal(self, literal: Optional[str]) -> Any: normalized_literal = str(literal or "").strip() if not normalized_literal: @@ -241,7 +246,7 @@ class SupersetContextTemplatesMixin: except ValueError: return normalized_literal - # #endregion SupersetContextTemplatesMixin._normalize_default_literal + # [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function] # #endregion SupersetContextTemplatesMixin diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index 7a759dbc..e1f2a335 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -1,6 +1,6 @@ # #region AppDependencies [C:3] [TYPE Module] [SEMANTICS dependency, injection, singleton, factory, auth, jwt] # @BRIEF Manages creation and provision of shared application dependencies, such as PluginLoader and TaskManager, to avoid circular imports. -# @LAYER Core +# @LAYER: Core # @RELATION Used by main app and API routers to get access to shared instances. # @RELATION CALLS -> [CleanReleaseRepository] # @RELATION CALLS -> [ConfigManager] @@ -39,13 +39,11 @@ from .services.clean_release.repository import CleanReleaseRepository from .services.clean_release.facade import CleanReleaseFacade from .services.reports.report_service import ReportsService from .core.database import init_db, get_auth_db, get_db -from .core.cot_logger import MarkerLogger +from .core.logger import logger from .core.auth.jwt import decode_token from .core.auth.repository import AuthRepository from .models.auth import User -log = MarkerLogger("Dependencies") - # Initialize singletons lazily to avoid import-time DB side effects during test collection. # Use absolute path relative to this file to ensure plugins are found regardless of CWD project_root = Path(__file__).parent.parent.parent @@ -60,9 +58,8 @@ resource_service: Optional[ResourceService] = None # #region get_config_manager [C:1] [TYPE Function] # @BRIEF Dependency injector for ConfigManager. -# @PRE Global config_manager must be initialized. -# @POST Returns shared ConfigManager instance. -# @RETURN ConfigManager - The shared config manager instance. +# @PRE: Global config_manager must be initialized. +# @POST: Returns shared ConfigManager instance. def get_config_manager() -> ConfigManager: """Dependency injector for ConfigManager.""" global config_manager @@ -84,16 +81,15 @@ plugin_dir = Path(__file__).parent / "plugins" # #region get_plugin_loader [C:1] [TYPE Function] # @BRIEF Dependency injector for PluginLoader. -# @PRE Global plugin_loader must be initialized. -# @POST Returns shared PluginLoader instance. -# @RETURN PluginLoader - The shared plugin loader instance. +# @PRE: Global plugin_loader must be initialized. +# @POST: Returns shared PluginLoader instance. def get_plugin_loader() -> PluginLoader: """Dependency injector for PluginLoader.""" global plugin_loader if plugin_loader is None: plugin_loader = PluginLoader(plugin_dir=str(plugin_dir)) - log.reason(f"PluginLoader initialized with directory: {plugin_dir}") - log.reason( + logger.info(f"PluginLoader initialized with directory: {plugin_dir}") + logger.info( f"Available plugins: {[config.name for config in plugin_loader.get_all_plugin_configs()]}" ) return plugin_loader @@ -104,15 +100,14 @@ def get_plugin_loader() -> PluginLoader: # #region get_task_manager [C:1] [TYPE Function] # @BRIEF Dependency injector for TaskManager. -# @PRE Global task_manager must be initialized. -# @POST Returns shared TaskManager instance. -# @RETURN TaskManager - The shared task manager instance. +# @PRE: Global task_manager must be initialized. +# @POST: Returns shared TaskManager instance. def get_task_manager() -> TaskManager: """Dependency injector for TaskManager.""" global task_manager if task_manager is None: task_manager = TaskManager(get_plugin_loader()) - log.reason("TaskManager initialized") + logger.info("TaskManager initialized") return task_manager @@ -121,15 +116,14 @@ def get_task_manager() -> TaskManager: # #region get_scheduler_service [C:1] [TYPE Function] # @BRIEF Dependency injector for SchedulerService. -# @PRE Global scheduler_service must be initialized. -# @POST Returns shared SchedulerService instance. -# @RETURN SchedulerService - The shared scheduler service instance. +# @PRE: Global scheduler_service must be initialized. +# @POST: Returns shared SchedulerService instance. def get_scheduler_service() -> SchedulerService: """Dependency injector for SchedulerService.""" global scheduler_service if scheduler_service is None: scheduler_service = SchedulerService(get_task_manager(), get_config_manager()) - log.reason("SchedulerService initialized") + logger.info("SchedulerService initialized") return scheduler_service @@ -138,15 +132,14 @@ def get_scheduler_service() -> SchedulerService: # #region get_resource_service [C:1] [TYPE Function] # @BRIEF Dependency injector for ResourceService. -# @PRE Global resource_service must be initialized. -# @POST Returns shared ResourceService instance. -# @RETURN ResourceService - The shared resource service instance. +# @PRE: Global resource_service must be initialized. +# @POST: Returns shared ResourceService instance. def get_resource_service() -> ResourceService: """Dependency injector for ResourceService.""" global resource_service if resource_service is None: resource_service = ResourceService() - log.reason("ResourceService initialized") + logger.info("ResourceService initialized") return resource_service @@ -155,9 +148,8 @@ def get_resource_service() -> ResourceService: # #region get_mapping_service [C:1] [TYPE Function] # @BRIEF Dependency injector for MappingService. -# @PRE Global config_manager must be initialized. -# @POST Returns new MappingService instance. -# @RETURN MappingService - A new mapping service instance. +# @PRE: Global config_manager must be initialized. +# @POST: Returns new MappingService instance. def get_mapping_service() -> MappingService: """Dependency injector for MappingService.""" return MappingService(get_config_manager()) @@ -171,7 +163,7 @@ _clean_release_repository = CleanReleaseRepository() # #region get_clean_release_repository [C:1] [TYPE Function] # @BRIEF Legacy compatibility shim for CleanReleaseRepository. -# @POST Returns a shared CleanReleaseRepository instance. +# @POST: Returns a shared CleanReleaseRepository instance. def get_clean_release_repository() -> CleanReleaseRepository: """Legacy compatibility shim for CleanReleaseRepository.""" return _clean_release_repository @@ -182,7 +174,7 @@ def get_clean_release_repository() -> CleanReleaseRepository: # #region get_clean_release_facade [C:1] [TYPE Function] # @BRIEF Dependency injector for CleanReleaseFacade. -# @POST Returns a facade instance with a fresh DB session. +# @POST: Returns a facade instance with a fresh DB session. def get_clean_release_facade(db=Depends(get_db)) -> CleanReleaseFacade: candidate_repo = CandidateRepository(db) artifact_repo = ArtifactRepository(db) @@ -211,21 +203,17 @@ def get_clean_release_facade(db=Depends(get_db)) -> CleanReleaseFacade: # #endregion get_clean_release_facade # #region oauth2_scheme [C:1] [TYPE Variable] +# @RELATION DEPENDS_ON -> OAuth2PasswordBearer # @BRIEF OAuth2 password bearer scheme for token extraction. -# @RELATION DEPENDS_ON -> [OAuth2PasswordBearer] oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login") # #endregion oauth2_scheme # #region get_current_user [C:3] [TYPE Function] +# @RELATION CALLS -> AuthRepository # @BRIEF Dependency for retrieving currently authenticated user from a JWT. -# @PRE JWT token provided in Authorization header. -# @POST Returns User object if token is valid. -# @RELATION CALLS -> [AuthRepository] -# @THROW: HTTPException 401 if token is invalid or user not found. -# @PARAM: token (str) - Extracted JWT token. -# @PARAM: db (Session) - Auth database session. -# @RETURN: User - The authenticated user. +# @PRE: JWT token provided in Authorization header. +# @POST: Returns User object if token is valid. def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db)): credentials_exception = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -252,14 +240,10 @@ def get_current_user(token: str = Depends(oauth2_scheme), db=Depends(get_auth_db # #region has_permission [C:3] [TYPE Function] +# @RELATION CALLS -> AuthRepository # @BRIEF Dependency for checking if the current user has a specific permission. -# @PRE User is authenticated. -# @POST Returns True if user has permission. -# @RELATION CALLS -> [AuthRepository] -# @THROW: HTTPException 403 if permission is denied. -# @PARAM: resource (str) - The resource identifier. -# @PARAM: action (str) - The action identifier (READ, EXECUTE, WRITE). -# @RETURN: User - The authenticated user if permission granted. +# @PRE: User is authenticated. +# @POST: Returns True if user has permission. def has_permission(resource: str, action: str): def permission_checker(current_user: User = Depends(get_current_user)): # Union of all permissions across all roles diff --git a/backend/src/models/__init__.py b/backend/src/models/__init__.py index 3c21243d..2a749b43 100644 --- a/backend/src/models/__init__.py +++ b/backend/src/models/__init__.py @@ -1,2 +1,3 @@ -# #region ModelsPackage [C:1] [TYPE Package] +# #region ModelsPackage [TYPE Package] +# @BRIEF Domain model package root. # #endregion ModelsPackage diff --git a/backend/src/models/__tests__/test_clean_release.py b/backend/src/models/__tests__/test_clean_release.py index 2304fbbd..14e6e8d0 100644 --- a/backend/src/models/__tests__/test_clean_release.py +++ b/backend/src/models/__tests__/test_clean_release.py @@ -1,7 +1,7 @@ -# #region TestCleanReleaseModels [TYPE Module] -# @BRIEF Contract testing for Clean Release models -# @RELATION VERIFIES -> [CleanReleaseModels] -# #endregion TestCleanReleaseModels +# [DEF:TestCleanReleaseModels:Module] +# @RELATION: VERIFIES -> [CleanReleaseModels] +# @PURPOSE: Contract testing for Clean Release models +# [/DEF:TestCleanReleaseModels:Module] import pytest from datetime import datetime @@ -38,21 +38,21 @@ def valid_candidate_data(): } -# #region test_release_candidate_valid [TYPE Function] -# @BRIEF Verify that a valid release candidate can be instantiated. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_release_candidate_valid:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that a valid release candidate can be instantiated. def test_release_candidate_valid(valid_candidate_data): rc = ReleaseCandidate(**valid_candidate_data) assert rc.candidate_id == "RC-001" assert rc.status == ReleaseCandidateStatus.DRAFT -# #endregion test_release_candidate_valid +# [/DEF:test_release_candidate_valid:Function] -# #region test_release_candidate_empty_id [TYPE Function] -# @BRIEF Verify that a release candidate with an empty ID is rejected. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_release_candidate_empty_id:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that a release candidate with an empty ID is rejected. def test_release_candidate_empty_id(valid_candidate_data): valid_candidate_data["candidate_id"] = " " with pytest.raises(ValueError, match="candidate_id must be non-empty"): @@ -60,7 +60,7 @@ def test_release_candidate_empty_id(valid_candidate_data): # @TEST_FIXTURE: valid_enterprise_policy -# #endregion test_release_candidate_empty_id +# [/DEF:test_release_candidate_empty_id:Function] @pytest.fixture @@ -78,21 +78,21 @@ def valid_policy_data(): # @TEST_INVARIANT: policy_purity -# #region test_enterprise_policy_valid [TYPE Function] -# @BRIEF Verify that a valid enterprise policy is accepted. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_enterprise_policy_valid:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that a valid enterprise policy is accepted. def test_enterprise_policy_valid(valid_policy_data): policy = CleanProfilePolicy(**valid_policy_data) assert policy.external_source_forbidden is True # @TEST_EDGE: enterprise_policy_missing_prohibited -# #endregion test_enterprise_policy_valid +# [/DEF:test_enterprise_policy_valid:Function] -# #region test_enterprise_policy_missing_prohibited [TYPE Function] -# @BRIEF Verify that an enterprise policy without prohibited categories is rejected. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_enterprise_policy_missing_prohibited:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that an enterprise policy without prohibited categories is rejected. def test_enterprise_policy_missing_prohibited(valid_policy_data): valid_policy_data["prohibited_artifact_categories"] = [] with pytest.raises( @@ -103,12 +103,12 @@ def test_enterprise_policy_missing_prohibited(valid_policy_data): # @TEST_EDGE: enterprise_policy_external_allowed -# #endregion test_enterprise_policy_missing_prohibited +# [/DEF:test_enterprise_policy_missing_prohibited:Function] -# #region test_enterprise_policy_external_allowed [TYPE Function] -# @BRIEF Verify that an enterprise policy allowing external sources is rejected. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_enterprise_policy_external_allowed:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that an enterprise policy allowing external sources is rejected. def test_enterprise_policy_external_allowed(valid_policy_data): valid_policy_data["external_source_forbidden"] = False with pytest.raises( @@ -120,12 +120,12 @@ def test_enterprise_policy_external_allowed(valid_policy_data): # @TEST_INVARIANT: manifest_consistency # @TEST_EDGE: manifest_count_mismatch -# #endregion test_enterprise_policy_external_allowed +# [/DEF:test_enterprise_policy_external_allowed:Function] -# #region test_manifest_count_mismatch [TYPE Function] -# @BRIEF Verify that a manifest with count mismatches is rejected. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_manifest_count_mismatch:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify that a manifest with count mismatches is rejected. def test_manifest_count_mismatch(): summary = ManifestSummary( included_count=1, excluded_count=0, prohibited_detected_count=0 @@ -165,12 +165,12 @@ def test_manifest_count_mismatch(): # @TEST_INVARIANT: run_integrity # @TEST_EDGE: compliant_run_stage_fail -# #endregion test_manifest_count_mismatch +# [/DEF:test_manifest_count_mismatch:Function] -# #region test_compliant_run_validation [TYPE Function] -# @BRIEF Verify compliant run validation logic and mandatory stage checks. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_compliant_run_validation:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify compliant run validation logic and mandatory stage checks. def test_compliant_run_validation(): base_run = { "check_run_id": "run1", @@ -211,12 +211,12 @@ def test_compliant_run_validation(): ComplianceCheckRun(**base_run) -# #endregion test_compliant_run_validation +# [/DEF:test_compliant_run_validation:Function] -# #region test_report_validation [TYPE Function] -# @BRIEF Verify compliance report validation based on status and violation counts. -# @RELATION BINDS_TO -> [TestCleanReleaseModels] +# [DEF:test_report_validation:Function] +# @RELATION: BINDS_TO -> [TestCleanReleaseModels] +# @PURPOSE: Verify compliance report validation based on status and violation counts. def test_report_validation(): # Valid blocked report ComplianceReport( @@ -246,4 +246,4 @@ def test_report_validation(): ) -# #endregion test_report_validation +# [/DEF:test_report_validation:Function] diff --git a/backend/src/models/__tests__/test_models.py b/backend/src/models/__tests__/test_models.py index b7166526..fd3e321a 100644 --- a/backend/src/models/__tests__/test_models.py +++ b/backend/src/models/__tests__/test_models.py @@ -1,7 +1,8 @@ -# #region test_models [C:1] [TYPE Module] -# @BRIEF Unit tests for data models -# @LAYER Domain -# @RELATION VERIFIES -> [ModelsPackage] +# [DEF:test_models:Module] +# @COMPLEXITY: 1 +# @PURPOSE: Unit tests for data models +# @LAYER: Domain +# @RELATION: VERIFIES -> [ModelsPackage] import sys from pathlib import Path @@ -13,11 +14,11 @@ from src.core.config_models import Environment from src.core.logger import belief_scope -# #region test_environment_model [TYPE Function] -# @BRIEF Tests that Environment model correctly stores values. -# @PRE Environment class is available. -# @POST Values are verified. -# @RELATION BINDS_TO -> [test_models] +# [DEF:test_environment_model:Function] +# @RELATION: BINDS_TO -> test_models +# @PURPOSE: Tests that Environment model correctly stores values. +# @PRE: Environment class is available. +# @POST: Values are verified. def test_environment_model(): with belief_scope("test_environment_model"): env = Environment( @@ -32,7 +33,7 @@ def test_environment_model(): assert env.url == "http://localhost:8088/api/v1" -# #endregion test_environment_model +# [/DEF:test_environment_model:Function] -# #endregion test_models +# [/DEF:test_models:Module] diff --git a/backend/src/models/__tests__/test_report_models.py b/backend/src/models/__tests__/test_report_models.py index bee3302e..b39a9892 100644 --- a/backend/src/models/__tests__/test_report_models.py +++ b/backend/src/models/__tests__/test_report_models.py @@ -1,7 +1,8 @@ -# #region test_report_models [C:3] [TYPE Module] -# @BRIEF Unit tests for report Pydantic models and their validators -# @LAYER Domain -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:test_report_models:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for report Pydantic models and their validators +# @LAYER: Domain import sys from pathlib import Path @@ -231,4 +232,4 @@ class TestReportDetailView: assert detail.diagnostics["cause"] == "timeout" assert "Retry" in detail.next_actions -# #endregion test_report_models +# [/DEF:test_report_models:Module] diff --git a/backend/src/models/assistant.py b/backend/src/models/assistant.py index df87ccff..55f3c281 100644 --- a/backend/src/models/assistant.py +++ b/backend/src/models/assistant.py @@ -1,8 +1,8 @@ # #region AssistantModels [C:3] [TYPE Module] [SEMANTICS assistant, audit, confirmation, chat] # @BRIEF SQLAlchemy models for assistant audit trail and confirmation tokens. -# @LAYER Domain -# @INVARIANT Assistant records preserve immutable ids and creation timestamps. -# @RELATION DEPENDS_ON -> [MappingModels] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> MappingModels +# @INVARIANT: Assistant records preserve immutable ids and creation timestamps. from datetime import datetime @@ -13,9 +13,9 @@ from .mapping import Base # #region AssistantAuditRecord [C:3] [TYPE Class] # @BRIEF Store audit decisions and outcomes produced by assistant command handling. -# @PRE user_id must identify the actor for every record. -# @POST Audit payload remains available for compliance and debugging. -# @RELATION INHERITS -> [MappingModels] +# @RELATION INHERITS -> MappingModels +# @PRE: user_id must identify the actor for every record. +# @POST: Audit payload remains available for compliance and debugging. class AssistantAuditRecord(Base): __tablename__ = "assistant_audit" @@ -34,9 +34,9 @@ class AssistantAuditRecord(Base): # #region AssistantMessageRecord [C:3] [TYPE Class] # @BRIEF Persist chat history entries for assistant conversations. -# @PRE user_id, conversation_id, role and text must be present. -# @POST Message row can be queried in chronological order. -# @RELATION INHERITS -> [MappingModels] +# @RELATION INHERITS -> MappingModels +# @PRE: user_id, conversation_id, role and text must be present. +# @POST: Message row can be queried in chronological order. class AssistantMessageRecord(Base): __tablename__ = "assistant_messages" @@ -57,9 +57,9 @@ class AssistantMessageRecord(Base): # #region AssistantConfirmationRecord [C:3] [TYPE Class] # @BRIEF Persist risky operation confirmation tokens with lifecycle state. -# @PRE intent/dispatch and expiry timestamp must be provided. -# @POST State transitions can be tracked and audited. -# @RELATION INHERITS -> [MappingModels] +# @RELATION INHERITS -> MappingModels +# @PRE: intent/dispatch and expiry timestamp must be provided. +# @POST: State transitions can be tracked and audited. class AssistantConfirmationRecord(Base): __tablename__ = "assistant_confirmations" diff --git a/backend/src/models/auth.py b/backend/src/models/auth.py index 09303904..8ec04b89 100644 --- a/backend/src/models/auth.py +++ b/backend/src/models/auth.py @@ -1,22 +1,20 @@ # #region AuthModels [C:3] [TYPE Module] [SEMANTICS auth, models, user, role, permission, sqlalchemy] # @BRIEF SQLAlchemy models for multi-user authentication and authorization. -# @LAYER Domain -# @INVARIANT Usernames and emails must be unique. +# @LAYER: Domain # @RELATION INHERITS_FROM -> [Base] # +# @INVARIANT: Usernames and emails must be unique. -# [SECTION: IMPORTS] import uuid from datetime import datetime from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey, Table from sqlalchemy.orm import relationship from .mapping import Base -# [/SECTION] # #region generate_uuid [TYPE Function] # @BRIEF Generates a unique UUID string. -# @POST Returns a string representation of a new UUID. +# @POST: Returns a string representation of a new UUID. # @RELATION DEPENDS_ON -> [uuid] def generate_uuid(): return str(uuid.uuid4()) diff --git a/backend/src/models/clean_release.py b/backend/src/models/clean_release.py index ed172607..71a7b3b8 100644 --- a/backend/src/models/clean_release.py +++ b/backend/src/models/clean_release.py @@ -1,12 +1,12 @@ # #region CleanReleaseModels [C:3] [TYPE Module] [SEMANTICS clean-release, models, lifecycle, compliance, evidence, immutability] # @BRIEF Define canonical clean release domain entities and lifecycle guards. -# @LAYER Domain -# @PRE Base mapping model and release enums are available. -# @POST Provides SQLAlchemy and dataclass definitions for governance domain. -# @SIDE_EFFECT None (schema definition). -# @DATA_CONTRACT Model[ReleaseCandidate, CandidateArtifact, DistributionManifest, ComplianceRun, ComplianceReport] -# @INVARIANT Immutable snapshots are never mutated; forbidden lifecycle transitions are rejected. -# @RELATION DEPENDS_ON -> [MappingModels] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> MappingModels +# @PRE: Base mapping model and release enums are available. +# @POST: Provides SQLAlchemy and dataclass definitions for governance domain. +# @SIDE_EFFECT: None (schema definition). +# @DATA_CONTRACT: Model[ReleaseCandidate, CandidateArtifact, DistributionManifest, ComplianceRun, ComplianceReport] +# @INVARIANT: Immutable snapshots are never mutated; forbidden lifecycle transitions are rejected. from datetime import datetime from dataclasses import dataclass @@ -229,8 +229,8 @@ class ComplianceCheckRun: # #region ReleaseCandidate [TYPE Class] # @BRIEF Represents the release unit being prepared and governed. -# @PRE id, version, source_snapshot_ref are non-empty. -# @POST status advances only through legal transitions. +# @PRE: id, version, source_snapshot_ref are non-empty. +# @POST: status advances only through legal transitions. class ReleaseCandidate(Base): __tablename__ = "clean_release_candidates" @@ -328,7 +328,7 @@ class ManifestSummary: # #region DistributionManifest [TYPE Class] # @BRIEF Immutable snapshot of the candidate payload. -# @INVARIANT Immutable after creation. +# @INVARIANT: Immutable after creation. class DistributionManifest(Base): __tablename__ = "clean_release_manifests" @@ -579,7 +579,7 @@ class ComplianceViolation(Base): # #region ComplianceReport [TYPE Class] # @BRIEF Immutable result derived from a completed run. -# @INVARIANT Immutable after creation. +# @INVARIANT: Immutable after creation. class ComplianceReport(Base): __tablename__ = "clean_release_compliance_reports" @@ -694,4 +694,4 @@ class CleanReleaseAuditLog(Base): details_json = Column(JSON, default=dict) # #endregion CleanReleaseAuditLog -# #endregion CleanReleaseModels +# #endregion CleanReleaseModels \ No newline at end of file diff --git a/backend/src/models/config.py b/backend/src/models/config.py index 84a56f46..209507c0 100644 --- a/backend/src/models/config.py +++ b/backend/src/models/config.py @@ -1,10 +1,10 @@ # #region ConfigModels [C:3] [TYPE Module] [SEMANTICS database, config, settings, sqlalchemy, notification] -# @BRIEF Defines SQLAlchemy persistence models for application and notification configuration records. -# @LAYER Domain -# @INVARIANT Configuration payload and notification credentials must remain persisted as non-null JSON documents. -# @RELATION DEPENDS_ON -> [MappingModels:Base] # +# @BRIEF Defines SQLAlchemy persistence models for application and notification configuration records. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [MappingModels:Base] +# @INVARIANT: Configuration payload and notification credentials must remain persisted as non-null JSON documents. from sqlalchemy import Column, String, DateTime, JSON, Boolean from sqlalchemy.sql import func @@ -14,10 +14,10 @@ from .mapping import Base # #region AppConfigRecord [TYPE Class] # @BRIEF Stores persisted application configuration as a single authoritative record model. -# @PRE SQLAlchemy declarative Base is initialized and table metadata registration is active. -# @POST ORM table 'app_configurations' exposes id, payload, and updated_at fields with declared nullability/default semantics. -# @SIDE_EFFECT Registers ORM mapping metadata during module import. -# @DATA_CONTRACT Input -> persistence row {id:str, payload:json, updated_at:datetime}; Output -> AppConfigRecord ORM entity. +# @PRE: SQLAlchemy declarative Base is initialized and table metadata registration is active. +# @POST: ORM table 'app_configurations' exposes id, payload, and updated_at fields with declared nullability/default semantics. +# @SIDE_EFFECT: Registers ORM mapping metadata during module import. +# @DATA_CONTRACT: Input -> persistence row {id:str, payload:json, updated_at:datetime}; Output -> AppConfigRecord ORM entity. class AppConfigRecord(Base): __tablename__ = "app_configurations" @@ -30,10 +30,10 @@ class AppConfigRecord(Base): # #region NotificationConfig [TYPE Class] # @BRIEF Stores persisted provider-level notification configuration and encrypted credentials metadata. -# @PRE SQLAlchemy declarative Base is initialized and uuid generation is available at instance creation time. -# @POST ORM table 'notification_configs' exposes id, type, name, credentials, is_active, created_at, updated_at fields with declared constraints/defaults. -# @SIDE_EFFECT Registers ORM mapping metadata during module import; may generate UUID values for new entity instances. -# @DATA_CONTRACT Input -> persistence row {id:str, type:str, name:str, credentials:json, is_active:bool, created_at:datetime, updated_at:datetime}; Output -> NotificationConfig ORM entity. +# @PRE: SQLAlchemy declarative Base is initialized and uuid generation is available at instance creation time. +# @POST: ORM table 'notification_configs' exposes id, type, name, credentials, is_active, created_at, updated_at fields with declared constraints/defaults. +# @SIDE_EFFECT: Registers ORM mapping metadata during module import; may generate UUID values for new entity instances. +# @DATA_CONTRACT: Input -> persistence row {id:str, type:str, name:str, credentials:json, is_active:bool, created_at:datetime, updated_at:datetime}; Output -> NotificationConfig ORM entity. class NotificationConfig(Base): __tablename__ = "notification_configs" diff --git a/backend/src/models/connection.py b/backend/src/models/connection.py index a358dc4a..7d60a1c8 100644 --- a/backend/src/models/connection.py +++ b/backend/src/models/connection.py @@ -1,17 +1,15 @@ # #region ConnectionModels [C:1] [TYPE Module] [SEMANTICS database, connection, configuration, sqlalchemy, sqlite] +# # @BRIEF Defines the database schema for external database connection configurations. -# @LAYER Domain -# @INVARIANT All primary keys are UUID strings. -# @RELATION DEPENDS_ON -> [sqlalchemy] -# +# @LAYER: Domain +# @RELATION DEPENDS_ON -> sqlalchemy # +# @INVARIANT: All primary keys are UUID strings. -# [SECTION: IMPORTS] from sqlalchemy import Column, String, Integer, DateTime from sqlalchemy.sql import func from .mapping import Base import uuid -# [/SECTION] # #region ConnectionConfig [C:1] [TYPE Class] # @BRIEF Stores credentials for external databases used for column mapping. @@ -30,4 +28,4 @@ class ConnectionConfig(Base): updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) # #endregion ConnectionConfig -# #endregion ConnectionModels +# #endregion ConnectionModels \ No newline at end of file diff --git a/backend/src/models/dashboard.py b/backend/src/models/dashboard.py index 3e7a4225..951418f2 100644 --- a/backend/src/models/dashboard.py +++ b/backend/src/models/dashboard.py @@ -1,7 +1,7 @@ # #region DashboardModels [C:3] [TYPE Module] [SEMANTICS dashboard, model, metadata, migration] # @BRIEF Defines data models for dashboard metadata and selection. -# @LAYER Model -# @RELATION USED_BY -> [MigrationApi] +# @LAYER: Model +# @RELATION USED_BY -> MigrationApi from pydantic import BaseModel from typing import List @@ -25,4 +25,4 @@ class DashboardSelection(BaseModel): fix_cross_filters: bool = True # #endregion DashboardSelection -# #endregion DashboardModels +# #endregion DashboardModels \ No newline at end of file diff --git a/backend/src/models/dataset_review.py b/backend/src/models/dataset_review.py index aa6e0696..a0f5856f 100644 --- a/backend/src/models/dataset_review.py +++ b/backend/src/models/dataset_review.py @@ -1,7 +1,6 @@ # #region DatasetReviewModels [C:2] [TYPE Module] [SEMANTICS dataset_review, session, profile, findings, semantics, clarification, execution, sqlalchemy] # @BRIEF Thin facade re-exporting all dataset review domain models from the decomposed sub-package. -# @LAYER Domain -# @INVARIANT All public model classes and enums remain importable from `src.models.dataset_review` without changes. +# @LAYER: Domain # @RELATION EXPORTS -> [DatasetReviewEnums:Module] # @RELATION EXPORTS -> [DatasetReviewSessionModels:Module] # @RELATION EXPORTS -> [DatasetReviewProfileModels:Module] @@ -11,8 +10,9 @@ # @RELATION EXPORTS -> [DatasetReviewMappingModels:Module] # @RELATION EXPORTS -> [DatasetReviewClarificationModels:Module] # @RELATION EXPORTS -> [DatasetReviewExecutionModels:Module] -# @RATIONALE Original 984-line monolith violated INV_7 (400-line module limit). Decomposed into domain-focused sub-modules while preserving backward-compatible import paths. -# @REJECTED Keeping all models in a single file because it exceeded the fractal limit by 2.5x and accumulated structural erosion risk. +# @INVARIANT: All public model classes and enums remain importable from `src.models.dataset_review` without changes. +# @RATIONALE: Original 984-line monolith violated INV_7 (400-line module limit). Decomposed into domain-focused sub-modules while preserving backward-compatible import paths. +# @REJECTED: Keeping all models in a single file because it exceeded the fractal limit by 2.5x and accumulated structural erosion risk. from src.models.dataset_review_pkg._enums import ( # noqa: F401 SessionStatus, diff --git a/backend/src/models/dataset_review_pkg/__init__.py b/backend/src/models/dataset_review_pkg/__init__.py index fa6a627c..1804ed0b 100644 --- a/backend/src/models/dataset_review_pkg/__init__.py +++ b/backend/src/models/dataset_review_pkg/__init__.py @@ -1,6 +1,6 @@ # #region DatasetReviewModels [C:3] [TYPE Module] [SEMANTICS dataset_review, session, profile, findings, semantics, clarification, execution, sqlalchemy] # @BRIEF Re-export all dataset review domain models from decomposed sub-modules for backward-compatible imports. -# @LAYER Domain +# @LAYER: Domain from src.models.dataset_review_pkg._enums import ( SessionStatus, diff --git a/backend/src/models/dataset_review_pkg/_clarification_models.py b/backend/src/models/dataset_review_pkg/_clarification_models.py index 08411b32..6c26b7a3 100644 --- a/backend/src/models/dataset_review_pkg/_clarification_models.py +++ b/backend/src/models/dataset_review_pkg/_clarification_models.py @@ -1,9 +1,9 @@ # #region DatasetReviewClarificationModels [C:3] [TYPE Module] # @BRIEF Clarification session, question, option, and answer models for guided review flow. -# @LAYER Domain -# @INVARIANT Only one active clarification question may exist at a time per session. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] +# @INVARIANT: Only one active clarification question may exist at a time per session. import uuid from datetime import datetime diff --git a/backend/src/models/dataset_review_pkg/_enums.py b/backend/src/models/dataset_review_pkg/_enums.py index a0063acc..3ac70a7e 100644 --- a/backend/src/models/dataset_review_pkg/_enums.py +++ b/backend/src/models/dataset_review_pkg/_enums.py @@ -1,7 +1,7 @@ # #region DatasetReviewEnums [C:2] [TYPE Module] # @BRIEF All enumeration types for the dataset review domain, grouped for stable cross-module reuse. -# @LAYER Domain -# @INVARIANT Enum values are string-based for JSON serialization compatibility. +# @LAYER: Domain +# @INVARIANT: Enum values are string-based for JSON serialization compatibility. import enum diff --git a/backend/src/models/dataset_review_pkg/_execution_models.py b/backend/src/models/dataset_review_pkg/_execution_models.py index 8c50f928..2bd2015b 100644 --- a/backend/src/models/dataset_review_pkg/_execution_models.py +++ b/backend/src/models/dataset_review_pkg/_execution_models.py @@ -1,6 +1,6 @@ # #region DatasetReviewExecutionModels [C:3] [TYPE Module] # @BRIEF Compiled preview, run context, session event, and export artifact models for execution and audit. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] diff --git a/backend/src/models/dataset_review_pkg/_filter_models.py b/backend/src/models/dataset_review_pkg/_filter_models.py index 220f8c04..fdbb123f 100644 --- a/backend/src/models/dataset_review_pkg/_filter_models.py +++ b/backend/src/models/dataset_review_pkg/_filter_models.py @@ -1,6 +1,6 @@ # #region DatasetReviewFilterModels [C:3] [TYPE Module] # @BRIEF Imported filter and template variable models for Superset context recovery and execution mapping. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] diff --git a/backend/src/models/dataset_review_pkg/_finding_models.py b/backend/src/models/dataset_review_pkg/_finding_models.py index 23a61672..32d0e38b 100644 --- a/backend/src/models/dataset_review_pkg/_finding_models.py +++ b/backend/src/models/dataset_review_pkg/_finding_models.py @@ -1,6 +1,6 @@ # #region DatasetReviewFindingModels [C:2] [TYPE Module] # @BRIEF Validation finding model for tracking blocking, warning, and informational issues during review. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] diff --git a/backend/src/models/dataset_review_pkg/_mapping_models.py b/backend/src/models/dataset_review_pkg/_mapping_models.py index 6fccadb7..0c1bd1bf 100644 --- a/backend/src/models/dataset_review_pkg/_mapping_models.py +++ b/backend/src/models/dataset_review_pkg/_mapping_models.py @@ -1,6 +1,6 @@ # #region DatasetReviewMappingModels [C:2] [TYPE Module] # @BRIEF Execution mapping model linking imported filters to template variables with approval gates. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] @@ -29,8 +29,8 @@ from src.models.dataset_review_pkg._enums import ( # #region ExecutionMapping [C:2] [TYPE Class] # @BRIEF One filter-to-variable mapping with approval gate, effective value, and transformation metadata. -# @INVARIANT Explicit approval is required before launch when requires_explicit_approval is true. # @RELATION DEPENDS_ON -> [DatasetReviewSession] +# @INVARIANT: Explicit approval is required before launch when requires_explicit_approval is true. class ExecutionMapping(Base): __tablename__ = "execution_mappings" diff --git a/backend/src/models/dataset_review_pkg/_profile_models.py b/backend/src/models/dataset_review_pkg/_profile_models.py index 5035dfbf..5baa4f14 100644 --- a/backend/src/models/dataset_review_pkg/_profile_models.py +++ b/backend/src/models/dataset_review_pkg/_profile_models.py @@ -1,6 +1,6 @@ # #region DatasetReviewProfileModels [C:2] [TYPE Module] # @BRIEF Dataset profile model capturing business summary, confidence, and completeness metadata. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] diff --git a/backend/src/models/dataset_review_pkg/_semantic_models.py b/backend/src/models/dataset_review_pkg/_semantic_models.py index 0d0e737d..c6b2d9d0 100644 --- a/backend/src/models/dataset_review_pkg/_semantic_models.py +++ b/backend/src/models/dataset_review_pkg/_semantic_models.py @@ -1,9 +1,9 @@ # #region DatasetReviewSemanticModels [C:3] [TYPE Module] # @BRIEF Semantic source, field entry, and candidate models for dictionary-driven semantic enrichment. -# @LAYER Domain -# @INVARIANT Manual overrides are never silently replaced by imported, inferred, or AI-generated values. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] +# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values. import uuid from datetime import datetime @@ -64,9 +64,9 @@ class SemanticSource(Base): # #region SemanticFieldEntry [C:3] [TYPE Class] # @BRIEF Per-field semantic metadata entry with provenance tracking, lock state, and candidate set. -# @INVARIANT Locked fields preserve their active value regardless of later candidate proposals. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [SemanticCandidate] +# @INVARIANT: Locked fields preserve their active value regardless of later candidate proposals. class SemanticFieldEntry(Base): __tablename__ = "semantic_field_entries" diff --git a/backend/src/models/dataset_review_pkg/_session_models.py b/backend/src/models/dataset_review_pkg/_session_models.py index aea8031e..cdbd5638 100644 --- a/backend/src/models/dataset_review_pkg/_session_models.py +++ b/backend/src/models/dataset_review_pkg/_session_models.py @@ -1,9 +1,9 @@ # #region DatasetReviewSessionModels [C:3] [TYPE Module] # @BRIEF Session aggregate root and collaborator models for dataset review orchestration. -# @LAYER Domain -# @INVARIANT Session and profile entities are strictly scoped to an authenticated user. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewEnums:Module] # @RELATION DEPENDS_ON -> [MappingModels] +# @INVARIANT: Session and profile entities are strictly scoped to an authenticated user. import uuid from datetime import datetime @@ -51,7 +51,6 @@ class SessionCollaborator(Base): # #region DatasetReviewSession [C:3] [TYPE Class] # @BRIEF Aggregate root for the dataset review lifecycle, owning all child entities and driving readiness transitions. -# @INVARIANT Optimistic-lock version column prevents lost-update races on concurrent mutations. # @RELATION DEPENDS_ON -> [SessionCollaborator] # @RELATION DEPENDS_ON -> [DatasetProfile] # @RELATION DEPENDS_ON -> [ValidationFinding] @@ -65,6 +64,7 @@ class SessionCollaborator(Base): # @RELATION DEPENDS_ON -> [DatasetRunContext] # @RELATION DEPENDS_ON -> [ExportArtifact] # @RELATION DEPENDS_ON -> [SessionEvent] +# @INVARIANT: Optimistic-lock version column prevents lost-update races on concurrent mutations. class DatasetReviewSession(Base): __tablename__ = "dataset_review_sessions" diff --git a/backend/src/models/filter_state.py b/backend/src/models/filter_state.py index 932c642d..f13c3932 100644 --- a/backend/src/models/filter_state.py +++ b/backend/src/models/filter_state.py @@ -1,18 +1,16 @@ # #region FilterStateModels [C:2] [TYPE Module] [SEMANTICS superset, native, filters, pydantic, models, dataclasses] -# @BRIEF Pydantic models for Superset native filter state extraction and restoration. -# @LAYER Models -# @RELATION DEPENDS_ON -> [pydantic] # +# @BRIEF Pydantic models for Superset native filter state extraction and restoration. +# @LAYER: Models +# @RELATION DEPENDS_ON -> [pydantic] -# [SECTION: IMPORTS] from typing import Any, Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field -# [/SECTION] # #region FilterState [C:2] [TYPE Model] # @BRIEF Represents the state of a single native filter. -# @DATA_CONTRACT Input[extraFormData: Dict, filterState: Dict, ownState: Optional[Dict]] -> Model[FilterState] +# @DATA_CONTRACT: Input[extraFormData: Dict, filterState: Dict, ownState: Optional[Dict]] -> Model[FilterState] class FilterState(BaseModel): """Single native filter state with extraFormData, filterState, and ownState.""" @@ -26,7 +24,7 @@ class FilterState(BaseModel): # #region NativeFilterDataMask [C:2] [TYPE Model] # @BRIEF Represents the dataMask containing all native filter states. -# @DATA_CONTRACT Input[Dict[filter_id, FilterState]] -> Model[NativeFilterDataMask] +# @DATA_CONTRACT: Input[Dict[filter_id, FilterState]] -> Model[NativeFilterDataMask] class NativeFilterDataMask(BaseModel): """Container for all native filter states in a dashboard.""" @@ -49,7 +47,7 @@ class NativeFilterDataMask(BaseModel): # #region ParsedNativeFilters [C:2] [TYPE Model] # @BRIEF Result of parsing native filters from permalink or native_filters_key. -# @DATA_CONTRACT Input[dataMask: Dict, metadata: Dict] -> Model[ParsedNativeFilters] +# @DATA_CONTRACT: Input[dataMask: Dict, metadata: Dict] -> Model[ParsedNativeFilters] class ParsedNativeFilters(BaseModel): """Result of extracting native filters from a Superset URL.""" @@ -76,7 +74,7 @@ class ParsedNativeFilters(BaseModel): # #region DashboardURLFilterExtraction [C:2] [TYPE Model] # @BRIEF Result of parsing a complete dashboard URL for filter information. -# @DATA_CONTRACT Input[url: str, dashboard_id: Optional, filter_type: Optional, filters: Dict] -> Model[DashboardURLFilterExtraction] +# @DATA_CONTRACT: Input[url: str, dashboard_id: Optional, filter_type: Optional, filters: Dict] -> Model[DashboardURLFilterExtraction] class DashboardURLFilterExtraction(BaseModel): """Result of parsing a Superset dashboard URL to extract filter state.""" @@ -93,7 +91,7 @@ class DashboardURLFilterExtraction(BaseModel): # #region ExtraFormDataMerge [C:2] [TYPE Model] # @BRIEF Configuration for merging extraFormData from different sources. -# @DATA_CONTRACT Input[append_keys: List[str], override_keys: List[str]] -> Model[ExtraFormDataMerge] +# @DATA_CONTRACT: Input[append_keys: List[str], override_keys: List[str]] -> Model[ExtraFormDataMerge] class ExtraFormDataMerge(BaseModel): """Configuration for merging extraFormData between original and new filter values.""" @@ -141,4 +139,4 @@ class ExtraFormDataMerge(BaseModel): # #endregion ExtraFormDataMerge -# #endregion FilterStateModels +# #endregion FilterStateModels \ No newline at end of file diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index b106f17f..f42e8e42 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -1,7 +1,7 @@ # #region LlmModels [C:3] [TYPE Module] [SEMANTICS llm, models, sqlalchemy, persistence] # @BRIEF SQLAlchemy models for LLM provider configuration and validation results. -# @LAYER Domain -# @RELATION INHERITS_FROM -> [MappingModels:Base] +# @LAYER: Domain +# @RELATION INHERITS_FROM -> MappingModels:Base from sqlalchemy import Column, String, Boolean, DateTime, JSON, Text, Time, ForeignKey from datetime import datetime @@ -63,4 +63,4 @@ class ValidationRecord(Base): raw_response = Column(Text, nullable=True) # #endregion ValidationRecord -# #endregion LlmModels +# #endregion LlmModels \ No newline at end of file diff --git a/backend/src/models/mapping.py b/backend/src/models/mapping.py index cea0e1fc..713056c0 100644 --- a/backend/src/models/mapping.py +++ b/backend/src/models/mapping.py @@ -1,20 +1,18 @@ # #region MappingModels [C:3] [TYPE Module] [SEMANTICS database, mapping, environment, migration, sqlalchemy, sqlite] -# @BRIEF Defines the database schema for environment metadata and database mappings using SQLAlchemy. -# @LAYER Domain -# @INVARIANT All primary keys are UUID strings. -# @RELATION DEPENDS_ON -> [sqlalchemy] # +# @BRIEF Defines the database schema for environment metadata and database mappings using SQLAlchemy. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> sqlalchemy # +# @INVARIANT: All primary keys are UUID strings. # @CONSTRAINT: source_env_id and target_env_id must be valid environment IDs. -# [SECTION: IMPORTS] from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey, Enum as SQLEnum from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import func import uuid import enum -# [/SECTION] Base = declarative_base() @@ -39,7 +37,7 @@ class MigrationStatus(enum.Enum): # #region Environment [C:3] [TYPE Class] # @BRIEF Represents a Superset instance environment. -# @RELATION DEPENDS_ON -> [MappingModels] +# @RELATION DEPENDS_ON -> MappingModels class Environment(Base): __tablename__ = "environments" @@ -80,7 +78,7 @@ class MigrationJob(Base): # #region ResourceMapping [C:3] [TYPE Class] # @BRIEF Maps a universal UUID for a resource to its actual ID on a specific environment. # @TEST_DATA: resource_mapping_record -> {'environment_id': 'prod-env-1', 'resource_type': 'chart', 'uuid': '123e4567-e89b-12d3-a456-426614174000', 'remote_integer_id': '42'} -# @RELATION: DEPENDS_ON -> MappingModels +# @RELATION DEPENDS_ON -> MappingModels class ResourceMapping(Base): __tablename__ = "resource_mappings" @@ -94,3 +92,4 @@ class ResourceMapping(Base): # #endregion ResourceMapping # #endregion MappingModels + diff --git a/backend/src/models/profile.py b/backend/src/models/profile.py index 2524b9bc..22712845 100644 --- a/backend/src/models/profile.py +++ b/backend/src/models/profile.py @@ -1,25 +1,23 @@ # #region ProfileModels [C:3] [TYPE Module] [SEMANTICS profile, preferences, persistence, user, dashboard-filter, git, ui-preferences, sqlalchemy] +# # @BRIEF Defines persistent per-user profile settings for dashboard filter, Git identity/token, and UX preferences. -# @LAYER Domain -# @INVARIANT Exactly one preference row exists per user_id. -# @INVARIANT Sensitive Git token is stored encrypted and never returned in plaintext. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [AuthModels] # @RELATION INHERITS_FROM -> [MappingModels:Base] # -# +# @INVARIANT: Exactly one preference row exists per user_id. +# @INVARIANT: Sensitive Git token is stored encrypted and never returned in plaintext. -# [SECTION: IMPORTS] import uuid from datetime import datetime from sqlalchemy import Column, String, Boolean, DateTime, ForeignKey from sqlalchemy.orm import relationship from .mapping import Base -# [/SECTION] # #region UserDashboardPreference [C:3] [TYPE Class] # @BRIEF Stores Superset username binding and default "my dashboards" toggle for one authenticated user. -# @RELATION INHERITS -> [MappingModels:Base] +# @RELATION INHERITS -> MappingModels:Base class UserDashboardPreference(Base): __tablename__ = "user_dashboard_preferences" diff --git a/backend/src/models/report.py b/backend/src/models/report.py index e7b27e34..cd561ca5 100644 --- a/backend/src/models/report.py +++ b/backend/src/models/report.py @@ -1,26 +1,24 @@ # #region ReportModels [C:3] [TYPE Module] [SEMANTICS reports, models, pydantic, normalization, pagination] # @BRIEF Canonical report schemas for unified task reporting across heterogeneous task types. -# @LAYER Domain -# @PRE Pydantic library and task manager models are available. -# @POST Provides validated schemas for cross-plugin reporting and UI consumption. -# @SIDE_EFFECT None (schema definition). -# @DATA_CONTRACT Model[TaskReport, ReportCollection, ReportDetailView] -# @INVARIANT Canonical report fields are always present for every report item. +# @LAYER: Domain +# @PRE: Pydantic library and task manager models are available. +# @POST: Provides validated schemas for cross-plugin reporting and UI consumption. +# @SIDE_EFFECT: None (schema definition). +# @DATA_CONTRACT: Model[TaskReport, ReportCollection, ReportDetailView] # @RELATION DEPENDS_ON -> [TaskModels] +# @INVARIANT: Canonical report fields are always present for every report item. -# [SECTION: IMPORTS] from datetime import datetime from enum import Enum from typing import Any, Dict, List, Optional from pydantic import BaseModel, Field, field_validator, model_validator -# [/SECTION] # #region TaskType [C:3] [TYPE Class] [SEMANTICS enum, type, task] +# @INVARIANT: Must contain valid generic task type mappings. +# @RELATION DEPENDS_ON -> ReportModels # @BRIEF Supported normalized task report types. -# @INVARIANT Must contain valid generic task type mappings. -# @RELATION DEPENDS_ON -> [ReportModels] class TaskType(str, Enum): LLM_VERIFICATION = "llm_verification" BACKUP = "backup" @@ -34,9 +32,9 @@ class TaskType(str, Enum): # #region ReportStatus [C:3] [TYPE Class] [SEMANTICS enum, status, task] +# @INVARIANT: TaskStatus enum mapping logic holds. # @BRIEF Supported normalized report status values. -# @INVARIANT TaskStatus enum mapping logic holds. -# @RELATION DEPENDS_ON -> [ReportModels] +# @RELATION DEPENDS_ON -> ReportModels class ReportStatus(str, Enum): SUCCESS = "success" FAILED = "failed" @@ -48,8 +46,8 @@ class ReportStatus(str, Enum): # #region ErrorContext [C:3] [TYPE Class] [SEMANTICS error, context, payload] +# @INVARIANT: The properties accurately describe error state. # @BRIEF Error and recovery context for failed/partial reports. -# @INVARIANT The properties accurately describe error state. # # @TEST_CONTRACT: ErrorContextModel -> # { @@ -63,7 +61,7 @@ class ReportStatus(str, Enum): # } # @TEST_FIXTURE: basic_error -> {"message": "Connection timeout", "code": "ERR_504", "next_actions": ["retry"]} # @TEST_EDGE: missing_message -> {"code": "ERR_504"} -# @RELATION: DEPENDS_ON -> ReportModels +# @RELATION DEPENDS_ON -> ReportModels class ErrorContext(BaseModel): code: Optional[str] = None message: str @@ -74,8 +72,8 @@ class ErrorContext(BaseModel): # #region TaskReport [C:3] [TYPE Class] [SEMANTICS report, model, summary] +# @INVARIANT: Must represent canonical task record attributes. # @BRIEF Canonical normalized report envelope for one task execution. -# @INVARIANT Must represent canonical task record attributes. # # @TEST_CONTRACT: TaskReportModel -> # { @@ -106,7 +104,7 @@ class ErrorContext(BaseModel): # @TEST_EDGE: empty_summary -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "migration", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": ""} # @TEST_EDGE: invalid_task_type -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "invalid_type", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"} # @TEST_INVARIANT: non_empty_validators -> verifies: [empty_report_id, empty_summary] -# @RELATION: DEPENDS_ON -> ReportModels +# @RELATION DEPENDS_ON -> ReportModels class TaskReport(BaseModel): report_id: str task_id: str @@ -132,8 +130,8 @@ class TaskReport(BaseModel): # #region ReportQuery [C:3] [TYPE Class] [SEMANTICS query, filter, search] +# @INVARIANT: Time and pagination queries are mutually consistent. # @BRIEF Query object for server-side report filtering, sorting, and pagination. -# @INVARIANT Time and pagination queries are mutually consistent. # # @TEST_CONTRACT: ReportQueryModel -> # { @@ -153,7 +151,7 @@ class TaskReport(BaseModel): # @TEST_EDGE: invalid_sort_by -> {"sort_by": "unknown_field"} # @TEST_EDGE: invalid_time_range -> {"time_from": "2026-02-26T12:00:00Z", "time_to": "2026-02-25T12:00:00Z"} # @TEST_INVARIANT: attribute_constraints_enforced -> verifies: [invalid_page_size_large, invalid_sort_by, invalid_time_range] -# @RELATION: DEPENDS_ON -> ReportModels +# @RELATION DEPENDS_ON -> ReportModels class ReportQuery(BaseModel): page: int = Field(default=1, ge=1) page_size: int = Field(default=20, ge=1, le=100) @@ -191,8 +189,8 @@ class ReportQuery(BaseModel): # #region ReportCollection [C:3] [TYPE Class] [SEMANTICS collection, pagination] +# @INVARIANT: Represents paginated data correctly. # @BRIEF Paginated collection of normalized task reports. -# @INVARIANT Represents paginated data correctly. # # @TEST_CONTRACT: ReportCollectionModel -> # { @@ -203,7 +201,7 @@ class ReportQuery(BaseModel): # } # @TEST_FIXTURE: empty_collection -> {"items": [], "total": 0, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}} # @TEST_EDGE: negative_total -> {"items": [], "total": -5, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}} -# @RELATION: DEPENDS_ON -> ReportModels +# @RELATION DEPENDS_ON -> ReportModels class ReportCollection(BaseModel): items: List[TaskReport] total: int = Field(ge=0) @@ -217,8 +215,8 @@ class ReportCollection(BaseModel): # #region ReportDetailView [C:3] [TYPE Class] [SEMANTICS view, detail, logs] +# @INVARIANT: Incorporates a report and logs correctly. # @BRIEF Detailed report representation including diagnostics and recovery actions. -# @INVARIANT Incorporates a report and logs correctly. # # @TEST_CONTRACT: ReportDetailViewModel -> # { @@ -227,7 +225,7 @@ class ReportCollection(BaseModel): # } # @TEST_FIXTURE: valid_detail -> {"report": {"report_id": "rep-1", "task_id": "task-1", "task_type": "backup", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}} # @TEST_EDGE: missing_report -> {} -# @RELATION: DEPENDS_ON -> ReportModels +# @RELATION DEPENDS_ON -> ReportModels class ReportDetailView(BaseModel): report: TaskReport timeline: List[Dict[str, Any]] = Field(default_factory=list) diff --git a/backend/src/models/storage.py b/backend/src/models/storage.py index 5800b2c1..c272befe 100644 --- a/backend/src/models/storage.py +++ b/backend/src/models/storage.py @@ -34,4 +34,4 @@ class StoredFile(BaseModel): mime_type: Optional[str] = Field(None, description="MIME type of the file.") # #endregion StoredFile -# #endregion StorageModels +# #endregion StorageModels \ No newline at end of file diff --git a/backend/src/models/task.py b/backend/src/models/task.py index 22d9ad8d..8b03af7d 100644 --- a/backend/src/models/task.py +++ b/backend/src/models/task.py @@ -1,17 +1,15 @@ # #region TaskModels [C:1] [TYPE Module] [SEMANTICS database, task, record, sqlalchemy, sqlite] +# # @BRIEF Defines the database schema for task execution records. -# @LAYER Domain -# @INVARIANT All primary keys are UUID strings. -# @RELATION DEPENDS_ON -> [sqlalchemy] -# +# @LAYER: Domain +# @RELATION DEPENDS_ON -> sqlalchemy # +# @INVARIANT: All primary keys are UUID strings. -# [SECTION: IMPORTS] from sqlalchemy import Column, String, DateTime, JSON, ForeignKey, Text, Integer, Index from sqlalchemy.sql import func from .mapping import Base import uuid -# [/SECTION] # #region TaskRecord [C:1] [TYPE Class] # @BRIEF Represents a persistent record of a task execution. @@ -33,8 +31,8 @@ class TaskRecord(Base): # #region TaskLogRecord [C:3] [TYPE Class] # @BRIEF Represents a single persistent log entry for a task. -# @INVARIANT Each log entry belongs to exactly one task. -# @RELATION DEPENDS_ON -> [TaskRecord] +# @RELATION DEPENDS_ON -> TaskRecord +# @INVARIANT: Each log entry belongs to exactly one task. # # @TEST_CONTRACT: TaskLogCreate -> # { @@ -109,4 +107,4 @@ class TaskLogRecord(Base): ) # #endregion TaskLogRecord -# #endregion TaskModels +# #endregion TaskModels \ No newline at end of file diff --git a/backend/src/models/translate.py b/backend/src/models/translate.py index f71b96af..5bac0578 100644 --- a/backend/src/models/translate.py +++ b/backend/src/models/translate.py @@ -1,6 +1,7 @@ -# #region TranslateModels [C:2] [TYPE Module] +# #region TranslateModels [C:3] [TYPE Module] [SEMANTICS translate, models, sqlalchemy, persistence] # @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects. -# @LAYER Domain +# @LAYER: Domain +# @RELATION INHERITS_FROM -> MappingModels:Base from sqlalchemy import ( Column, String, Boolean, DateTime, JSON, Text, @@ -13,13 +14,14 @@ import hashlib from .mapping import Base -# #region generate_uuid [C:1] [TYPE Function] def generate_uuid(): return str(uuid.uuid4()) -# #endregion generate_uuid -# #region TranslationJob [C:1] [TYPE Class] +# #region TranslationJob [TYPE Class] +# @BRIEF A translation job representing a multi-dialect conversion task with column mappings, LLM config, and dictionary attachments. +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. class TranslationJob(Base): __tablename__ = "translation_jobs" @@ -51,7 +53,6 @@ class TranslationJob(Base): # Environment association environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access") - target_database_id = Column(String, nullable=True, comment="Superset database ID for INSERT via SQL Lab") created_by = Column(String, nullable=True) created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) @@ -59,7 +60,8 @@ class TranslationJob(Base): # #endregion TranslationJob -# #region TranslationRun [C:1] [TYPE Class] +# #region TranslationRun [TYPE Class] +# @BRIEF Represents a single execution of a translation job, capturing status, timing, and Superset execution metadata. class TranslationRun(Base): __tablename__ = "translation_runs" @@ -86,7 +88,8 @@ class TranslationRun(Base): # #endregion TranslationRun -# #region TranslationBatch [C:1] [TYPE Class] +# #region TranslationBatch [TYPE Class] +# @BRIEF Groups translation records within a run into manageable batches with timing and record counts. class TranslationBatch(Base): __tablename__ = "translation_batches" @@ -103,7 +106,8 @@ class TranslationBatch(Base): # #endregion TranslationBatch -# #region TranslationRecord [C:1] [TYPE Class] +# #region TranslationRecord [TYPE Class] +# @BRIEF Individual translation result for a single SQL statement or dashboard element, tracking source, target, and error state. class TranslationRecord(Base): __tablename__ = "translation_records" @@ -120,7 +124,6 @@ class TranslationRecord(Base): token_count_input = Column(Integer, nullable=True) token_count_output = Column(Integer, nullable=True) translation_duration_ms = Column(Integer, nullable=True) - source_data = Column(JSON, nullable=True, comment="Snapshot of source row key/context column values for INSERT phase") created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) __table_args__ = ( @@ -129,7 +132,8 @@ class TranslationRecord(Base): # #endregion TranslationRecord -# #region TranslationEvent [C:1] [TYPE Class] +# #region TranslationEvent [TYPE Class] +# @BRIEF Audit/event log for translation operations, with optional run_id for context. class TranslationEvent(Base): __tablename__ = "translation_events" @@ -143,7 +147,8 @@ class TranslationEvent(Base): # #endregion TranslationEvent -# #region TranslationPreviewSession [C:1] [TYPE Class] +# #region TranslationPreviewSession [TYPE Class] +# @BRIEF A preview session allowing users to review proposed translations before applying them. class TranslationPreviewSession(Base): __tablename__ = "translation_preview_sessions" @@ -157,7 +162,8 @@ class TranslationPreviewSession(Base): # #endregion TranslationPreviewSession -# #region TranslationPreviewRecord [C:1] [TYPE Class] +# #region TranslationPreviewRecord [TYPE Class] +# @BRIEF Individual preview entry within a preview session, showing original and translated content side by side. class TranslationPreviewRecord(Base): __tablename__ = "translation_preview_records" @@ -170,12 +176,12 @@ class TranslationPreviewRecord(Base): source_object_name = Column(String, nullable=True) status = Column(String, nullable=False, default="PENDING") # PENDING, APPROVED, REJECTED feedback = Column(Text, nullable=True) - source_data = Column(JSON, nullable=True, comment="Snapshot of source row values for key/context columns") created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) # #endregion TranslationPreviewRecord -# #region TerminologyDictionary [C:1] [TYPE Class] +# #region TerminologyDictionary [TYPE Class] +# @BRIEF A named collection of terminology mappings used during translation to ensure consistent term translation. class TerminologyDictionary(Base): __tablename__ = "terminology_dictionaries" @@ -191,7 +197,8 @@ class TerminologyDictionary(Base): # #endregion TerminologyDictionary -# #region DictionaryEntry [C:1] [TYPE Class] +# #region DictionaryEntry [TYPE Class] +# @BRIEF A single term mapping entry with case-insensitive unique constraint on source_term_normalized and origin tracking. class DictionaryEntry(Base): __tablename__ = "dictionary_entries" @@ -216,7 +223,8 @@ class DictionaryEntry(Base): # #endregion DictionaryEntry -# #region TranslationSchedule [C:1] [TYPE Class] +# #region TranslationSchedule [TYPE Class] +# @BRIEF Defines a cron-based schedule for recurring translation jobs. class TranslationSchedule(Base): __tablename__ = "translation_schedules" @@ -233,7 +241,8 @@ class TranslationSchedule(Base): # #endregion TranslationSchedule -# #region TranslationJobDictionary [C:1] [TYPE Class] +# #region TranslationJobDictionary [TYPE Class] +# @BRIEF Many-to-many association between translation jobs and terminology dictionaries. class TranslationJobDictionary(Base): __tablename__ = "translation_job_dictionaries" @@ -251,7 +260,8 @@ class TranslationJobDictionary(Base): # #endregion TranslationJobDictionary -# #region MetricSnapshot [C:1] [TYPE Class] +# #region MetricSnapshot [TYPE Class] +# @BRIEF Captures translation performance metrics at a point in time with config/content/dictionary hash keys for aggregation. class MetricSnapshot(Base): __tablename__ = "translation_metric_snapshots" diff --git a/backend/src/plugins/backup.py b/backend/src/plugins/backup.py index 1e196f99..7054ff8f 100755 --- a/backend/src/plugins/backup.py +++ b/backend/src/plugins/backup.py @@ -1,10 +1,10 @@ # #region BackupPlugin [TYPE Module] [SEMANTICS backup, superset, automation, dashboard, plugin] # @BRIEF A plugin that provides functionality to back up Superset dashboards. -# @LAYER App -# @RELATION IMPLEMENTS -> [PluginBase] -# @RELATION DEPENDS_ON -> [superset_tool.client] -# @RELATION DEPENDS_ON -> [superset_tool.utils] -# @RELATION USES -> [TaskContext] +# @LAYER: App +# @RELATION IMPLEMENTS -> PluginBase +# @RELATION DEPENDS_ON -> superset_tool.client +# @RELATION DEPENDS_ON -> superset_tool.utils +# @RELATION USES -> TaskContext from typing import Dict, Any, Optional from pathlib import Path @@ -33,63 +33,63 @@ class BackupPlugin(PluginBase): """ @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the backup plugin. - # @PRE Plugin instance exists. - # @POST Returns string ID. - # @RETURN str - "superset-backup" + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the backup plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string ID. + # @RETURN: str - "superset-backup" def id(self) -> str: with belief_scope("id"): return "superset-backup" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the backup plugin. - # @PRE Plugin instance exists. - # @POST Returns string name. - # @RETURN str - Plugin name. + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the backup plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string name. + # @RETURN: str - Plugin name. def name(self) -> str: with belief_scope("name"): return "Superset Dashboard Backup" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns a description of the backup plugin. - # @PRE Plugin instance exists. - # @POST Returns string description. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a description of the backup plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string description. + # @RETURN: str - Plugin description. def description(self) -> str: with belief_scope("description"): return "Backs up all dashboards from a Superset instance." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the version of the backup plugin. - # @PRE Plugin instance exists. - # @POST Returns string version. - # @RETURN str - "1.0.0" + # [DEF:version:Function] + # @PURPOSE: Returns the version of the backup plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string version. + # @RETURN: str - "1.0.0" def version(self) -> str: with belief_scope("version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the backup plugin. - # @RETURN str - "/tools/backups" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the backup plugin. + # @RETURN: str - "/tools/backups" def ui_route(self) -> str: with belief_scope("ui_route"): return "/tools/backups" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for backup plugin parameters. - # @PRE Plugin instance exists. - # @POST Returns dictionary schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for backup plugin parameters. + # @PRE: Plugin instance exists. + # @POST: Returns dictionary schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: with belief_scope("get_schema"): config_manager = get_config_manager() @@ -108,10 +108,10 @@ class BackupPlugin(PluginBase): }, "required": ["env"], } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Executes the dashboard backup logic with TaskContext support. + # [DEF:execute:Function] + # @PURPOSE: Executes the dashboard backup logic with TaskContext support. # @PARAM: params (Dict[str, Any]) - Backup parameters (env, backup_path, dashboard_ids). # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: Target environment must be configured. params must be a dictionary. @@ -255,6 +255,6 @@ class BackupPlugin(PluginBase): except (RequestException, IOError, KeyError) as e: log.error(f"Fatal error during backup for {env}: {e}") raise e - # #endregion execute + # [/DEF:execute:Function] # #endregion BackupPlugin # #endregion BackupPlugin diff --git a/backend/src/plugins/debug.py b/backend/src/plugins/debug.py index 3c19c045..6a6fabf4 100644 --- a/backend/src/plugins/debug.py +++ b/backend/src/plugins/debug.py @@ -1,16 +1,14 @@ # #region DebugPluginModule [TYPE Module] [SEMANTICS plugin, debug, api, database, superset] # @BRIEF Implements a plugin for system diagnostics and debugging Superset API responses. -# @LAYER Plugins +# @LAYER: Plugins # @RELATION Inherits from PluginBase. Uses SupersetClient from core. -# @RELATION USES -> [TaskContext] +# @RELATION USES -> TaskContext -# [SECTION: IMPORTS] from typing import Dict, Any, Optional from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient from ..core.logger import logger, belief_scope from ..core.task_manager.context import TaskContext -# [/SECTION] # #region DebugPlugin [TYPE Class] # @BRIEF Plugin for system diagnostics and debugging. @@ -20,63 +18,63 @@ class DebugPlugin(PluginBase): """ @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the debug plugin. - # @PRE Plugin instance exists. - # @POST Returns string ID. - # @RETURN str - "system-debug" + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the debug plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string ID. + # @RETURN: str - "system-debug" def id(self) -> str: with belief_scope("id"): return "system-debug" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the debug plugin. - # @PRE Plugin instance exists. - # @POST Returns string name. - # @RETURN str - Plugin name. + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the debug plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string name. + # @RETURN: str - Plugin name. def name(self) -> str: with belief_scope("name"): return "System Debug" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns a description of the debug plugin. - # @PRE Plugin instance exists. - # @POST Returns string description. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a description of the debug plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string description. + # @RETURN: str - Plugin description. def description(self) -> str: with belief_scope("description"): return "Run system diagnostics and debug Superset API responses." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the version of the debug plugin. - # @PRE Plugin instance exists. - # @POST Returns string version. - # @RETURN str - "1.0.0" + # [DEF:version:Function] + # @PURPOSE: Returns the version of the debug plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string version. + # @RETURN: str - "1.0.0" def version(self) -> str: with belief_scope("version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the debug plugin. - # @RETURN str - "/tools/debug" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the debug plugin. + # @RETURN: str - "/tools/debug" def ui_route(self) -> str: with belief_scope("ui_route"): return "/tools/debug" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for the debug plugin parameters. - # @PRE Plugin instance exists. - # @POST Returns dictionary schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for the debug plugin parameters. + # @PRE: Plugin instance exists. + # @POST: Returns dictionary schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: with belief_scope("get_schema"): return { @@ -111,10 +109,10 @@ class DebugPlugin(PluginBase): }, "required": ["action"] } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Executes the debug logic with TaskContext support. + # [DEF:execute:Function] + # @PURPOSE: Executes the debug logic with TaskContext support. # @PARAM: params (Dict[str, Any]) - Debug parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: action must be provided in params. @@ -138,12 +136,12 @@ class DebugPlugin(PluginBase): else: debug_log.error(f"Unknown action: {action}") raise ValueError(f"Unknown action: {action}") - # #endregion execute + # [/DEF:execute:Function] - # #region _test_db_api [TYPE Function] - # @BRIEF Tests database API connectivity for source and target environments. - # @PRE source_env and target_env params exist in params. - # @POST Returns DB counts for both envs. + # [DEF:_test_db_api:Function] + # @PURPOSE: Tests database API connectivity for source and target environments. + # @PRE: source_env and target_env params exist in params. + # @POST: Returns DB counts for both envs. # @PARAM: params (Dict) - Plugin parameters. # @PARAM: log - Logger instance for superset_api source. # @RETURN: Dict - Comparison results. @@ -176,12 +174,12 @@ class DebugPlugin(PluginBase): } return results - # #endregion _test_db_api + # [/DEF:_test_db_api:Function] - # #region _get_dataset_structure [TYPE Function] - # @BRIEF Retrieves the structure of a dataset. - # @PRE env and dataset_id params exist in params. - # @POST Returns dataset JSON structure. + # [DEF:_get_dataset_structure:Function] + # @PURPOSE: Retrieves the structure of a dataset. + # @PRE: env and dataset_id params exist in params. + # @POST: Returns dataset JSON structure. # @PARAM: params (Dict) - Plugin parameters. # @PARAM: log - Logger instance for superset_api source. # @RETURN: Dict - Dataset structure. @@ -208,7 +206,7 @@ class DebugPlugin(PluginBase): dataset_response = client.get_dataset(dataset_id) log.debug(f"Retrieved dataset structure for {dataset_id}") return dataset_response.get('result') or {} - # #endregion _get_dataset_structure + # [/DEF:_get_dataset_structure:Function] # #endregion DebugPlugin -# #endregion DebugPluginModule +# #endregion DebugPluginModule \ No newline at end of file diff --git a/backend/src/plugins/git/llm_extension.py b/backend/src/plugins/git/llm_extension.py index 3526a4f7..1c5d8303 100644 --- a/backend/src/plugins/git/llm_extension.py +++ b/backend/src/plugins/git/llm_extension.py @@ -1,6 +1,8 @@ -# #region backend/src/plugins/git/llm_extension [C:3] [TYPE Module] [SEMANTICS git, llm, commit] +# [DEF:backend/src/plugins/git/llm_extension:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: git, llm, commit # @BRIEF LLM-based extensions for the Git plugin, specifically for commit message generation. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [LLMClient] from typing import List @@ -15,8 +17,8 @@ class GitLLMExtension: def __init__(self, client: LLMClient): self.client = client - # #region suggest_commit_message [TYPE Function] - # @BRIEF Generates a suggested commit message based on a diff and history. + # [DEF:suggest_commit_message:Function] + # @PURPOSE: Generates a suggested commit message based on a diff and history. # @PARAM: diff (str) - The git diff of staged changes. # @PARAM: history (List[str]) - Recent commit messages for context. # @RETURN: str - The suggested commit message. @@ -56,7 +58,7 @@ class GitLLMExtension: return "Update dashboard configurations (LLM generation failed)" return response.choices[0].message.content.strip() - # #endregion suggest_commit_message + # [/DEF:suggest_commit_message:Function] # #endregion GitLLMExtension -# #endregion backend/src/plugins/git/llm_extension +# [/DEF:backend/src/plugins/git/llm_extension:Module] diff --git a/backend/src/plugins/git_plugin.py b/backend/src/plugins/git_plugin.py index c1dd9950..746993df 100644 --- a/backend/src/plugins/git_plugin.py +++ b/backend/src/plugins/git_plugin.py @@ -1,17 +1,16 @@ # #region GitPluginModule [TYPE Module] [SEMANTICS git, plugin, dashboard, version_control, sync, deploy] +# # @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset. -# @LAYER Plugin -# @INVARIANT Все операции с Git должны выполняться через GitService. -# @RELATION INHERITS_FROM -> [src.core.plugin_base.PluginBase] -# @RELATION USES -> [src.services.git_service.GitService] -# @RELATION USES -> [src.core.superset_client.SupersetClient] -# @RELATION USES -> [src.core.config_manager.ConfigManager] -# @RELATION USES -> [TaskContext] -# +# @LAYER: Plugin +# @RELATION INHERITS_FROM -> src.core.plugin_base.PluginBase +# @RELATION USES -> src.services.git_service.GitService +# @RELATION USES -> src.core.superset_client.SupersetClient +# @RELATION USES -> src.core.config_manager.ConfigManager +# @RELATION USES -> TaskContext # +# @INVARIANT: Все операции с Git должны выполняться через GitService. # @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset. -# [SECTION: IMPORTS] import os import io import shutil @@ -21,27 +20,23 @@ from typing import Dict, Any, Optional from src.core.plugin_base import PluginBase from src.services.git_service import GitService from src.core.logger import logger as app_logger, belief_scope -from src.core.cot_logger import MarkerLogger - -log = MarkerLogger("GitPlugin") from src.core.config_manager import ConfigManager from src.core.superset_client import SupersetClient from src.core.task_manager.context import TaskContext from src.core.database import SessionLocal from src.core.mapping_service import IdMappingService -# [/SECTION] # #region GitPlugin [TYPE Class] # @BRIEF Реализация плагина Git Integration для управления версиями дашбордов. class GitPlugin(PluginBase): - # #region __init__ [TYPE Function] - # @BRIEF Инициализирует плагин и его зависимости. - # @PRE config.json exists or shared config_manager is available. - # @POST Инициализированы git_service и config_manager. + # [DEF:__init__:Function] + # @PURPOSE: Инициализирует плагин и его зависимости. + # @PRE: config.json exists or shared config_manager is available. + # @POST: Инициализированы git_service и config_manager. def __init__(self): with belief_scope("GitPlugin.__init__"): - log.reason("Initializing GitPlugin.") + app_logger.info("Initializing GitPlugin.") self.git_service = GitService() # Robust config path resolution: @@ -56,69 +51,69 @@ class GitPlugin(PluginBase): try: from src.dependencies import config_manager self.config_manager = config_manager - log.reflect("GitPlugin initialized using shared config_manager.") + app_logger.info("GitPlugin initialized using shared config_manager.") return except Exception: config_path = "config.json" self.config_manager = ConfigManager(config_path) - log.reflect(f"GitPlugin initialized with {config_path}") - # #endregion __init__ + app_logger.info(f"GitPlugin initialized with {config_path}") + # [/DEF:__init__:Function] @property - # #region id [TYPE Function] - # @BRIEF Returns the plugin identifier. - # @PRE GitPlugin is initialized. - # @POST Returns 'git-integration'. + # [DEF:id:Function] + # @PURPOSE: Returns the plugin identifier. + # @PRE: GitPlugin is initialized. + # @POST: Returns 'git-integration'. def id(self) -> str: with belief_scope("GitPlugin.id"): return "git-integration" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the plugin name. - # @PRE GitPlugin is initialized. - # @POST Returns the human-readable name. + # [DEF:name:Function] + # @PURPOSE: Returns the plugin name. + # @PRE: GitPlugin is initialized. + # @POST: Returns the human-readable name. def name(self) -> str: with belief_scope("GitPlugin.name"): return "Git Integration" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns the plugin description. - # @PRE GitPlugin is initialized. - # @POST Returns the plugin's purpose description. + # [DEF:description:Function] + # @PURPOSE: Returns the plugin description. + # @PRE: GitPlugin is initialized. + # @POST: Returns the plugin's purpose description. def description(self) -> str: with belief_scope("GitPlugin.description"): return "Version control for Superset dashboards" - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the plugin version. - # @PRE GitPlugin is initialized. - # @POST Returns the version string. + # [DEF:version:Function] + # @PURPOSE: Returns the plugin version. + # @PRE: GitPlugin is initialized. + # @POST: Returns the version string. def version(self) -> str: with belief_scope("GitPlugin.version"): return "0.1.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the git plugin. - # @RETURN str - "/git" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the git plugin. + # @RETURN: str - "/git" def ui_route(self) -> str: with belief_scope("GitPlugin.ui_route"): return "/git" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Возвращает JSON-схему параметров для выполнения задач плагина. - # @PRE GitPlugin is initialized. - # @POST Returns a JSON schema dictionary. - # @RETURN Dict[str, Any] - Схема параметров. + # [DEF:get_schema:Function] + # @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина. + # @PRE: GitPlugin is initialized. + # @POST: Returns a JSON schema dictionary. + # @RETURN: Dict[str, Any] - Схема параметров. def get_schema(self) -> Dict[str, Any]: with belief_scope("GitPlugin.get_schema"): return { @@ -131,15 +126,15 @@ class GitPlugin(PluginBase): }, "required": ["operation", "dashboard_id"] } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region initialize [TYPE Function] - # @BRIEF Выполняет начальную настройку плагина. - # @PRE GitPlugin is initialized. - # @POST Плагин готов к выполнению задач. + # [DEF:initialize:Function] + # @PURPOSE: Выполняет начальную настройку плагина. + # @PRE: GitPlugin is initialized. + # @POST: Плагин готов к выполнению задач. async def initialize(self): with belief_scope("GitPlugin.initialize"): - log.reason("Initializing Git Integration Plugin logic") + app_logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.") # [DEF:execute:Function] # @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext. @@ -250,15 +245,15 @@ class GitPlugin(PluginBase): # 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff) try: repo.git.add(A=True) - log.reason("Changes staged in git") + app_logger.info("[_handle_sync][Action] Changes staged in git") except Exception as ge: - log.explore("Failed to stage changes", error=str(ge)) + app_logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}") - log.reflect("Dashboard synced successfully", payload={"dashboard_id": dashboard_id}) + app_logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.") return {"status": "success", "message": "Dashboard synced and flattened in local repository"} except Exception as e: - log.explore("Sync failed", error=str(e)) + app_logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}") raise # [/DEF:_handle_sync:Function] @@ -320,16 +315,16 @@ class GitPlugin(PluginBase): f.write(zip_buffer.getvalue()) try: - log.reason("Importing dashboard", payload={"environment": env.name}) + app_logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}") result = client.import_dashboard(temp_zip_path) - log.reflect("Deployment successful", payload={"dashboard_id": dashboard_id}) + app_logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.") return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result} finally: if temp_zip_path.exists(): os.remove(temp_zip_path) except Exception as e: - log.explore("Deployment failed", error=str(e)) + app_logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}") raise # [/DEF:_handle_deploy:Function] @@ -341,13 +336,13 @@ class GitPlugin(PluginBase): # @RETURN: Environment - Объект конфигурации окружения. def _get_env(self, env_id: Optional[str] = None): with belief_scope("GitPlugin._get_env"): - log.reason("Fetching environment", payload={"env_id": env_id}) + app_logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}") # Priority 1: ConfigManager (config.json) if env_id: env = self.config_manager.get_environment(env_id) if env: - log.reflect("Found environment by ID in ConfigManager", payload={"name": env.name}) + app_logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}") return env # Priority 2: Database (DeploymentEnvironment) @@ -365,7 +360,7 @@ class GitPlugin(PluginBase): db_env = db.query(DeploymentEnvironment).first() if db_env: - log.reflect("Found environment in DB", payload={"name": db_env.name}) + app_logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}") from src.core.config_models import Environment # Use token as password for SupersetClient return Environment( @@ -387,17 +382,17 @@ class GitPlugin(PluginBase): # but we have other envs, maybe it's one of them? env = next((e for e in envs if e.id == env_id), None) if env: - log.reflect("Found environment in ConfigManager list", payload={"env_id": env_id}) + app_logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list") return env if not env_id: - log.reflect("Using first environment from ConfigManager", payload={"name": envs[0].name}) + app_logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}") return envs[0] - log.explore("No environments configured", payload={"env_id": env_id}, error="No Superset environments found in config or database") + app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}") raise ValueError("No environments configured. Please add a Superset Environment in Settings.") # [/DEF:_get_env:Function] - # #endregion initialize + # [/DEF:initialize:Function] # #endregion GitPlugin -# #endregion GitPluginModule +# #endregion GitPluginModule \ No newline at end of file diff --git a/backend/src/plugins/llm_analysis/__init__.py b/backend/src/plugins/llm_analysis/__init__.py index 91a5efa3..4064146c 100644 --- a/backend/src/plugins/llm_analysis/__init__.py +++ b/backend/src/plugins/llm_analysis/__init__.py @@ -1,4 +1,4 @@ -# #region backend/src/plugins/llm_analysis/__init__.py [TYPE Module] +# [DEF:backend/src/plugins/llm_analysis/__init__.py:Module] """ LLM Analysis Plugin for automated dashboard validation and dataset documentation. @@ -8,4 +8,4 @@ from .plugin import DashboardValidationPlugin, DocumentationPlugin __all__ = ['DashboardValidationPlugin', 'DocumentationPlugin'] -# #endregion backend/src/plugins/llm_analysis/__init__.py +# [/DEF:backend/src/plugins/llm_analysis/__init__.py:Module] diff --git a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py index d53c958e..6e38c85c 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_client_headers.py @@ -1,16 +1,18 @@ -# #region TestClientHeaders [C:3] [TYPE Module] [SEMANTICS tests, llm-client, openrouter, headers] -# @BRIEF Verify OpenRouter client initialization includes provider-specific headers. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestClientHeaders:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, llm-client, openrouter, headers +# @PURPOSE: Verify OpenRouter client initialization includes provider-specific headers. from src.plugins.llm_analysis.models import LLMProviderType from src.plugins.llm_analysis.service import LLMClient -# #region test_openrouter_client_includes_referer_and_title_headers [TYPE Function] -# @BRIEF OpenRouter requests should carry site/app attribution headers for compatibility. -# @PRE Client is initialized for OPENROUTER provider. -# @POST Async client headers include Authorization, HTTP-Referer, and X-Title. -# @RELATION BINDS_TO -> [TestClientHeaders] +# [DEF:test_openrouter_client_includes_referer_and_title_headers:Function] +# @RELATION: BINDS_TO -> TestClientHeaders +# @PURPOSE: OpenRouter requests should carry site/app attribution headers for compatibility. +# @PRE: Client is initialized for OPENROUTER provider. +# @POST: Async client headers include Authorization, HTTP-Referer, and X-Title. def test_openrouter_client_includes_referer_and_title_headers(monkeypatch): monkeypatch.setenv("OPENROUTER_SITE_URL", "http://localhost:8000") monkeypatch.setenv("OPENROUTER_APP_NAME", "ss-tools-test") @@ -26,5 +28,5 @@ def test_openrouter_client_includes_referer_and_title_headers(monkeypatch): assert headers["Authorization"] == "Bearer sk-test-provider-key-123456" assert headers["HTTP-Referer"] == "http://localhost:8000" assert headers["X-Title"] == "ss-tools-test" -# #endregion test_openrouter_client_includes_referer_and_title_headers -# #endregion TestClientHeaders +# [/DEF:test_openrouter_client_includes_referer_and_title_headers:Function] +# [/DEF:TestClientHeaders:Module] diff --git a/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py b/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py index 3369162c..93c48373 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_screenshot_service.py @@ -1,17 +1,19 @@ -# #region TestScreenshotService [C:3] [TYPE Module] [SEMANTICS tests, screenshot-service, navigation, timeout-regression] -# @BRIEF Protect dashboard screenshot navigation from brittle networkidle waits. -# @RELATION VERIFIES -> [src.plugins.llm_analysis.service.ScreenshotService] +# [DEF:TestScreenshotService:Module] +# @RELATION: VERIFIES ->[src.plugins.llm_analysis.service.ScreenshotService] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, screenshot-service, navigation, timeout-regression +# @PURPOSE: Protect dashboard screenshot navigation from brittle networkidle waits. import pytest from src.plugins.llm_analysis.service import ScreenshotService -# #region test_iter_login_roots_includes_child_frames [TYPE Function] -# @BRIEF Login discovery must search embedded auth frames, not only the main page. -# @PRE Page exposes child frames list. -# @POST Returned roots include page plus child frames in order. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_iter_login_roots_includes_child_frames:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Login discovery must search embedded auth frames, not only the main page. +# @PRE: Page exposes child frames list. +# @POST: Returned roots include page plus child frames in order. def test_iter_login_roots_includes_child_frames(): frame_a = object() frame_b = object() @@ -23,14 +25,14 @@ def test_iter_login_roots_includes_child_frames(): assert roots == [fake_page, frame_a, frame_b] -# #endregion test_iter_login_roots_includes_child_frames +# [/DEF:test_iter_login_roots_includes_child_frames:Function] -# #region test_response_looks_like_login_page_detects_login_markup [TYPE Function] -# @BRIEF Direct login fallback must reject responses that render the login screen again. -# @PRE Response body contains stable login-page markers. -# @POST Helper returns True so caller treats fallback as failed authentication. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_response_looks_like_login_page_detects_login_markup:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Direct login fallback must reject responses that render the login screen again. +# @PRE: Response body contains stable login-page markers. +# @POST: Helper returns True so caller treats fallback as failed authentication. def test_response_looks_like_login_page_detects_login_markup(): service = ScreenshotService(env=type("Env", (), {})()) @@ -50,14 +52,14 @@ def test_response_looks_like_login_page_detects_login_markup(): assert result is True -# #endregion test_response_looks_like_login_page_detects_login_markup +# [/DEF:test_response_looks_like_login_page_detects_login_markup:Function] -# #region test_find_first_visible_locator_skips_hidden_first_match [TYPE Function] -# @BRIEF Locator helper must not reject a selector collection just because its first element is hidden. -# @PRE First matched element is hidden and second matched element is visible. -# @POST Helper returns the second visible candidate. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_find_first_visible_locator_skips_hidden_first_match:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Locator helper must not reject a selector collection just because its first element is hidden. +# @PRE: First matched element is hidden and second matched element is visible. +# @POST: Helper returns the second visible candidate. @pytest.mark.anyio async def test_find_first_visible_locator_skips_hidden_first_match(): class _FakeElement: @@ -91,14 +93,14 @@ async def test_find_first_visible_locator_skips_hidden_first_match(): assert result.label == "visible" -# #endregion test_find_first_visible_locator_skips_hidden_first_match +# [/DEF:test_find_first_visible_locator_skips_hidden_first_match:Function] -# #region test_submit_login_via_form_post_uses_browser_context_request [TYPE Function] -# @BRIEF Fallback login must submit hidden fields and credentials through the context request cookie jar. -# @PRE Login DOM exposes csrf hidden field and request context returns authenticated HTML. -# @POST Helper returns True and request payload contains csrf_token plus credentials plus request options. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_submit_login_via_form_post_uses_browser_context_request:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Fallback login must submit hidden fields and credentials through the context request cookie jar. +# @PRE: Login DOM exposes csrf hidden field and request context returns authenticated HTML. +# @POST: Helper returns True and request payload contains csrf_token plus credentials plus request options. @pytest.mark.anyio async def test_submit_login_via_form_post_uses_browser_context_request(): class _FakeInput: @@ -202,14 +204,14 @@ async def test_submit_login_via_form_post_uses_browser_context_request(): ] -# #endregion test_submit_login_via_form_post_uses_browser_context_request +# [/DEF:test_submit_login_via_form_post_uses_browser_context_request:Function] -# #region test_submit_login_via_form_post_accepts_authenticated_redirect [TYPE Function] -# @BRIEF Fallback login must treat non-login 302 redirect as success without waiting for redirect target. -# @PRE Request response is 302 with Location outside login path. -# @POST Helper returns True. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_submit_login_via_form_post_accepts_authenticated_redirect:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Fallback login must treat non-login 302 redirect as success without waiting for redirect target. +# @PRE: Request response is 302 with Location outside login path. +# @POST: Helper returns True. @pytest.mark.anyio async def test_submit_login_via_form_post_accepts_authenticated_redirect(): class _FakeInput: @@ -277,14 +279,14 @@ async def test_submit_login_via_form_post_accepts_authenticated_redirect(): assert result is True -# #endregion test_submit_login_via_form_post_accepts_authenticated_redirect +# [/DEF:test_submit_login_via_form_post_accepts_authenticated_redirect:Function] -# #region test_submit_login_via_form_post_rejects_login_markup_response [TYPE Function] -# @BRIEF Fallback login must fail when POST response still contains login form content. -# @PRE Login DOM exposes csrf hidden field and request response renders login markup. -# @POST Helper returns False. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_submit_login_via_form_post_rejects_login_markup_response:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Fallback login must fail when POST response still contains login form content. +# @PRE: Login DOM exposes csrf hidden field and request response renders login markup. +# @POST: Helper returns False. @pytest.mark.anyio async def test_submit_login_via_form_post_rejects_login_markup_response(): class _FakeInput: @@ -360,14 +362,14 @@ async def test_submit_login_via_form_post_rejects_login_markup_response(): assert result is False -# #endregion test_submit_login_via_form_post_rejects_login_markup_response +# [/DEF:test_submit_login_via_form_post_rejects_login_markup_response:Function] -# #region test_goto_resilient_falls_back_from_domcontentloaded_to_load [TYPE Function] -# @BRIEF Pages with unstable primary wait must retry with fallback wait strategy. -# @PRE First page.goto call raises; second succeeds. -# @POST Helper returns second response and attempts both wait modes in order. -# @RELATION BINDS_TO -> [TestScreenshotService] +# [DEF:test_goto_resilient_falls_back_from_domcontentloaded_to_load:Function] +# @RELATION: BINDS_TO ->[TestScreenshotService] +# @PURPOSE: Pages with unstable primary wait must retry with fallback wait strategy. +# @PRE: First page.goto call raises; second succeeds. +# @POST: Helper returns second response and attempts both wait modes in order. @pytest.mark.anyio async def test_goto_resilient_falls_back_from_domcontentloaded_to_load(): class _FakePage: @@ -398,5 +400,5 @@ async def test_goto_resilient_falls_back_from_domcontentloaded_to_load(): ] -# #endregion test_goto_resilient_falls_back_from_domcontentloaded_to_load -# #endregion TestScreenshotService +# [/DEF:test_goto_resilient_falls_back_from_domcontentloaded_to_load:Function] +# [/DEF:TestScreenshotService:Module] diff --git a/backend/src/plugins/llm_analysis/__tests__/test_service.py b/backend/src/plugins/llm_analysis/__tests__/test_service.py index 0a032163..c975b5cd 100644 --- a/backend/src/plugins/llm_analysis/__tests__/test_service.py +++ b/backend/src/plugins/llm_analysis/__tests__/test_service.py @@ -1,6 +1,8 @@ -# #region TestService [C:3] [TYPE Module] [SEMANTICS tests, llm-analysis, fallback, provider-error, unknown-status] -# @BRIEF Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestService:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, llm-analysis, fallback, provider-error, unknown-status +# @PURPOSE: Verify LLM analysis transport/provider failures do not masquerade as dashboard FAIL results. import pytest @@ -8,11 +10,11 @@ from src.plugins.llm_analysis.models import LLMProviderType from src.plugins.llm_analysis.service import LLMClient -# #region test_test_runtime_connection_uses_json_completion_transport [TYPE Function] -# @BRIEF Provider self-test must exercise the same chat completion transport as runtime analysis. -# @PRE get_json_completion is available on initialized client. -# @POST Self-test forwards a lightweight user message into get_json_completion and returns its payload. -# @RELATION BINDS_TO -> [TestService] +# [DEF:test_test_runtime_connection_uses_json_completion_transport:Function] +# @RELATION: BINDS_TO -> TestService +# @PURPOSE: Provider self-test must exercise the same chat completion transport as runtime analysis. +# @PRE: get_json_completion is available on initialized client. +# @POST: Self-test forwards a lightweight user message into get_json_completion and returns its payload. @pytest.mark.anyio async def test_test_runtime_connection_uses_json_completion_transport(monkeypatch): client = LLMClient( @@ -34,14 +36,14 @@ async def test_test_runtime_connection_uses_json_completion_transport(monkeypatc assert result == {"ok": True} assert recorded["messages"][0]["role"] == "user" assert "Return exactly this JSON object" in recorded["messages"][0]["content"] -# #endregion test_test_runtime_connection_uses_json_completion_transport +# [/DEF:test_test_runtime_connection_uses_json_completion_transport:Function] -# #region test_analyze_dashboard_provider_error_maps_to_unknown [TYPE Function] -# @BRIEF Infrastructure/provider failures must produce UNKNOWN analysis status rather than FAIL. -# @PRE LLMClient.get_json_completion raises provider/auth exception. -# @POST Returned payload uses status=UNKNOWN and issue severity UNKNOWN. -# @RELATION BINDS_TO -> [TestService] +# [DEF:test_analyze_dashboard_provider_error_maps_to_unknown:Function] +# @RELATION: BINDS_TO -> TestService +# @PURPOSE: Infrastructure/provider failures must produce UNKNOWN analysis status rather than FAIL. +# @PRE: LLMClient.get_json_completion raises provider/auth exception. +# @POST: Returned payload uses status=UNKNOWN and issue severity UNKNOWN. @pytest.mark.anyio async def test_analyze_dashboard_provider_error_maps_to_unknown(monkeypatch, tmp_path): screenshot_path = tmp_path / "shot.jpg" @@ -64,5 +66,5 @@ async def test_analyze_dashboard_provider_error_maps_to_unknown(monkeypatch, tmp assert result["status"] == "UNKNOWN" assert "Failed to get response from LLM" in result["summary"] assert result["issues"][0]["severity"] == "UNKNOWN" -# #endregion test_analyze_dashboard_provider_error_maps_to_unknown -# #endregion TestService +# [/DEF:test_analyze_dashboard_provider_error_maps_to_unknown:Function] +# [/DEF:TestService:Module] diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 9b638e24..72598481 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -1,9 +1,9 @@ # #region LLMAnalysisModels [C:3] [TYPE Module] [SEMANTICS pydantic, models, llm] # @BRIEF Define Pydantic models for LLM Analysis plugin. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [pydantic] -# @RELATION DEPENDS_on -> [pydantic] -# @RELATION DEPENDs_on -> [pydantic] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> pydantic +# @RELATION DEPENDS_on -> pydantic +# @RELATION DEPENDs_on -> pydantic from typing import List, Optional from pydantic import BaseModel, Field @@ -18,22 +18,6 @@ class LLMProviderType(str, Enum): KILO = "kilo" # #endregion LLMProviderType -# #region FetchModelsRequest [TYPE Class] -# @BRIEF Request model to fetch available models from a provider. -class FetchModelsRequest(BaseModel): - base_url: str = Field(description="Provider base URL (e.g. https://api.openai.com/v1)") - api_key: Optional[str] = Field(None, description="Optional API key for authenticated providers") - provider_type: LLMProviderType = Field(LLMProviderType.OPENAI, description="Provider type for auth headers") - provider_id: Optional[str] = Field(None, description="Existing provider ID — resolves api_key from DB if not provided directly") -# #endregion FetchModelsRequest - -# #region FetchModelsResponse [TYPE Class] -# @BRIEF Response with available model IDs from a provider. -class FetchModelsResponse(BaseModel): - models: List[str] = Field(description="List of available model IDs") - total: int = Field(description="Total number of models found") -# #endregion FetchModelsResponse - # #region LLMProviderConfig [TYPE Class] # @BRIEF Configuration for an LLM provider. class LLMProviderConfig(BaseModel): diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index af502b8a..dea59043 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -1,12 +1,14 @@ -# #region backend/src/plugins/llm_analysis/plugin.py [C:3] [TYPE Module] [SEMANTICS plugin, llm, analysis, documentation] +# [DEF:backend/src/plugins/llm_analysis/plugin.py:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: plugin, llm, analysis, documentation # @BRIEF Implements DashboardValidationPlugin and DocumentationPlugin. -# @LAYER Domain -# @INVARIANT All LLM interactions must be executed as asynchronous tasks. +# @LAYER: Domain # @RELATION INHERITS -> [PluginBase] # @RELATION CALLS -> [ScreenshotService] # @RELATION CALLS -> [LLMClient] # @RELATION CALLS -> [LLMProviderService] -# @RELATION USES -> [TaskContext] +# @RELATION USES -> TaskContext +# @INVARIANT: All LLM interactions must be executed as asynchronous tasks. from typing import Dict, Any, Optional import os @@ -31,8 +33,8 @@ from ...services.llm_prompt_templates import ( # #region _is_masked_or_invalid_api_key [TYPE Function] # @BRIEF Guards against placeholder or malformed API keys in runtime. -# @PRE value may be None. -# @POST Returns True when value cannot be used for authenticated provider calls. +# @PRE: value may be None. +# @POST: Returns True when value cannot be used for authenticated provider calls. def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool: key = (value or "").strip() if not key: @@ -45,8 +47,8 @@ def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool: # #region _json_safe_value [TYPE Function] # @BRIEF Recursively normalize payload values for JSON serialization. -# @PRE value may be nested dict/list with datetime values. -# @POST datetime values are converted to ISO strings. +# @PRE: value may be nested dict/list with datetime values. +# @POST: datetime values are converted to ISO strings. def _json_safe_value(value: Any): if isinstance(value, datetime): return value.isoformat() @@ -59,7 +61,7 @@ def _json_safe_value(value: Any): # #region DashboardValidationPlugin [TYPE Class] # @BRIEF Plugin for automated dashboard health analysis using LLMs. -# @RELATION IMPLEMENTS -> [backend.src.core.plugin_base.PluginBase] +# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase class DashboardValidationPlugin(PluginBase): @property def id(self) -> str: @@ -88,8 +90,8 @@ class DashboardValidationPlugin(PluginBase): "required": ["dashboard_id", "environment_id", "provider_id"] } - # #region DashboardValidationPlugin.execute [TYPE Function] - # @BRIEF Executes the dashboard validation task with TaskContext support. + # [DEF:DashboardValidationPlugin.execute:Function] + # @PURPOSE: Executes the dashboard validation task with TaskContext support. # @PARAM: params (Dict[str, Any]) - Validation parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: params contains dashboard_id, environment_id, and provider_id. @@ -317,12 +319,12 @@ class DashboardValidationPlugin(PluginBase): finally: db.close() - # #endregion DashboardValidationPlugin.execute + # [/DEF:DashboardValidationPlugin.execute:Function] # #endregion DashboardValidationPlugin # #region DocumentationPlugin [TYPE Class] # @BRIEF Plugin for automated dataset documentation using LLMs. -# @RELATION IMPLEMENTS -> [backend.src.core.plugin_base.PluginBase] +# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase class DocumentationPlugin(PluginBase): @property def id(self) -> str: @@ -351,8 +353,8 @@ class DocumentationPlugin(PluginBase): "required": ["dataset_id", "environment_id", "provider_id"] } - # #region DocumentationPlugin.execute [TYPE Function] - # @BRIEF Executes the dataset documentation task with TaskContext support. + # [DEF:DocumentationPlugin.execute:Function] + # @PURPOSE: Executes the dataset documentation task with TaskContext support. # @PARAM: params (Dict[str, Any]) - Documentation parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: params contains dataset_id, environment_id, and provider_id. @@ -473,7 +475,7 @@ class DocumentationPlugin(PluginBase): finally: db.close() - # #endregion DocumentationPlugin.execute + # [/DEF:DocumentationPlugin.execute:Function] # #endregion DocumentationPlugin -# #endregion backend/src/plugins/llm_analysis/plugin.py +# [/DEF:backend/src/plugins/llm_analysis/plugin.py:Module] diff --git a/backend/src/plugins/llm_analysis/scheduler.py b/backend/src/plugins/llm_analysis/scheduler.py index 382cb271..69ac2af6 100644 --- a/backend/src/plugins/llm_analysis/scheduler.py +++ b/backend/src/plugins/llm_analysis/scheduler.py @@ -1,6 +1,8 @@ -# #region backend/src/plugins/llm_analysis/scheduler.py [C:3] [TYPE Module] [SEMANTICS scheduler, task, automation] +# [DEF:backend/src/plugins/llm_analysis/scheduler.py:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: scheduler, task, automation # @BRIEF Provides helper functions to schedule LLM-based validation tasks. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [SchedulerService] from typing import Dict, Any @@ -9,9 +11,6 @@ from ...core.logger import belief_scope, logger # #region schedule_dashboard_validation [TYPE Function] # @BRIEF Schedules a recurring dashboard validation task. -# @PARAM: dashboard_id (str) - ID of the dashboard to validate. -# @PARAM: cron_expression (str) - Standard cron expression for scheduling. -# @PARAM: params (Dict[str, Any]) - Task parameters (environment_id, provider_id). # @SIDE_EFFECT: Adds a job to the scheduler service. def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, params: Dict[str, Any]): with belief_scope("schedule_dashboard_validation", f"dashboard_id={dashboard_id}"): @@ -41,8 +40,6 @@ def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, param # #region _parse_cron [TYPE Function] # @BRIEF Basic cron parser placeholder. -# @PARAM: cron (str) - Cron expression. -# @RETURN: Dict[str, str] - Parsed cron parts. def _parse_cron(cron: str) -> Dict[str, str]: # Basic cron parser placeholder parts = cron.split() @@ -57,4 +54,4 @@ def _parse_cron(cron: str) -> Dict[str, str]: } # #endregion _parse_cron -# #endregion backend/src/plugins/llm_analysis/scheduler.py +# [/DEF:backend/src/plugins/llm_analysis/scheduler.py:Module] diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index 738a9178..15b1f105 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -1,10 +1,12 @@ -# #region backend/src/plugins/llm_analysis/service.py [C:3] [TYPE Module] [SEMANTICS service, llm, screenshot, playwright, openai] +# [DEF:backend/src/plugins/llm_analysis/service.py:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: service, llm, screenshot, playwright, openai # @BRIEF Services for LLM interaction and dashboard screenshots. -# @LAYER Domain -# @INVARIANT Screenshots must be 1920px width and capture full page height. -# @RELATION DEPENDS_ON -> [playwright] -# @RELATION DEPENDS_ON -> [openai] -# @RELATION DEPENDS_ON -> [tenacity] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> playwright +# @RELATION DEPENDS_ON -> openai +# @RELATION DEPENDS_ON -> tenacity +# @INVARIANT: Screenshots must be 1920px width and capture full page height. import asyncio import base64 @@ -26,17 +28,17 @@ from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt # #region ScreenshotService [TYPE Class] # @BRIEF Handles capturing screenshots of Superset dashboards. class ScreenshotService: - # #region ScreenshotService.__init__ [TYPE Function] - # @BRIEF Initializes the ScreenshotService with environment configuration. - # @PRE env is a valid Environment object. + # [DEF:ScreenshotService.__init__:Function] + # @PURPOSE: Initializes the ScreenshotService with environment configuration. + # @PRE: env is a valid Environment object. def __init__(self, env: Environment): self.env = env - # #endregion ScreenshotService.__init__ + # [/DEF:ScreenshotService.__init__:Function] - # #region ScreenshotService._find_first_visible_locator [TYPE Function] - # @BRIEF Resolve the first visible locator from multiple Playwright locator strategies. - # @PRE candidates is a non-empty list of locator-like objects. - # @POST Returns a locator ready for interaction or None when nothing matches. + # [DEF:ScreenshotService._find_first_visible_locator:Function] + # @PURPOSE: Resolve the first visible locator from multiple Playwright locator strategies. + # @PRE: candidates is a non-empty list of locator-like objects. + # @POST: Returns a locator ready for interaction or None when nothing matches. async def _find_first_visible_locator(self, candidates) -> Any: for locator in candidates: try: @@ -48,12 +50,12 @@ class ScreenshotService: except Exception: continue return None - # #endregion ScreenshotService._find_first_visible_locator + # [/DEF:ScreenshotService._find_first_visible_locator:Function] - # #region ScreenshotService._iter_login_roots [TYPE Function] - # @BRIEF Enumerate page and child frames where login controls may be rendered. - # @PRE page is a Playwright page-like object. - # @POST Returns ordered roots starting with main page followed by frames. + # [DEF:ScreenshotService._iter_login_roots:Function] + # @PURPOSE: Enumerate page and child frames where login controls may be rendered. + # @PRE: page is a Playwright page-like object. + # @POST: Returns ordered roots starting with main page followed by frames. def _iter_login_roots(self, page) -> List[Any]: roots = [page] page_frames = getattr(page, "frames", []) @@ -64,12 +66,12 @@ class ScreenshotService: except Exception: pass return roots - # #endregion ScreenshotService._iter_login_roots + # [/DEF:ScreenshotService._iter_login_roots:Function] - # #region ScreenshotService._extract_hidden_login_fields [TYPE Function] - # @BRIEF Collect hidden form fields required for direct login POST fallback. - # @PRE Login page is loaded. - # @POST Returns hidden input name/value mapping aggregated from page and child frames. + # [DEF:ScreenshotService._extract_hidden_login_fields:Function] + # @PURPOSE: Collect hidden form fields required for direct login POST fallback. + # @PRE: Login page is loaded. + # @POST: Returns hidden input name/value mapping aggregated from page and child frames. async def _extract_hidden_login_fields(self, page) -> Dict[str, str]: hidden_fields: Dict[str, str] = {} for root in self._iter_login_roots(page): @@ -85,21 +87,21 @@ class ScreenshotService: except Exception: continue return hidden_fields - # #endregion ScreenshotService._extract_hidden_login_fields + # [/DEF:ScreenshotService._extract_hidden_login_fields:Function] - # #region ScreenshotService._extract_csrf_token [TYPE Function] - # @BRIEF Resolve CSRF token value from main page or embedded login frame. - # @PRE Login page is loaded. - # @POST Returns first non-empty csrf token or empty string. + # [DEF:ScreenshotService._extract_csrf_token:Function] + # @PURPOSE: Resolve CSRF token value from main page or embedded login frame. + # @PRE: Login page is loaded. + # @POST: Returns first non-empty csrf token or empty string. async def _extract_csrf_token(self, page) -> str: hidden_fields = await self._extract_hidden_login_fields(page) return str(hidden_fields.get("csrf_token") or "").strip() - # #endregion ScreenshotService._extract_csrf_token + # [/DEF:ScreenshotService._extract_csrf_token:Function] - # #region ScreenshotService._response_looks_like_login_page [TYPE Function] - # @BRIEF Detect when fallback login POST returned the login form again instead of an authenticated page. - # @PRE response_text is normalized HTML or text from login POST response. - # @POST Returns True when login-page markers dominate the response body. + # [DEF:ScreenshotService._response_looks_like_login_page:Function] + # @PURPOSE: Detect when fallback login POST returned the login form again instead of an authenticated page. + # @PRE: response_text is normalized HTML or text from login POST response. + # @POST: Returns True when login-page markers dominate the response body. def _response_looks_like_login_page(self, response_text: str) -> bool: normalized = str(response_text or "").strip().lower() if not normalized: @@ -113,23 +115,23 @@ class ScreenshotService: 'name="csrf_token"', ] return sum(marker in normalized for marker in markers) >= 3 - # #endregion ScreenshotService._response_looks_like_login_page + # [/DEF:ScreenshotService._response_looks_like_login_page:Function] - # #region ScreenshotService._redirect_looks_authenticated [TYPE Function] - # @BRIEF Treat non-login redirects after form POST as successful authentication without waiting for redirect target. - # @PRE redirect_location may be empty or relative. - # @POST Returns True when redirect target does not point back to login flow. + # [DEF:ScreenshotService._redirect_looks_authenticated:Function] + # @PURPOSE: Treat non-login redirects after form POST as successful authentication without waiting for redirect target. + # @PRE: redirect_location may be empty or relative. + # @POST: Returns True when redirect target does not point back to login flow. def _redirect_looks_authenticated(self, redirect_location: str) -> bool: normalized = str(redirect_location or "").strip().lower() if not normalized: return True return "/login" not in normalized - # #endregion ScreenshotService._redirect_looks_authenticated + # [/DEF:ScreenshotService._redirect_looks_authenticated:Function] - # #region ScreenshotService._submit_login_via_form_post [TYPE Function] - # @BRIEF Fallback login path that submits credentials directly with csrf token. - # @PRE login_url is same-origin and csrf token can be read from DOM. - # @POST Browser context receives authenticated cookies when login succeeds. + # [DEF:ScreenshotService._submit_login_via_form_post:Function] + # @PURPOSE: Fallback login path that submits credentials directly with csrf token. + # @PRE: login_url is same-origin and csrf token can be read from DOM. + # @POST: Browser context receives authenticated cookies when login succeeds. async def _submit_login_via_form_post(self, page, login_url: str) -> bool: hidden_fields = await self._extract_hidden_login_fields(page) csrf_token = str(hidden_fields.get("csrf_token") or "").strip() @@ -186,12 +188,12 @@ class ScreenshotService: f"[DEBUG] Direct form login fallback response: status={response_status} url={response_url} login_markup={looks_like_login_page} snippet={text_snippet!r}" ) return not looks_like_login_page - # #endregion ScreenshotService._submit_login_via_form_post + # [/DEF:ScreenshotService._submit_login_via_form_post:Function] - # #region ScreenshotService._find_login_field_locator [TYPE Function] - # @BRIEF Resolve login form input using semantic label text plus generic visible-input fallbacks. - # @PRE field_name is `username` or `password`. - # @POST Returns a locator for the corresponding input or None. + # [DEF:ScreenshotService._find_login_field_locator:Function] + # @PURPOSE: Resolve login form input using semantic label text plus generic visible-input fallbacks. + # @PRE: field_name is `username` or `password`. + # @POST: Returns a locator for the corresponding input or None. async def _find_login_field_locator(self, page, field_name: str) -> Any: normalized = str(field_name or "").strip().lower() for root in self._iter_login_roots(page): @@ -226,12 +228,12 @@ class ScreenshotService: return locator return None - # #endregion ScreenshotService._find_login_field_locator + # [/DEF:ScreenshotService._find_login_field_locator:Function] - # #region ScreenshotService._find_submit_locator [TYPE Function] - # @BRIEF Resolve login submit button from main page or embedded auth frame. - # @PRE page is ready for login interaction. - # @POST Returns visible submit locator or None. + # [DEF:ScreenshotService._find_submit_locator:Function] + # @PURPOSE: Resolve login submit button from main page or embedded auth frame. + # @PRE: page is ready for login interaction. + # @POST: Returns visible submit locator or None. async def _find_submit_locator(self, page) -> Any: selectors = [ lambda root: root.get_by_role("button", name="Sign in", exact=False), @@ -246,12 +248,12 @@ class ScreenshotService: if locator: return locator return None - # #endregion ScreenshotService._find_submit_locator + # [/DEF:ScreenshotService._find_submit_locator:Function] - # #region ScreenshotService._goto_resilient [TYPE Function] - # @BRIEF Navigate without relying on networkidle for pages with long-polling or persistent requests. - # @PRE page is a valid Playwright page and url is non-empty. - # @POST Returns last navigation response or raises when both primary and fallback waits fail. + # [DEF:ScreenshotService._goto_resilient:Function] + # @PURPOSE: Navigate without relying on networkidle for pages with long-polling or persistent requests. + # @PRE: page is a valid Playwright page and url is non-empty. + # @POST: Returns last navigation response or raises when both primary and fallback waits fail. async def _goto_resilient( self, page, @@ -267,13 +269,13 @@ class ScreenshotService: f"[ScreenshotService._goto_resilient] Primary navigation wait '{primary_wait_until}' failed for {url}: {primary_error}" ) return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout) - # #endregion ScreenshotService._goto_resilient + # [/DEF:ScreenshotService._goto_resilient:Function] - # #region ScreenshotService.capture_dashboard [TYPE Function] - # @BRIEF Captures a full-page screenshot of a dashboard using Playwright and CDP. - # @PRE dashboard_id is a valid string, output_path is a writable path. - # @POST Returns True if screenshot is saved successfully. - # @SIDE_EFFECT Launches a browser, performs UI login, switches tabs, and writes a PNG file. + # [DEF:ScreenshotService.capture_dashboard:Function] + # @PURPOSE: Captures a full-page screenshot of a dashboard using Playwright and CDP. + # @PRE: dashboard_id is a valid string, output_path is a writable path. + # @POST: Returns True if screenshot is saved successfully. + # @SIDE_EFFECT: Launches a browser, performs UI login, switches tabs, and writes a PNG file. # @UX_STATE: [Navigating] -> Loading dashboard UI # @UX_STATE: [TabSwitching] -> Iterating through dashboard tabs to trigger lazy loading # @UX_STATE: [CalculatingHeight] -> Determining dashboard dimensions @@ -669,15 +671,15 @@ class ScreenshotService: await browser.close() return True - # #endregion ScreenshotService.capture_dashboard + # [/DEF:ScreenshotService.capture_dashboard:Function] # #endregion ScreenshotService # #region LLMClient [TYPE Class] # @BRIEF Wrapper for LLM provider APIs. class LLMClient: - # #region LLMClient.__init__ [TYPE Function] - # @BRIEF Initializes the LLMClient with provider settings. - # @PRE api_key, base_url, and default_model are non-empty strings. + # [DEF:LLMClient.__init__:Function] + # @PURPOSE: Initializes the LLMClient with provider settings. + # @PRE: api_key, base_url, and default_model are non-empty strings. def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str): self.provider_type = provider_type normalized_key = (api_key or "").strip() @@ -715,12 +717,12 @@ class LLMClient: default_headers=default_headers, http_client=http_client, ) - # #endregion LLMClient.__init__ + # [/DEF:LLMClient.__init__:Function] - # #region LLMClient._supports_json_response_format [TYPE Function] - # @BRIEF Detect whether provider/model is likely compatible with response_format=json_object. - # @PRE Client initialized with base_url and default_model. - # @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors. + # [DEF:LLMClient._supports_json_response_format:Function] + # @PURPOSE: Detect whether provider/model is likely compatible with response_format=json_object. + # @PRE: Client initialized with base_url and default_model. + # @POST: Returns False for known-incompatible combinations to avoid avoidable 400 errors. def _supports_json_response_format(self) -> bool: base = (self.base_url or "").lower() model = (self.default_model or "").lower() @@ -735,13 +737,13 @@ class LLMClient: if any(token in model for token in incompatible_tokens): return False return True - # #endregion LLMClient._supports_json_response_format + # [/DEF:LLMClient._supports_json_response_format:Function] - # #region LLMClient.get_json_completion [TYPE Function] - # @BRIEF Helper to handle LLM calls with JSON mode and fallback parsing. - # @PRE messages is a list of valid message dictionaries. - # @POST Returns a parsed JSON dictionary. - # @SIDE_EFFECT Calls external LLM API. + # [DEF:LLMClient.get_json_completion:Function] + # @PURPOSE: Helper to handle LLM calls with JSON mode and fallback parsing. + # @PRE: messages is a list of valid message dictionaries. + # @POST: Returns a parsed JSON dictionary. + # @SIDE_EFFECT: Calls external LLM API. def _should_retry(exception: Exception) -> bool: """Custom retry predicate that excludes authentication errors.""" # Don't retry on authentication errors @@ -857,13 +859,13 @@ class LLMClient: return json.loads(json_str) else: raise - # #endregion LLMClient.get_json_completion + # [/DEF:LLMClient.get_json_completion:Function] - # #region LLMClient.test_runtime_connection [TYPE Function] - # @BRIEF Validate provider credentials using the same chat completions transport as runtime analysis. - # @PRE Client is initialized with provider credentials and default_model. - # @POST Returns lightweight JSON payload when runtime auth/model path is valid. - # @SIDE_EFFECT Calls external LLM API. + # [DEF:LLMClient.test_runtime_connection:Function] + # @PURPOSE: Validate provider credentials using the same chat completions transport as runtime analysis. + # @PRE: Client is initialized with provider credentials and default_model. + # @POST: Returns lightweight JSON payload when runtime auth/model path is valid. + # @SIDE_EFFECT: Calls external LLM API. async def test_runtime_connection(self) -> Dict[str, Any]: with belief_scope("test_runtime_connection"): messages = [ @@ -873,13 +875,13 @@ class LLMClient: } ] return await self.get_json_completion(messages) - # #endregion LLMClient.test_runtime_connection + # [/DEF:LLMClient.test_runtime_connection:Function] - # #region LLMClient.analyze_dashboard [TYPE Function] - # @BRIEF Sends dashboard data (screenshot + logs) to LLM for health analysis. - # @PRE screenshot_path exists, logs is a list of strings. - # @POST Returns a structured analysis dictionary (status, summary, issues). - # @SIDE_EFFECT Reads screenshot file and calls external LLM API. + # [DEF:LLMClient.analyze_dashboard:Function] + # @PURPOSE: Sends dashboard data (screenshot + logs) to LLM for health analysis. + # @PRE: screenshot_path exists, logs is a list of strings. + # @POST: Returns a structured analysis dictionary (status, summary, issues). + # @SIDE_EFFECT: Reads screenshot file and calls external LLM API. async def analyze_dashboard( self, screenshot_path: str, @@ -945,7 +947,7 @@ class LLMClient: "summary": f"Failed to get response from LLM: {str(e)}", "issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}] } - # #endregion LLMClient.analyze_dashboard + # [/DEF:LLMClient.analyze_dashboard:Function] # #endregion LLMClient -# #endregion backend/src/plugins/llm_analysis/service.py +# [/DEF:backend/src/plugins/llm_analysis/service.py:Module] diff --git a/backend/src/plugins/mapper.py b/backend/src/plugins/mapper.py index 7f537a53..cfd2a30e 100644 --- a/backend/src/plugins/mapper.py +++ b/backend/src/plugins/mapper.py @@ -1,10 +1,9 @@ # #region MapperPluginModule [TYPE Module] [SEMANTICS plugin, mapper, datasets, postgresql, excel] # @BRIEF Implements a plugin for mapping dataset columns using external database connections or Excel files. -# @LAYER Plugins +# @LAYER: Plugins # @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool. -# @RELATION USES -> [TaskContext] +# @RELATION USES -> TaskContext -# [SECTION: IMPORTS] from typing import Dict, Any, Optional from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient @@ -13,7 +12,6 @@ from ..core.database import SessionLocal from ..models.connection import ConnectionConfig from ..core.utils.dataset_mapper import DatasetMapper from ..core.task_manager.context import TaskContext -# [/SECTION] # #region MapperPlugin [TYPE Class] # @BRIEF Plugin for mapping dataset columns verbose names. @@ -23,63 +21,63 @@ class MapperPlugin(PluginBase): """ @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the mapper plugin. - # @PRE Plugin instance exists. - # @POST Returns string ID. - # @RETURN str - "dataset-mapper" + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the mapper plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string ID. + # @RETURN: str - "dataset-mapper" def id(self) -> str: with belief_scope("id"): return "dataset-mapper" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the mapper plugin. - # @PRE Plugin instance exists. - # @POST Returns string name. - # @RETURN str - Plugin name. + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the mapper plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string name. + # @RETURN: str - Plugin name. def name(self) -> str: with belief_scope("name"): return "Dataset Mapper" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns a description of the mapper plugin. - # @PRE Plugin instance exists. - # @POST Returns string description. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a description of the mapper plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string description. + # @RETURN: str - Plugin description. def description(self) -> str: with belief_scope("description"): return "Map dataset column verbose names using PostgreSQL comments or Excel files." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the version of the mapper plugin. - # @PRE Plugin instance exists. - # @POST Returns string version. - # @RETURN str - "1.0.0" + # [DEF:version:Function] + # @PURPOSE: Returns the version of the mapper plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string version. + # @RETURN: str - "1.0.0" def version(self) -> str: with belief_scope("version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the mapper plugin. - # @RETURN str - "/tools/mapper" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the mapper plugin. + # @RETURN: str - "/tools/mapper" def ui_route(self) -> str: with belief_scope("ui_route"): return "/tools/mapper" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for the mapper plugin parameters. - # @PRE Plugin instance exists. - # @POST Returns dictionary schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for the mapper plugin parameters. + # @PRE: Plugin instance exists. + # @POST: Returns dictionary schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: with belief_scope("get_schema"): return { @@ -125,10 +123,10 @@ class MapperPlugin(PluginBase): }, "required": ["env", "dataset_id", "source"] } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Executes the dataset mapping logic with TaskContext support. + # [DEF:execute:Function] + # @PURPOSE: Executes the dataset mapping logic with TaskContext support. # @PARAM: params (Dict[str, Any]) - Mapping parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary. @@ -207,7 +205,7 @@ class MapperPlugin(PluginBase): except Exception as e: log.error(f"Mapping failed: {e}") raise - # #endregion execute + # [/DEF:execute:Function] # #endregion MapperPlugin -# #endregion MapperPluginModule +# #endregion MapperPluginModule \ No newline at end of file diff --git a/backend/src/plugins/migration.py b/backend/src/plugins/migration.py index 932f472b..2c2bdb4a 100755 --- a/backend/src/plugins/migration.py +++ b/backend/src/plugins/migration.py @@ -1,26 +1,23 @@ # #region MigrationPlugin [C:5] [TYPE Module] [SEMANTICS migration, superset, automation, dashboard, plugin, transformation] # @BRIEF Orchestrates export, DB-mapping transformation, and import of Superset dashboards across environments. -# @LAYER App -# @PRE Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context. -# @POST Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees. -# @SIDE_EFFECT Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows. -# @DATA_CONTRACT Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set] -# @INVARIANT Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. -# @RELATION IMPLEMENTS -> [PluginBase] -# @RELATION DEPENDS_ON -> [SupersetClient] -# @RELATION DEPENDS_ON -> [MigrationEngine] -# @RELATION DEPENDS_ON -> [IdMappingService] -# @RELATION USES -> [TaskContext] +# @LAYER: App +# @RELATION IMPLEMENTS -> PluginBase +# @RELATION DEPENDS_ON -> SupersetClient +# @RELATION DEPENDS_ON -> MigrationEngine +# @RELATION DEPENDS_ON -> IdMappingService +# @RELATION USES -> TaskContext +# @PRE: Plugin loader can resolve infrastructure dependencies and execution requests provide validated migration context. +# @POST: Plugin metadata remains stable and migration execution preserves mapped-environment import guarantees. +# @SIDE_EFFECT: Reads config, opens database sessions, creates temporary artifacts, and triggers Superset export/import workflows. +# @DATA_CONTRACT: Input[TaskContext{from_env,to_env,dashboard_regex,replace_db_config,from_db_id,to_db_id,passwords?}] -> Output[MigrationResult|artifact set] +# @INVARIANT: Dashboards must never be imported with unmapped/source DB connections to prevent data leaks or cross-environment pollution. from typing import Dict, Any, Optional import re from ..core.plugin_base import PluginBase -from ..core.logger import belief_scope -from ..core.cot_logger import MarkerLogger +from ..core.logger import belief_scope, logger as app_logger from ..core.superset_client import SupersetClient - -log = MarkerLogger("MigrationPlugin") from ..core.utils.fileio import create_temp_file from ..dependencies import get_config_manager from ..core.migration_engine import MigrationEngine @@ -31,8 +28,8 @@ from ..core.task_manager.context import TaskContext # #region MigrationPlugin [TYPE Class] # @BRIEF Implementation of the migration plugin workflow and transformation orchestration. -# @PRE SupersetClient authenticated, database session active -# @POST Returns MigrationResult with success/failure status and artifact list +# @PRE: SupersetClient authenticated, database session active +# @POST: Returns MigrationResult with success/failure status and artifact list # @TEST_FIXTURE: superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip # @TEST_FIXTURE: db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}} # @TEST_FIXTURE: password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}} @@ -45,68 +42,68 @@ class MigrationPlugin(PluginBase): """ @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the migration plugin. - # @PRE None. - # @POST Returns stable string "superset-migration". - # @RETURN str + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the migration plugin. + # @PRE: None. + # @POST: Returns stable string "superset-migration". + # @RETURN: str def id(self) -> str: with belief_scope("MigrationPlugin.id"): return "superset-migration" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the plugin. - # @PRE None. - # @POST Returns "Superset Dashboard Migration". - # @RETURN str + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the plugin. + # @PRE: None. + # @POST: Returns "Superset Dashboard Migration". + # @RETURN: str def name(self) -> str: with belief_scope("MigrationPlugin.name"): return "Superset Dashboard Migration" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns the semantic description of the plugin. - # @PRE None. - # @POST Returns description string. - # @RETURN str + # [DEF:description:Function] + # @PURPOSE: Returns the semantic description of the plugin. + # @PRE: None. + # @POST: Returns description string. + # @RETURN: str def description(self) -> str: with belief_scope("MigrationPlugin.description"): return "Migrates dashboards between Superset environments." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the semantic version of the migration plugin. - # @PRE None. - # @POST Returns "1.0.0". - # @RETURN str + # [DEF:version:Function] + # @PURPOSE: Returns the semantic version of the migration plugin. + # @PRE: None. + # @POST: Returns "1.0.0". + # @RETURN: str def version(self) -> str: with belief_scope("MigrationPlugin.version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend routing anchor for the plugin. - # @PRE None. - # @POST Returns "/migration". - # @RETURN str + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend routing anchor for the plugin. + # @PRE: None. + # @POST: Returns "/migration". + # @RETURN: str def ui_route(self) -> str: with belief_scope("MigrationPlugin.ui_route"): return "/migration" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Generates the JSON Schema for the plugin execution form dynamically. - # @PRE ConfigManager is accessible and environments are defined. - # @POST Returns a JSON Schema dict matching current system environments. - # @RETURN Dict[str, Any] + # [DEF:get_schema:Function] + # @PURPOSE: Generates the JSON Schema for the plugin execution form dynamically. + # @PRE: ConfigManager is accessible and environments are defined. + # @POST: Returns a JSON Schema dict matching current system environments. + # @RETURN: Dict[str, Any] def get_schema(self) -> Dict[str, Any]: with belief_scope("MigrationPlugin.get_schema"): - log.reason("Generating migration UI schema") + app_logger.reason("Generating migration UI schema") config_manager = get_config_manager() envs = [e.name for e in config_manager.get_environments()] @@ -149,12 +146,12 @@ class MigrationPlugin(PluginBase): }, "required": ["from_env", "to_env", "dashboard_regex"], } - log.reflect("Schema generated successfully", payload={"environments_count": len(envs)}) + app_logger.reflect("Schema generated successfully", extra={"environments_count": len(envs)}) return schema - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion. + # [DEF:execute:Function] + # @PURPOSE: Orchestrates the dashboard migration pipeline including extraction, AST mutation, and ingestion. # @PARAM: params (Dict[str, Any]) - Extracted parameters from UI/API execution request. # @PARAM: context (Optional[TaskContext]) - Dependency injected TaskContext for IO tracing. # @PRE: Source and target environments must resolve. Matching dashboards must exist. @@ -169,7 +166,7 @@ class MigrationPlugin(PluginBase): # @TEST_EDGE: target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS] async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): with belief_scope("MigrationPlugin.execute"): - log.reason("Evaluating migration task parameters", payload={"params": params}) + app_logger.reason("Evaluating migration task parameters", extra={"params": params}) source_env_id = params.get("source_env_id") target_env_id = params.get("target_env_id") @@ -185,11 +182,11 @@ class MigrationPlugin(PluginBase): from ..dependencies import get_task_manager tm = get_task_manager() - exec_log = context.logger if context else log - superset_log = exec_log.with_source("superset_api") if context else exec_log - migration_log = exec_log.with_source("migration") if context else exec_log + log = context.logger if context else app_logger + superset_log = log.with_source("superset_api") if context else log + migration_log = log.with_source("migration") if context else log - exec_log.info("Starting migration task.") + log.info("Starting migration task.") try: config_manager = get_config_manager() @@ -200,13 +197,13 @@ class MigrationPlugin(PluginBase): tgt_env = next((e for e in environments if e.id == target_env_id), None) if target_env_id else next((e for e in environments if e.name == to_env_name), None) if not src_env or not tgt_env: - log.explore("Environment resolution failed", error="Could not resolve source or target environment", payload={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) + app_logger.explore("Environment resolution failed", extra={"src": source_env_id or from_env_name, "tgt": target_env_id or to_env_name}) raise ValueError(f"Could not resolve source or target environment. Source: {source_env_id or from_env_name}, Target: {target_env_id or to_env_name}") from_env_name = src_env.name to_env_name = tgt_env.name - log.reason("Environments resolved successfully", payload={"from": from_env_name, "to": to_env_name}) + app_logger.reason("Environments resolved successfully", extra={"from": from_env_name, "to": to_env_name}) migration_result = { "status": "SUCCESS", @@ -233,12 +230,12 @@ class MigrationPlugin(PluginBase): regex_pattern = re.compile(str(dashboard_regex), re.IGNORECASE) dashboards_to_migrate = [d for d in all_dashboards if regex_pattern.search(d.get("dashboard_title", ""))] else: - log.explore("No deterministic selection criteria provided", error="No dashboard selection criteria (selected_ids or dashboard_regex)") + app_logger.explore("No deterministic selection criteria provided") migration_result["status"] = "NO_SELECTION" return migration_result if not dashboards_to_migrate: - log.explore("Zero dashboards match selection criteria", error="No dashboards matched the given selection criteria") + app_logger.explore("Zero dashboards match selection criteria") migration_result["status"] = "NO_MATCHES" return migration_result @@ -250,7 +247,7 @@ class MigrationPlugin(PluginBase): db_mapping = {} if replace_db_config: - log.reason("Fetching environment DB mappings from catalog") + app_logger.reason("Fetching environment DB mappings from catalog") db = SessionLocal() try: src_env_db = db.query(Environment).filter(Environment.name == from_env_name).first() @@ -264,7 +261,7 @@ class MigrationPlugin(PluginBase): stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings} stored_map_dict.update(db_mapping) db_mapping = stored_map_dict - exec_log.info(f"Loaded {len(stored_mappings)} database mappings from database.") + log.info(f"Loaded {len(stored_mappings)} database mappings from database.") finally: db.close() @@ -274,7 +271,7 @@ class MigrationPlugin(PluginBase): # Migration Loop for dash in dashboards_to_migrate: dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"] - log.reason(f"Starting pipeline for dashboard '{title}'", payload={"dash_id": dash_id}) + app_logger.reason(f"Starting pipeline for dashboard '{title}'", extra={"dash_id": dash_id}) try: exported_content, _ = from_c.export_dashboard(dash_id) @@ -292,10 +289,10 @@ class MigrationPlugin(PluginBase): if not success and replace_db_config: if task_id: - log.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", error="AST transform blocked by missing mappings", payload={"task_id": task_id}) + app_logger.explore("Missing mapping blocks AST transform. Pausing task for user intervention.", extra={"task_id": task_id}) await tm.wait_for_resolution(task_id) - log.reason("Task resumed, re-evaluating mapping states") + app_logger.reason("Task resumed, re-evaluating mapping states") db = SessionLocal() try: src_env_rt = db.query(Environment).filter(Environment.name == from_env_name).first() @@ -318,12 +315,12 @@ class MigrationPlugin(PluginBase): ) if success: - log.reason("Pushing transformed ZIP to target Superset") + app_logger.reason("Pushing transformed ZIP to target Superset") to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug) migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) - log.reflect("Import successful", payload={"title": title}) + app_logger.reflect("Import successful", extra={"title": title}) else: - log.explore("Transformation strictly failed, bypassing ingestion", error="Dashboard ZIP transformation failed") + app_logger.explore("Transformation strictly failed, bypassing ingestion") migration_log.error(f"Failed to transform ZIP for dashboard {title}") migration_result["failed_dashboards"].append({ "id": dash_id, "title": title, "error": "Failed to transform ZIP" @@ -341,7 +338,7 @@ class MigrationPlugin(PluginBase): if match_alt: db_name = match_alt.group(1) - log.explore("Missing DB password detected during ingestion. Escalating to UI.", error="Missing DB password for import", payload={"db_name": db_name}) + app_logger.explore(f"Missing DB password detected during ingestion. Escalating to UI.", extra={"db_name": db_name}) if task_id: tm.await_input(task_id, { @@ -355,15 +352,15 @@ class MigrationPlugin(PluginBase): passwords = task.params.get("passwords", {}) if passwords: - log.reason(f"Retrying import for {title} with injected credentials") + app_logger.reason(f"Retrying import for {title} with injected credentials") to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords) migration_result["migrated_dashboards"].append({"id": dash_id, "title": title}) - log.reflect("Password injection unblocked import") + app_logger.reflect("Password injection unblocked import") if "passwords" in task.params: del task.params["passwords"] continue - log.explore("Catastrophic dashboard ingestion failure", error=f"Dashboard ingestion failed: {exc}") + app_logger.explore(f"Catastrophic dashboard ingestion failure: {exc}") migration_result["failed_dashboards"].append({"id": dash_id, "title": title, "error": str(exc)}) if migration_result["failed_dashboards"]: @@ -371,21 +368,21 @@ class MigrationPlugin(PluginBase): # Post-Migration ID Mapping Synchronization try: - log.reason("Executing incremental ID catalog sync on target") + app_logger.reason("Executing incremental ID catalog sync on target") db_session = SessionLocal() mapping_service = IdMappingService(db_session) mapping_service.sync_environment(tgt_env.id, to_c, incremental=True) db_session.close() - log.reflect("Incremental catalog sync closed out cleanly") + app_logger.reflect("Incremental catalog sync closed out cleanly") except Exception as sync_exc: - log.explore("ID Mapping sync failed, mapping state might be degraded", error=f"ID Mapping sync error: {sync_exc}") + app_logger.explore(f"ID Mapping sync failed, mapping state might be degraded: {sync_exc}") - log.reflect("Migration cycle fully resolved", payload={"result": migration_result}) + app_logger.reflect("Migration cycle fully resolved", extra={"result": migration_result}) return migration_result except Exception as e: - log.explore("Fatal plugin failure", error=f"Plugin execution failed: {e}", exc_info=True) + app_logger.explore(f"Fatal plugin failure: {e}", exc_info=True) raise e - # #endregion execute -# #endregion MigrationPlugin + # [/DEF:execute:Function] # #endregion MigrationPlugin +# #endregion MigrationPlugin \ No newline at end of file diff --git a/backend/src/plugins/search.py b/backend/src/plugins/search.py index c2da73d6..d46f0fcd 100644 --- a/backend/src/plugins/search.py +++ b/backend/src/plugins/search.py @@ -1,17 +1,15 @@ # #region SearchPluginModule [TYPE Module] [SEMANTICS plugin, search, datasets, regex, superset] # @BRIEF Implements a plugin for searching text patterns across all datasets in a specific Superset environment. -# @LAYER Plugins +# @LAYER: Plugins # @RELATION Inherits from PluginBase. Uses SupersetClient from core. -# @RELATION USES -> [TaskContext] +# @RELATION USES -> TaskContext -# [SECTION: IMPORTS] import re from typing import Dict, Any, Optional from ..core.plugin_base import PluginBase from ..core.superset_client import SupersetClient from ..core.logger import logger, belief_scope from ..core.task_manager.context import TaskContext -# [/SECTION] # #region SearchPlugin [TYPE Class] # @BRIEF Plugin for searching text patterns in Superset datasets. @@ -21,63 +19,63 @@ class SearchPlugin(PluginBase): """ @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the search plugin. - # @PRE Plugin instance exists. - # @POST Returns string ID. - # @RETURN str - "search-datasets" + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the search plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string ID. + # @RETURN: str - "search-datasets" def id(self) -> str: with belief_scope("id"): return "search-datasets" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the search plugin. - # @PRE Plugin instance exists. - # @POST Returns string name. - # @RETURN str - Plugin name. + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the search plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string name. + # @RETURN: str - Plugin name. def name(self) -> str: with belief_scope("name"): return "Search Datasets" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns a description of the search plugin. - # @PRE Plugin instance exists. - # @POST Returns string description. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a description of the search plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string description. + # @RETURN: str - Plugin description. def description(self) -> str: with belief_scope("description"): return "Search for text patterns across all datasets in a specific environment." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the version of the search plugin. - # @PRE Plugin instance exists. - # @POST Returns string version. - # @RETURN str - "1.0.0" + # [DEF:version:Function] + # @PURPOSE: Returns the version of the search plugin. + # @PRE: Plugin instance exists. + # @POST: Returns string version. + # @RETURN: str - "1.0.0" def version(self) -> str: with belief_scope("version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the search plugin. - # @RETURN str - "/tools/search" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the search plugin. + # @RETURN: str - "/tools/search" def ui_route(self) -> str: with belief_scope("ui_route"): return "/tools/search" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for the search plugin parameters. - # @PRE Plugin instance exists. - # @POST Returns dictionary schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for the search plugin parameters. + # @PRE: Plugin instance exists. + # @POST: Returns dictionary schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: with belief_scope("get_schema"): return { @@ -96,10 +94,10 @@ class SearchPlugin(PluginBase): }, "required": ["env", "query"] } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Executes the dataset search logic with TaskContext support. + # [DEF:execute:Function] + # @PURPOSE: Executes the dataset search logic with TaskContext support. # @PARAM: params (Dict[str, Any]) - Search parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: Params contain valid 'env' and 'query'. @@ -175,10 +173,10 @@ class SearchPlugin(PluginBase): except Exception as e: log.error(f"Error during search: {e}") raise - # #endregion execute + # [/DEF:execute:Function] - # #region _get_context [TYPE Function] - # @BRIEF Extracts a small context around the match for display. + # [DEF:_get_context:Function] + # @PURPOSE: Extracts a small context around the match for display. # @PARAM: text (str) - The full text to extract context from. # @PARAM: match_text (str) - The matched text pattern. # @PARAM: context_lines (int) - Number of lines of context to include. @@ -213,7 +211,7 @@ class SearchPlugin(PluginBase): return "\n".join(context) return text[:100] + "..." if len(text) > 100 else text - # #endregion _get_context + # [/DEF:_get_context:Function] # #endregion SearchPlugin -# #endregion SearchPluginModule +# #endregion SearchPluginModule \ No newline at end of file diff --git a/backend/src/plugins/storage/plugin.py b/backend/src/plugins/storage/plugin.py index 3032b359..7380d49c 100644 --- a/backend/src/plugins/storage/plugin.py +++ b/backend/src/plugins/storage/plugin.py @@ -1,14 +1,13 @@ # #region StoragePlugin [TYPE Module] [SEMANTICS storage, files, filesystem, plugin] +# # @BRIEF Provides core filesystem operations for managing backups and repositories. -# @LAYER App -# @INVARIANT All file operations must be restricted to the configured storage root. -# @RELATION IMPLEMENTS -> [PluginBase] -# @RELATION DEPENDS_ON -> [backend.src.models.storage] -# @RELATION USES -> [TaskContext] -# +# @LAYER: App +# @RELATION IMPLEMENTS -> PluginBase +# @RELATION DEPENDS_ON -> backend.src.models.storage +# @RELATION USES -> TaskContext # +# @INVARIANT: All file operations must be restricted to the configured storage root. -# [SECTION: IMPORTS] import os import shutil from pathlib import Path @@ -17,14 +16,10 @@ from typing import Dict, Any, List, Optional from fastapi import UploadFile from ...core.plugin_base import PluginBase -from ...core.logger import belief_scope -from ...core.cot_logger import MarkerLogger +from ...core.logger import belief_scope, logger from ...models.storage import StoredFile, FileCategory - -log = MarkerLogger("StoragePlugin") from ...dependencies import get_config_manager from ...core.task_manager.context import TaskContext -# [/SECTION] # #region StoragePlugin [TYPE Class] # @BRIEF Implementation of the storage management plugin. @@ -33,73 +28,73 @@ class StoragePlugin(PluginBase): Plugin for managing local file storage for backups and repositories. """ - # #region __init__ [TYPE Function] - # @BRIEF Initializes the StoragePlugin and ensures required directories exist. - # @PRE Configuration manager must be accessible. - # @POST Storage root and category directories are created on disk. + # [DEF:__init__:Function] + # @PURPOSE: Initializes the StoragePlugin and ensures required directories exist. + # @PRE: Configuration manager must be accessible. + # @POST: Storage root and category directories are created on disk. def __init__(self): with belief_scope("StoragePlugin:init"): self.ensure_directories() - # #endregion __init__ + # [/DEF:__init__:Function] @property - # #region id [TYPE Function] - # @BRIEF Returns the unique identifier for the storage plugin. - # @PRE None. - # @POST Returns the plugin ID string. - # @RETURN str - "storage-manager" + # [DEF:id:Function] + # @PURPOSE: Returns the unique identifier for the storage plugin. + # @PRE: None. + # @POST: Returns the plugin ID string. + # @RETURN: str - "storage-manager" def id(self) -> str: with belief_scope("StoragePlugin:id"): return "storage-manager" - # #endregion id + # [/DEF:id:Function] @property - # #region name [TYPE Function] - # @BRIEF Returns the human-readable name of the storage plugin. - # @PRE None. - # @POST Returns the plugin name string. - # @RETURN str - "Storage Manager" + # [DEF:name:Function] + # @PURPOSE: Returns the human-readable name of the storage plugin. + # @PRE: None. + # @POST: Returns the plugin name string. + # @RETURN: str - "Storage Manager" def name(self) -> str: with belief_scope("StoragePlugin:name"): return "Storage Manager" - # #endregion name + # [/DEF:name:Function] @property - # #region description [TYPE Function] - # @BRIEF Returns a description of the storage plugin. - # @PRE None. - # @POST Returns the plugin description string. - # @RETURN str - Plugin description. + # [DEF:description:Function] + # @PURPOSE: Returns a description of the storage plugin. + # @PRE: None. + # @POST: Returns the plugin description string. + # @RETURN: str - Plugin description. def description(self) -> str: with belief_scope("StoragePlugin:description"): return "Manages local file storage for backups and repositories." - # #endregion description + # [/DEF:description:Function] @property - # #region version [TYPE Function] - # @BRIEF Returns the version of the storage plugin. - # @PRE None. - # @POST Returns the version string. - # @RETURN str - "1.0.0" + # [DEF:version:Function] + # @PURPOSE: Returns the version of the storage plugin. + # @PRE: None. + # @POST: Returns the version string. + # @RETURN: str - "1.0.0" def version(self) -> str: with belief_scope("StoragePlugin:version"): return "1.0.0" - # #endregion version + # [/DEF:version:Function] @property - # #region ui_route [TYPE Function] - # @BRIEF Returns the frontend route for the storage plugin. - # @RETURN str - "/tools/storage" + # [DEF:ui_route:Function] + # @PURPOSE: Returns the frontend route for the storage plugin. + # @RETURN: str - "/tools/storage" def ui_route(self) -> str: with belief_scope("StoragePlugin:ui_route"): return "/tools/storage" - # #endregion ui_route + # [/DEF:ui_route:Function] - # #region get_schema [TYPE Function] - # @BRIEF Returns the JSON schema for storage plugin parameters. - # @PRE None. - # @POST Returns a dictionary representing the JSON schema. - # @RETURN Dict[str, Any] - JSON schema. + # [DEF:get_schema:Function] + # @PURPOSE: Returns the JSON schema for storage plugin parameters. + # @PRE: None. + # @POST: Returns a dictionary representing the JSON schema. + # @RETURN: Dict[str, Any] - JSON schema. def get_schema(self) -> Dict[str, Any]: with belief_scope("StoragePlugin:get_schema"): return { @@ -113,23 +108,30 @@ class StoragePlugin(PluginBase): }, "required": ["category"] } - # #endregion get_schema + # [/DEF:get_schema:Function] - # #region execute [TYPE Function] - # @BRIEF Executes storage-related tasks with TaskContext support. + # [DEF:execute:Function] + # @PURPOSE: Executes storage-related tasks with TaskContext support. # @PARAM: params (Dict[str, Any]) - Storage parameters. # @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution. # @PRE: params must match the plugin schema. # @POST: Task is executed and logged. async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None): with belief_scope("StoragePlugin:execute"): - log.reason(f"Executing with params: {params}") - # #endregion execute + # Use TaskContext logger if available, otherwise fall back to app logger + log = context.logger if context else logger + + # Create sub-loggers for different components + storage_log = log.with_source("storage") if context else log + log.with_source("filesystem") if context else log + + storage_log.info(f"Executing with params: {params}") + # [/DEF:execute:Function] - # #region get_storage_root [TYPE Function] - # @BRIEF Resolves the absolute path to the storage root. - # @PRE Settings must define a storage root path. - # @POST Returns a Path object representing the storage root. + # [DEF:get_storage_root:Function] + # @PURPOSE: Resolves the absolute path to the storage root. + # @PRE: Settings must define a storage root path. + # @POST: Returns a Path object representing the storage root. def get_storage_root(self) -> Path: with belief_scope("StoragePlugin:get_storage_root"): config_manager = get_config_manager() @@ -146,10 +148,10 @@ class StoragePlugin(PluginBase): project_root = Path(__file__).parents[3] root = (project_root / root).resolve() return root - # #endregion get_storage_root + # [/DEF:get_storage_root:Function] - # #region resolve_path [TYPE Function] - # @BRIEF Resolves a dynamic path pattern using provided variables. + # [DEF:resolve_path:Function] + # @PURPOSE: Resolves a dynamic path pattern using provided variables. # @PARAM: pattern (str) - The path pattern to resolve. # @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern. # @PRE: pattern must be a valid format string. @@ -167,16 +169,16 @@ class StoragePlugin(PluginBase): # Clean up any double slashes or leading/trailing slashes for relative path return os.path.normpath(resolved).strip("/") except KeyError as e: - log.explore("Missing variable for path resolution", error=str(e)) + logger.warning(f"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}") # Fallback to literal pattern if formatting fails partially (or handle as needed) return pattern.replace("{", "").replace("}", "") - # #endregion resolve_path + # [/DEF:resolve_path:Function] - # #region ensure_directories [TYPE Function] - # @BRIEF Creates the storage root and category subdirectories if they don't exist. - # @PRE Storage root must be resolvable. - # @POST Directories are created on the filesystem. - # @SIDE_EFFECT Creates directories on the filesystem. + # [DEF:ensure_directories:Function] + # @PURPOSE: Creates the storage root and category subdirectories if they don't exist. + # @PRE: Storage root must be resolvable. + # @POST: Directories are created on the filesystem. + # @SIDE_EFFECT: Creates directories on the filesystem. def ensure_directories(self): with belief_scope("StoragePlugin:ensure_directories"): root = self.get_storage_root() @@ -184,13 +186,13 @@ class StoragePlugin(PluginBase): # Use singular name for consistency with BackupPlugin and GitService path = root / category.value path.mkdir(parents=True, exist_ok=True) - log.reason(f"Ensured directory: {path}") - # #endregion ensure_directories + logger.debug(f"[StoragePlugin][Action] Ensured directory: {path}") + # [/DEF:ensure_directories:Function] - # #region validate_path [TYPE Function] - # @BRIEF Prevents path traversal attacks by ensuring the path is within the storage root. - # @PRE path must be a Path object. - # @POST Returns the resolved absolute path if valid, otherwise raises ValueError. + # [DEF:validate_path:Function] + # @PURPOSE: Prevents path traversal attacks by ensuring the path is within the storage root. + # @PRE: path must be a Path object. + # @POST: Returns the resolved absolute path if valid, otherwise raises ValueError. def validate_path(self, path: Path) -> Path: with belief_scope("StoragePlugin:validate_path"): root = self.get_storage_root().resolve() @@ -198,13 +200,13 @@ class StoragePlugin(PluginBase): try: resolved.relative_to(root) except ValueError: - log.explore(f"Path traversal detected: {resolved} is not under {root}", error="Path outside storage root") + logger.error(f"[StoragePlugin][Coherence:Failed] Path traversal detected: {resolved} is not under {root}") raise ValueError("Access denied: Path is outside of storage root.") return resolved - # #endregion validate_path + # [/DEF:validate_path:Function] - # #region list_files [TYPE Function] - # @BRIEF Lists all files and directories in a specific category and subpath. + # [DEF:list_files:Function] + # @PURPOSE: Lists all files and directories in a specific category and subpath. # @PARAM: category (Optional[FileCategory]) - The category to list. # @PARAM: subpath (Optional[str]) - Nested path within the category. # @PARAM: recursive (bool) - Whether to scan nested subdirectories recursively. @@ -219,8 +221,8 @@ class StoragePlugin(PluginBase): ) -> List[StoredFile]: with belief_scope("StoragePlugin:list_files"): root = self.get_storage_root() - log.reason( - f"Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}" + logger.info( + f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}, recursive: {recursive}" ) files = [] @@ -256,7 +258,7 @@ class StoragePlugin(PluginBase): if not target_dir.exists(): continue - log.reason(f"Scanning directory: {target_dir}") + logger.debug(f"[StoragePlugin][Action] Scanning directory: {target_dir}") if recursive: for current_root, dirs, filenames in os.walk(target_dir): @@ -299,10 +301,10 @@ class StoragePlugin(PluginBase): # Sort: directories first, then by name return sorted(files, key=lambda x: (x.mime_type != "directory", x.name)) - # #endregion list_files + # [/DEF:list_files:Function] - # #region save_file [TYPE Function] - # @BRIEF Saves an uploaded file to the specified category and optional subpath. + # [DEF:save_file:Function] + # @PURPOSE: Saves an uploaded file to the specified category and optional subpath. # @PARAM: file (UploadFile) - The uploaded file. # @PARAM: category (FileCategory) - The target category. # @PARAM: subpath (Optional[str]) - The target subpath. @@ -333,10 +335,10 @@ class StoragePlugin(PluginBase): category=category, mime_type=file.content_type ) - # #endregion save_file + # [/DEF:save_file:Function] - # #region delete_file [TYPE Function] - # @BRIEF Deletes a file or directory from the specified category and path. + # [DEF:delete_file:Function] + # @PURPOSE: Deletes a file or directory from the specified category and path. # @PARAM: category (FileCategory) - The category. # @PARAM: path (str) - The relative path of the file or directory. # @PRE: path must belong to the specified category and exist on disk. @@ -356,13 +358,13 @@ class StoragePlugin(PluginBase): shutil.rmtree(full_path) else: full_path.unlink() - log.reflect(f"Deleted: {full_path}") + logger.info(f"[StoragePlugin][Action] Deleted: {full_path}") else: raise FileNotFoundError(f"Item {path} not found") - # #endregion delete_file + # [/DEF:delete_file:Function] - # #region get_file_path [TYPE Function] - # @BRIEF Returns the absolute path of a file for download. + # [DEF:get_file_path:Function] + # @PURPOSE: Returns the absolute path of a file for download. # @PARAM: category (FileCategory) - The category. # @PARAM: path (str) - The relative path of the file. # @PRE: path must belong to the specified category and be a file. @@ -380,7 +382,7 @@ class StoragePlugin(PluginBase): raise FileNotFoundError(f"File {path} not found") return file_path - # #endregion get_file_path + # [/DEF:get_file_path:Function] # #endregion StoragePlugin # #endregion StoragePlugin diff --git a/backend/src/plugins/translate/__init__.py b/backend/src/plugins/translate/__init__.py index eb4efb6c..16f76d62 100644 --- a/backend/src/plugins/translate/__init__.py +++ b/backend/src/plugins/translate/__init__.py @@ -1,2 +1,3 @@ -# #region TranslatePluginPackage [C:1] [TYPE Package] +# #region TranslatePluginPackage [TYPE Package] +# @BRIEF Translation plugin package for LLM-based cross-Superset SQL/dashboard translation. # #endregion TranslatePluginPackage diff --git a/backend/src/plugins/translate/__tests__/__init__.py b/backend/src/plugins/translate/__tests__/__init__.py index e82687da..b0a441a2 100644 --- a/backend/src/plugins/translate/__tests__/__init__.py +++ b/backend/src/plugins/translate/__tests__/__init__.py @@ -1,3 +1,3 @@ -# #region TranslatePluginTestsPackage [TYPE Package] -# @BRIEF Tests for the translate plugin package. -# #endregion TranslatePluginTestsPackage +# [DEF:TranslatePluginTestsPackage:Package] +# @PURPOSE: Tests for the translate plugin package. +# [/DEF:TranslatePluginTestsPackage:Package] diff --git a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py index 4e158885..b8d89df9 100644 --- a/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py +++ b/backend/src/plugins/translate/__tests__/test_clickhouse_insert_integration.py @@ -12,9 +12,7 @@ # @TEST_EDGE: unix_seconds_timestamp -> converted to YYYY-MM-DD # @TEST_EDGE: non_timestamp_string -> passed through unchanged -import pytest -from unittest.mock import MagicMock, patch, call -from typing import Any, Dict, List +from unittest.mock import MagicMock, patch from src.plugins.translate.sql_generator import ( SQLGenerator, @@ -37,11 +35,15 @@ def _make_mock_job(**overrides): job.upsert_strategy = "INSERT" job.target_database_id = "1" job.environment_id = "test-env" + job.context_columns = [] for k, v in overrides.items(): setattr(job, k, v) return job +# [/DEF:_make_mock_job:Function] + + # [DEF:_make_mock_record:Function] # @PURPOSE: Create a mock TranslationRecord with source_data containing a Unix timestamp. def _make_mock_record( @@ -68,6 +70,9 @@ def _make_mock_record( return rec +# [/DEF:_make_mock_record:Function] + + # #region TestClickHouseTimestampNormalization [TYPE Class] # @BRIEF Verify Unix timestamp detection and conversion for ClickHouse Date columns. class TestClickHouseTimestampNormalization: @@ -119,6 +124,15 @@ class TestClickHouseTimestampNormalization: def test_postgresql_encode_not_affected(self) -> None: result = _encode_sql_value("1726358400000.0", dialect="postgresql") assert result == "'1726358400000.0'" + + # [/DEF:test_financial_arrears_date_millis:Function] + # [/DEF:test_financial_arrears_date_seconds:Function] + # [/DEF:test_document_number_not_affected:Function] + # [/DEF:test_empty_string:Function] + # [/DEF:test_none_value:Function] + # [/DEF:test_clickhouse_encode_with_timestamp:Function] + # [/DEF:test_clickhouse_encode_with_document_number:Function] + # [/DEF:test_postgresql_encode_not_affected:Function] # #endregion TestClickHouseTimestampNormalization @@ -227,6 +241,10 @@ class TestClickHouseInsertSqlGeneration: assert "'2024-09-15'" in sql assert "1726358400" not in sql or "'2024-09-15'" in sql + + # [/DEF:test_insert_with_timestamp_key_cols:Function] + # [/DEF:test_insert_multiple_rows_mixed_timestamps:Function] + # [/DEF:test_insert_with_seconds_timestamp:Function] # #endregion TestClickHouseInsertSqlGeneration @@ -295,6 +313,9 @@ class TestClickHouseJoinVerification: assert "`report_date`" in insert_sql assert "`document_number`" in insert_sql assert "`comment_text_en`" in insert_sql + + # [/DEF:test_join_query_structure:Function] + # [/DEF:test_insert_then_join_consistency:Function] # #endregion TestClickHouseJoinVerification @@ -383,6 +404,8 @@ class TestOrchestratorInsertFlow: # Verify result assert result["status"] == "success" + + # [/DEF:test_generate_and_insert_sql_with_timestamps:Function] # #endregion TestOrchestratorInsertFlow @@ -509,4 +532,8 @@ LIMIT 10 assert "f_a.document_number = f_c.document_number" in join_sql assert "dm_view.financial_arrears" in join_sql assert "dm_view.financial_comments_translated" in join_sql + + # [/DEF:test_full_insert_sql_valid_for_clickhouse:Function] + # [/DEF:test_verification_join_query:Function] # #endregion TestClickHouseEndToEnd +# [/DEF:ClickHouseInsertIntegration:Module] diff --git a/backend/src/plugins/translate/__tests__/test_dictionary.py b/backend/src/plugins/translate/__tests__/test_dictionary.py index c7934462..90328c46 100644 --- a/backend/src/plugins/translate/__tests__/test_dictionary.py +++ b/backend/src/plugins/translate/__tests__/test_dictionary.py @@ -1,6 +1,8 @@ -# #region DictionaryTests [C:3] [TYPE Module] [SEMANTICS tests, dictionary, crud, import, filter] -# @BRIEF Validate DictionaryManager CRUD, import, deletion guards, and batch filtering. -# @RELATION BINDS_TO -> [DictionaryManager:Class] +# [DEF:DictionaryTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, dictionary, crud, import, filter +# @PURPOSE: Validate DictionaryManager CRUD, import, deletion guards, and batch filtering. +# @RELATION: BINDS_TO -> [DictionaryManager:Class] # # @TEST_CONTRACT: [DictionaryManager] -> { # invariants: [ @@ -35,15 +37,16 @@ from src.plugins.translate.dictionary import DictionaryManager from src.plugins.translate._utils import _normalize_term, _detect_delimiter -# #region _FakeJob [C:1] [TYPE Class] -# @BRIEF Helper to create inline TranslationJob records. +# [DEF:_FakeJob:Class] +# @COMPLEXITY: 1 +# @PURPOSE: Helper to create inline TranslationJob records. class _FakeJob: pass -# #endregion _FakeJob +# [/DEF:_FakeJob:Class] -# #region db_session [TYPE Fixture] -# @BRIEF Provide an in-memory SQLite session for each test, with tables created and torn down. +# [DEF:db_session:Fixture] +# @PURPOSE: Provide an in-memory SQLite session for each test, with tables created and torn down. @pytest.fixture def db_session(): engine = create_engine("sqlite:///:memory:", echo=False) @@ -62,11 +65,11 @@ def db_session(): finally: session.close() Base.metadata.drop_all(bind=engine) -# #endregion db_session +# [/DEF:db_session:Fixture] -# #region test_create_dictionary [TYPE Function] -# @BRIEF Verify dictionary creation and read-back. +# [DEF:test_create_dictionary:Function] +# @PURPOSE: Verify dictionary creation and read-back. def test_create_dictionary(db_session: Session): d = DictionaryManager.create_dictionary( db_session, @@ -87,11 +90,11 @@ def test_create_dictionary(db_session: Session): fetched = DictionaryManager.get_dictionary(db_session, d.id) assert fetched.id == d.id assert fetched.name == "Finance Terms" -# #endregion test_create_dictionary +# [/DEF:test_create_dictionary:Function] -# #region test_update_dictionary [TYPE Function] -# @BRIEF Verify dictionary metadata update. +# [DEF:test_update_dictionary:Function] +# @PURPOSE: Verify dictionary metadata update. def test_update_dictionary(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Old Name", @@ -106,11 +109,11 @@ def test_update_dictionary(db_session: Session): assert updated.name == "New Name" assert updated.description == "Updated desc" assert updated.is_active is False -# #endregion test_update_dictionary +# [/DEF:test_update_dictionary:Function] -# #region test_delete_dictionary [TYPE Function] -# @BRIEF Verify dictionary deletion. +# [DEF:test_delete_dictionary:Function] +# @PURPOSE: Verify dictionary deletion. def test_delete_dictionary(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="To Delete", @@ -123,11 +126,11 @@ def test_delete_dictionary(db_session: Session): DictionaryManager.delete_dictionary(db_session, d.id) with pytest.raises(ValueError, match="Dictionary not found"): DictionaryManager.get_dictionary(db_session, d.id) -# #endregion test_delete_dictionary +# [/DEF:test_delete_dictionary:Function] -# #region test_list_dictionaries [TYPE Function] -# @BRIEF Verify paginated dictionary listing. +# [DEF:test_list_dictionaries:Function] +# @PURPOSE: Verify paginated dictionary listing. def test_list_dictionaries(db_session: Session): for i in range(5): DictionaryManager.create_dictionary( @@ -138,11 +141,11 @@ def test_list_dictionaries(db_session: Session): dicts, total = DictionaryManager.list_dictionaries(db_session, page=1, page_size=2) assert total == 5 assert len(dicts) == 2 -# #endregion test_list_dictionaries +# [/DEF:test_list_dictionaries:Function] -# #region test_add_entry_duplicate [TYPE Function] -# @BRIEF Verify duplicate entry raises ValueError and unique constraint is enforced. +# [DEF:test_add_entry_duplicate:Function] +# @PURPOSE: Verify duplicate entry raises ValueError and unique constraint is enforced. def test_add_entry_duplicate(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -157,11 +160,11 @@ def test_add_entry_duplicate(db_session: Session): # Different case, same normalized should also raise with pytest.raises(ValueError, match="already exists"): DictionaryManager.add_entry(db_session, d.id, "HELLO", "Ciao") -# #endregion test_add_entry_duplicate +# [/DEF:test_add_entry_duplicate:Function] -# #region test_add_entry_duplicate_per_dictionary [TYPE Function] -# @BRIEF Verify duplicate is per-dictionary (same term in different dictionaries is OK). +# [DEF:test_add_entry_duplicate_per_dictionary:Function] +# @PURPOSE: Verify duplicate is per-dictionary (same term in different dictionaries is OK). def test_add_entry_duplicate_per_dictionary(db_session: Session): d1 = DictionaryManager.create_dictionary( db_session, name="Dict1", @@ -175,11 +178,11 @@ def test_add_entry_duplicate_per_dictionary(db_session: Session): # Same term in different dictionary should work entry = DictionaryManager.add_entry(db_session, d2.id, "hello", "bonjour") assert entry.id is not None -# #endregion test_add_entry_duplicate_per_dictionary +# [/DEF:test_add_entry_duplicate_per_dictionary:Function] -# #region test_edit_entry [TYPE Function] -# @BRIEF Verify entry edit updates fields and enforces uniqueness. +# [DEF:test_edit_entry:Function] +# @PURPOSE: Verify entry edit updates fields and enforces uniqueness. def test_edit_entry(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -194,11 +197,11 @@ def test_edit_entry(db_session: Session): DictionaryManager.add_entry(db_session, d.id, "world", "mundo") with pytest.raises(ValueError, match="already exists"): DictionaryManager.edit_entry(db_session, entry.id, source_term="WORLD") -# #endregion test_edit_entry +# [/DEF:test_edit_entry:Function] -# #region test_delete_entry [TYPE Function] -# @BRIEF Verify entry deletion. +# [DEF:test_delete_entry:Function] +# @PURPOSE: Verify entry deletion. def test_delete_entry(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -209,11 +212,11 @@ def test_delete_entry(db_session: Session): entries, total = DictionaryManager.list_entries(db_session, d.id) assert total == 0 -# #endregion test_delete_entry +# [/DEF:test_delete_entry:Function] -# #region test_import_csv_overwrite [TYPE Function] -# @BRIEF Verify CSV import with overwrite conflict mode. +# [DEF:test_import_csv_overwrite:Function] +# @PURPOSE: Verify CSV import with overwrite conflict mode. def test_import_csv_overwrite(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -236,11 +239,11 @@ def test_import_csv_overwrite(db_session: Session): entries, _ = DictionaryManager.list_entries(db_session, d.id) hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0] assert hello_entry.target_term == "HELLO" -# #endregion test_import_csv_overwrite +# [/DEF:test_import_csv_overwrite:Function] -# #region test_import_csv_keep_existing [TYPE Function] -# @BRIEF Verify CSV import with keep_existing conflict mode. +# [DEF:test_import_csv_keep_existing:Function] +# @PURPOSE: Verify CSV import with keep_existing conflict mode. def test_import_csv_keep_existing(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -261,11 +264,11 @@ def test_import_csv_keep_existing(db_session: Session): entries, _ = DictionaryManager.list_entries(db_session, d.id) hello_entry = [e for e in entries if e.source_term_normalized == "hello"][0] assert hello_entry.target_term == "hola" -# #endregion test_import_csv_keep_existing +# [/DEF:test_import_csv_keep_existing:Function] -# #region test_import_csv_cancel_on_conflict [TYPE Function] -# @BRIEF Verify CSV import with cancel conflict mode raises errors. +# [DEF:test_import_csv_cancel_on_conflict:Function] +# @PURPOSE: Verify CSV import with cancel conflict mode raises errors. def test_import_csv_cancel_on_conflict(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -283,11 +286,11 @@ def test_import_csv_cancel_on_conflict(db_session: Session): assert result["updated"] == 0 assert len(result["errors"]) == 1 # hello conflict error assert "Conflict" in result["errors"][0]["error"] -# #endregion test_import_csv_cancel_on_conflict +# [/DEF:test_import_csv_cancel_on_conflict:Function] -# #region test_import_tsv [TYPE Function] -# @BRIEF Verify TSV import. +# [DEF:test_import_tsv:Function] +# @PURPOSE: Verify TSV import. def test_import_tsv(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -300,11 +303,11 @@ def test_import_tsv(db_session: Session): ) assert result["created"] == 2 assert result["total"] == 2 -# #endregion test_import_tsv +# [/DEF:test_import_tsv:Function] -# #region test_import_invalid_format [TYPE Function] -# @BRIEF Verify import raises ValueError for missing required columns. +# [DEF:test_import_invalid_format:Function] +# @PURPOSE: Verify import raises ValueError for missing required columns. def test_import_invalid_format(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -316,11 +319,11 @@ def test_import_invalid_format(db_session: Session): db_session, d.id, bad_content, delimiter=",", on_conflict="overwrite", ) -# #endregion test_import_invalid_format +# [/DEF:test_import_invalid_format:Function] -# #region test_import_empty_rows [TYPE Function] -# @BRIEF Verify import handles rows with missing fields gracefully. +# [DEF:test_import_empty_rows:Function] +# @PURPOSE: Verify import handles rows with missing fields gracefully. def test_import_empty_rows(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -333,11 +336,11 @@ def test_import_empty_rows(db_session: Session): ) assert result["created"] == 1 # hello assert len(result["errors"]) == 2 # empty source or target -# #endregion test_import_empty_rows +# [/DEF:test_import_empty_rows:Function] -# #region test_import_preview [TYPE Function] -# @BRIEF Verify import preview returns conflicts without mutating. +# [DEF:test_import_preview:Function] +# @PURPOSE: Verify import preview returns conflicts without mutating. def test_import_preview(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -361,11 +364,11 @@ def test_import_preview(db_session: Session): # Verify no mutation happened entries, total = DictionaryManager.list_entries(db_session, d.id) assert total == 1 # still only the original entry -# #endregion test_import_preview +# [/DEF:test_import_preview:Function] -# #region test_delete_dictionary_blocked_by_active_job [TYPE Function] -# @BRIEF Verify deletion is blocked when dictionary is attached to active/scheduled jobs. +# [DEF:test_delete_dictionary_blocked_by_active_job:Function] +# @PURPOSE: Verify deletion is blocked when dictionary is attached to active/scheduled jobs. def test_delete_dictionary_blocked_by_active_job(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -392,11 +395,11 @@ def test_delete_dictionary_blocked_by_active_job(db_session: Session): with pytest.raises(ValueError, match="active/scheduled"): DictionaryManager.delete_dictionary(db_session, d.id) -# #endregion test_delete_dictionary_blocked_by_active_job +# [/DEF:test_delete_dictionary_blocked_by_active_job:Function] -# #region test_delete_dictionary_allowed_with_completed_job [TYPE Function] -# @BRIEF Verify deletion is allowed when only completed/failed jobs reference the dictionary. +# [DEF:test_delete_dictionary_allowed_with_completed_job:Function] +# @PURPOSE: Verify deletion is allowed when only completed/failed jobs reference the dictionary. def test_delete_dictionary_allowed_with_completed_job(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", @@ -424,11 +427,11 @@ def test_delete_dictionary_allowed_with_completed_job(db_session: Session): DictionaryManager.delete_dictionary(db_session, d.id) with pytest.raises(ValueError, match="Dictionary not found"): DictionaryManager.get_dictionary(db_session, d.id) -# #endregion test_delete_dictionary_allowed_with_completed_job +# [/DEF:test_delete_dictionary_allowed_with_completed_job:Function] -# #region test_filter_for_batch_no_dictionaries [TYPE Function] -# @BRIEF Verify filter_for_batch returns empty when job has no dictionaries. +# [DEF:test_filter_for_batch_no_dictionaries:Function] +# @PURPOSE: Verify filter_for_batch returns empty when job has no dictionaries. def test_filter_for_batch_no_dictionaries(db_session: Session): job = TranslationJob( name="No Dict Job", @@ -440,11 +443,11 @@ def test_filter_for_batch_no_dictionaries(db_session: Session): result = DictionaryManager.filter_for_batch(db_session, ["hello world"], job.id) assert result == [] -# #endregion test_filter_for_batch_no_dictionaries +# [/DEF:test_filter_for_batch_no_dictionaries:Function] -# #region test_filter_for_batch_matches [TYPE Function] -# @BRIEF Verify filter_for_batch returns correct matched entries with word-boundary awareness. +# [DEF:test_filter_for_batch_matches:Function] +# @PURPOSE: Verify filter_for_batch returns correct matched entries with word-boundary awareness. def test_filter_for_batch_matches(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test Dict", @@ -486,11 +489,11 @@ def test_filter_for_batch_matches(db_session: Session): foo_matches = [m for m in result if m["source_term"] == "foo"] assert len(foo_matches) == 1 assert foo_matches[0]["text_index"] == 2 -# #endregion test_filter_for_batch_matches +# [/DEF:test_filter_for_batch_matches:Function] -# #region test_filter_for_batch_case_insensitive [TYPE Function] -# @BRIEF Verify filter_for_batch matching is case-insensitive. +# [DEF:test_filter_for_batch_case_insensitive:Function] +# @PURPOSE: Verify filter_for_batch matching is case-insensitive. def test_filter_for_batch_case_insensitive(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test Dict", @@ -517,11 +520,11 @@ def test_filter_for_batch_case_insensitive(db_session: Session): assert len(result) == 1 assert result[0]["source_term"] == "Hello World" assert result[0]["target_term"] == "Hola Mundo" -# #endregion test_filter_for_batch_case_insensitive +# [/DEF:test_filter_for_batch_case_insensitive:Function] -# #region test_filter_for_batch_word_boundary [TYPE Function] -# @BRIEF Verify filter_for_batch respects word boundaries (no substring matching within words). +# [DEF:test_filter_for_batch_word_boundary:Function] +# @PURPOSE: Verify filter_for_batch respects word boundaries (no substring matching within words). def test_filter_for_batch_word_boundary(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test Dict", @@ -554,11 +557,11 @@ def test_filter_for_batch_word_boundary(db_session: Session): job.id, ) assert len(result) == 1 -# #endregion test_filter_for_batch_word_boundary +# [/DEF:test_filter_for_batch_word_boundary:Function] -# #region test_filter_for_batch_multi_dictionary_priority [TYPE Function] -# @BRIEF Verify filter_for_batch respects dictionary link order priority. +# [DEF:test_filter_for_batch_multi_dictionary_priority:Function] +# @PURPOSE: Verify filter_for_batch respects dictionary link order priority. def test_filter_for_batch_multi_dictionary_priority(db_session: Session): d1 = DictionaryManager.create_dictionary( db_session, name="Priority1", source_dialect="a", target_dialect="b", @@ -593,11 +596,11 @@ def test_filter_for_batch_multi_dictionary_priority(db_session: Session): assert result[0]["target_term"] == "hola" assert result[1]["dictionary_id"] == d2.id assert result[1]["target_term"] == "bonjour" -# #endregion test_filter_for_batch_multi_dictionary_priority +# [/DEF:test_filter_for_batch_multi_dictionary_priority:Function] -# #region test_normalize_term [TYPE Function] -# @BRIEF Verify _normalize_term produces lowercase NFC-normalized strings. +# [DEF:test_normalize_term:Function] +# @PURPOSE: Verify _normalize_term produces lowercase NFC-normalized strings. def test_normalize_term(): assert _normalize_term("Hello") == "hello" assert _normalize_term("HELLO") == "hello" @@ -606,11 +609,11 @@ def test_normalize_term(): composed = "\u00C9" # É precomposed decomposed = "\u0045\u0301" # E + combining acute assert _normalize_term(composed) == _normalize_term(decomposed) -# #endregion test_normalize_term +# [/DEF:test_normalize_term:Function] -# #region test_detect_delimiter [TYPE Function] -# @BRIEF Verify _detect_delimiter correctly identifies CSV vs TSV. +# [DEF:test_detect_delimiter:Function] +# @PURPOSE: Verify _detect_delimiter correctly identifies CSV vs TSV. def test_detect_delimiter(): csv_content = "source_term,target_term,context_notes\nhello,hola," assert _detect_delimiter(csv_content) == "," @@ -620,11 +623,11 @@ def test_detect_delimiter(): empty_content = "" assert _detect_delimiter(empty_content) == "," # default fallback -# #endregion test_detect_delimiter +# [/DEF:test_detect_delimiter:Function] -# #region test_clear_entries [TYPE Function] -# @BRIEF Verify clearing all entries for a dictionary. +# [DEF:test_clear_entries:Function] +# @PURPOSE: Verify clearing all entries for a dictionary. def test_clear_entries(db_session: Session): d = DictionaryManager.create_dictionary( db_session, name="Test", source_dialect="a", target_dialect="b", @@ -637,5 +640,5 @@ def test_clear_entries(db_session: Session): entries, total = DictionaryManager.list_entries(db_session, d.id) assert total == 0 -# #endregion test_clear_entries -# #endregion DictionaryTests +# [/DEF:test_clear_entries:Function] +# [/DEF:DictionaryTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_orchestrator.py b/backend/src/plugins/translate/__tests__/test_orchestrator.py index 04a3dde7..cc52fc54 100644 --- a/backend/src/plugins/translate/__tests__/test_orchestrator.py +++ b/backend/src/plugins/translate/__tests__/test_orchestrator.py @@ -1,8 +1,10 @@ -# #region OrchestratorTests [C:3] [TYPE Module] [SEMANTICS test, translate, orchestrator, events] -# @BRIEF Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling. -# @LAYER Test -# @RELATION BINDS_TO -> [TranslationOrchestrator:Module] -# @RELATION BINDS_TO -> [TranslationEventLog:Module] +# [DEF:OrchestratorTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, orchestrator, events +# @PURPOSE: Tests for TranslationOrchestrator: run lifecycle, partial failure, batch retry, event invariants, NULL handling. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslationOrchestrator:Module] +# @RELATION: BINDS_TO -> [TranslationEventLog:Module] # @TEST_CONTRACT: TranslationOrchestrator -> start_run, execute_run, cancel_run, retry_failed_batches # @TEST_FIXTURE: mock_db -> MagicMock SQLAlchemy session # @TEST_FIXTURE: mock_config_manager -> MagicMock ConfigManager @@ -10,16 +12,14 @@ # @TEST_EDGE: invalid_run_status -> raises ValueError import pytest -from unittest.mock import MagicMock, patch, PropertyMock +from unittest.mock import MagicMock from datetime import datetime, timezone -from typing import Any, Dict, List, Optional from src.plugins.translate.orchestrator import TranslationOrchestrator from src.plugins.translate.events import TranslationEventLog from src.models.translate import ( TranslationJob, TranslationRun, - TranslationBatch, TranslationRecord, TranslationPreviewSession, ) @@ -39,11 +39,11 @@ def mock_job() -> MagicMock: job.source_datasource_id = "42" job.translation_column = "name" job.context_columns = ["category"] - job.source_table = "source_table_name" job.target_schema = "public" job.target_table = "translated_data" job.target_key_cols = ["id"] job.source_key_cols = ["id"] + job.source_table = "source_data" job.target_language = "en" job.provider_id = "provider-1" job.batch_size = 50 @@ -51,6 +51,9 @@ def mock_job() -> MagicMock: return job +# [/DEF:mock_job:Function] + + # [DEF:mock_preview_session:Function] # @PURPOSE: Create a mock accepted preview session. @pytest.fixture @@ -63,8 +66,11 @@ def mock_preview_session() -> MagicMock: return session -# #region TestTranslationEventLog [TYPE Class] -# @BRIEF Tests for TranslationEventLog terminal event invariant. +# [/DEF:mock_preview_session:Function] + + +# [DEF:TestTranslationEventLog:Class] +# @PURPOSE: Tests for TranslationEventLog terminal event invariant. class TestTranslationEventLog: # [DEF:test_log_event_creates_record:Function] @@ -140,11 +146,17 @@ class TestTranslationEventLog: log = TranslationEventLog(db) result = log.prune_expired(retention_days=90) assert result["pruned"] >= 0 -# #endregion TestTranslationEventLog + + # [/DEF:test_log_event_creates_record:Function] + # [/DEF:test_invalid_event_type_raises:Function] + # [/DEF:test_terminal_event_invariant:Function] + # [/DEF:test_run_started_must_precede_other_events:Function] + # [/DEF:test_prune_expired_creates_snapshot:Function] +# [/DEF:TestTranslationEventLog:Class] -# #region TestTranslationOrchestrator [TYPE Class] -# @BRIEF Tests for TranslationOrchestrator run lifecycle. +# [DEF:TestTranslationOrchestrator:Class] +# @PURPOSE: Tests for TranslationOrchestrator run lifecycle. class TestTranslationOrchestrator: # [DEF:test_start_run_success:Function] @@ -233,7 +245,8 @@ class TestTranslationOrchestrator: run.job_id = "job-123" run.status = "RUNNING" - db.query.return_value.filter.return_value.first.return_value = run + # Chain: first query returns run, event-log queries return None (no existing terminal events) + db.query.return_value.filter.return_value.first.side_effect = [run, None] orch = TranslationOrchestrator(db, config_manager, "test-user") result = orch.cancel_run("run-1") @@ -370,5 +383,16 @@ class TestTranslationOrchestrator: assert total == 1 assert len(runs) == 1 assert runs[0]["id"] == "run-1" -# #endregion TestTranslationOrchestrator -# #endregion OrchestratorTests + + # [/DEF:test_start_run_success:Function] + # [/DEF:test_start_run_missing_preview_raises:Function] + # [/DEF:test_start_run_draft_job_raises:Function] + # [/DEF:test_execute_run_invalid_status:Function] + # [/DEF:test_cancel_run:Function] + # [/DEF:test_cancel_run_invalid_status:Function] + # [/DEF:test_retry_failed_batches_no_failures:Function] + # [/DEF:test_get_run_status:Function] + # [/DEF:test_get_run_records:Function] + # [/DEF:test_get_run_history:Function] +# [/DEF:TestTranslationOrchestrator:Class] +# [/DEF:OrchestratorTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_preview.py b/backend/src/plugins/translate/__tests__/test_preview.py index 09daf6a9..e95e6b73 100644 --- a/backend/src/plugins/translate/__tests__/test_preview.py +++ b/backend/src/plugins/translate/__tests__/test_preview.py @@ -1,7 +1,9 @@ -# #region TranslationPreviewTests [C:3] [TYPE Module] [SEMANTICS test, translate, preview, session] -# @BRIEF Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate. -# @LAYER Test -# @RELATION BINDS_TO -> [TranslationPreview:Module] +# [DEF:TranslationPreviewTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, preview, session +# @PURPOSE: Test TranslationPreview service: preview, approve/edit/reject state transitions, cost estimation, acceptance gate. +# @LAYER: Test +# @RELATION: BINDS_TO -> [TranslationPreview:Module] from unittest.mock import MagicMock, patch, PropertyMock from datetime import datetime, timezone, timedelta @@ -17,8 +19,8 @@ from src.models.translate import ( from src.plugins.translate.preview import TranslationPreview, TokenEstimator -# #region _make_mock_job [TYPE Function] -# @BRIEF Create a mock TranslationJob with test config. +# [DEF:_make_mock_job:Function] +# @PURPOSE: Create a mock TranslationJob with test config. def _make_mock_job(**overrides): job = MagicMock(spec=TranslationJob) job.id = overrides.get("id", "job-123") @@ -35,11 +37,11 @@ def _make_mock_job(**overrides): job.upsert_strategy = overrides.get("upsert_strategy", "MERGE") job.status = overrides.get("status", "DRAFT") return job -# #endregion _make_mock_job +# [/DEF:_make_mock_job:Function] -# #region _make_mock_provider [TYPE Function] -# @BRIEF Create a mock LLMProvider. +# [DEF:_make_mock_provider:Function] +# @PURPOSE: Create a mock LLMProvider. def _make_mock_provider(**overrides): provider = MagicMock() provider.id = overrides.get("id", "provider-1") @@ -48,15 +50,15 @@ def _make_mock_provider(**overrides): provider.default_model = overrides.get("default_model", "gpt-4o-mini") provider.api_key = "encrypted-key-123" return provider -# #endregion _make_mock_provider +# [/DEF:_make_mock_provider:Function] -# #region TestTranslationPreview [TYPE Class] -# @BRIEF Test suite for TranslationPreview service. +# [DEF:TestTranslationPreview:Class] +# @PURPOSE: Test suite for TranslationPreview service. class TestTranslationPreview: - # #region test_preview_valid_job [TYPE Function] - # @BRIEF Verify preview creates session and records successfully. + # [DEF:test_preview_valid_job:Function] + # @PURPOSE: Verify preview creates session and records successfully. def test_preview_valid_job(self): """Preview with valid job should create session and return records.""" db = MagicMock() @@ -159,10 +161,10 @@ class TestTranslationPreview: # Verify session was added to DB assert db.add.called assert db.commit.called - # #endregion test_preview_valid_job + # [/DEF:test_preview_valid_job:Function] - # #region test_preview_with_dictionary [TYPE Function] - # @BRIEF Verify glossary entries from dictionary are included in LLM prompt. + # [DEF:test_preview_with_dictionary:Function] + # @PURPOSE: Verify glossary entries from dictionary are included in LLM prompt. def test_preview_with_dictionary(self): """Preview should include dictionary glossary in the prompt sent to LLM.""" db = MagicMock() @@ -218,10 +220,10 @@ class TestTranslationPreview: # Verify LLM was called mock_llm.assert_called_once() assert len(result["records"]) == 1 - # #endregion test_preview_with_dictionary + # [/DEF:test_preview_with_dictionary:Function] - # #region test_preview_row_approve [TYPE Function] - # @BRIEF Verify approving a preview row changes its status. + # [DEF:test_preview_row_approve:Function] + # @PURPOSE: Verify approving a preview row changes its status. def test_preview_row_approve(self): """Approve action should set record status to APPROVED.""" db = MagicMock() @@ -257,10 +259,10 @@ class TestTranslationPreview: assert record.status == "APPROVED" assert result["status"] == "APPROVED" assert db.commit.called - # #endregion test_preview_row_approve + # [/DEF:test_preview_row_approve:Function] - # #region test_preview_row_reject [TYPE Function] - # @BRIEF Verify rejecting a preview row changes its status. + # [DEF:test_preview_row_reject:Function] + # @PURPOSE: Verify rejecting a preview row changes its status. def test_preview_row_reject(self): """Reject action should set record status to REJECTED.""" db = MagicMock() @@ -290,10 +292,10 @@ class TestTranslationPreview: assert record.status == "REJECTED" assert result["status"] == "REJECTED" - # #endregion test_preview_row_reject + # [/DEF:test_preview_row_reject:Function] - # #region test_preview_row_edit [TYPE Function] - # @BRIEF Verify editing a preview row updates its translation and status. + # [DEF:test_preview_row_edit:Function] + # @PURPOSE: Verify editing a preview row updates its translation and status. def test_preview_row_edit(self): """Edit action should update translation and set status to APPROVED.""" db = MagicMock() @@ -326,10 +328,10 @@ class TestTranslationPreview: assert record.status == "APPROVED" assert result["status"] == "APPROVED" assert result["target_sql"] == "edited translation" - # #endregion test_preview_row_edit + # [/DEF:test_preview_row_edit:Function] - # #region test_preview_row_invalid_action [TYPE Function] - # @BRIEF Verify invalid action raises ValueError. + # [DEF:test_preview_row_invalid_action:Function] + # @PURPOSE: Verify invalid action raises ValueError. def test_preview_row_invalid_action(self): """Invalid action should raise ValueError.""" db = MagicMock() @@ -355,10 +357,10 @@ class TestTranslationPreview: row_id="record-1", action="invalid_action", ) - # #endregion test_preview_row_invalid_action + # [/DEF:test_preview_row_invalid_action:Function] - # #region test_preview_accept_session [TYPE Function] - # @BRIEF Verify accepting a preview session gates execution. + # [DEF:test_preview_accept_session:Function] + # @PURPOSE: Verify accepting a preview session gates execution. def test_preview_accept_session(self): """Accept should set session status to APPLIED and return records.""" db = MagicMock() @@ -393,10 +395,10 @@ class TestTranslationPreview: assert len(result["records"]) == 1 assert result["records"][0]["status"] == "APPROVED" assert db.commit.called - # #endregion test_preview_accept_session + # [/DEF:test_preview_accept_session:Function] - # #region test_preview_no_active_session [TYPE Function] - # @BRIEF Verify accept raises error when no active session. + # [DEF:test_preview_no_active_session:Function] + # @PURPOSE: Verify accept raises error when no active session. def test_preview_no_active_session(self): """Accept without active session should raise ValueError.""" db = MagicMock() @@ -407,10 +409,10 @@ class TestTranslationPreview: preview_service = TranslationPreview(db, config_manager, "test-user") with pytest.raises(ValueError, match="No active preview session"): preview_service.accept_preview_session(job_id="job-123") - # #endregion test_preview_no_active_session + # [/DEF:test_preview_no_active_session:Function] - # #region test_cost_estimation [TYPE Function] - # @BRIEF Verify token and cost estimation methods. + # [DEF:test_cost_estimation:Function] + # @PURPOSE: Verify token and cost estimation methods. def test_cost_estimation(self): """Token and cost estimation should return reasonable values.""" prompt = "Translate this text from English to Russian. " * 100 @@ -426,10 +428,10 @@ class TestTranslationPreview: cost_default = TokenEstimator.estimate_cost(1000) assert cost_default == 0.002 - # #endregion test_cost_estimation + # [/DEF:test_cost_estimation:Function] - # #region test_preview_parse_llm_response [TYPE Function] - # @BRIEF Verify LLM JSON response parsing. + # [DEF:test_preview_parse_llm_response:Function] + # @PURPOSE: Verify LLM JSON response parsing. def test_preview_parse_llm_response(self): """Parse LLM response should extract translations keyed by row_id.""" response = json.dumps({ @@ -440,19 +442,19 @@ class TestTranslationPreview: }) result = TranslationPreview._parse_llm_response(response, 2) assert result == {"0": "Привет", "1": "Мир"} - # #endregion test_preview_parse_llm_response + # [/DEF:test_preview_parse_llm_response:Function] - # #region test_preview_parse_llm_response_with_code_block [TYPE Function] - # @BRIEF Verify LLM response parsing handles markdown code blocks. + # [DEF:test_preview_parse_llm_response_with_code_block:Function] + # @PURPOSE: Verify LLM response parsing handles markdown code blocks. def test_preview_parse_llm_response_with_code_block(self): """Parse LLM response should handle markdown code block wrapping.""" response = "```json\n{\n \"rows\": [\n {\"row_id\": \"0\", \"translation\": \"Test\"}\n ]\n}\n```" result = TranslationPreview._parse_llm_response(response, 1) assert result == {"0": "Test"} - # #endregion test_preview_parse_llm_response_with_code_block + # [/DEF:test_preview_parse_llm_response_with_code_block:Function] - # #region test_preview_compute_config_hash [TYPE Function] - # @BRIEF Verify config hash computation is deterministic. + # [DEF:test_preview_compute_config_hash:Function] + # @PURPOSE: Verify config hash computation is deterministic. def test_preview_compute_config_hash(self): """Config hash should be deterministic and non-empty.""" job = _make_mock_job() @@ -460,10 +462,10 @@ class TestTranslationPreview: hash2 = TranslationPreview._compute_config_hash(job) assert hash1 == hash2 assert len(hash1) == 16 - # #endregion test_preview_compute_config_hash + # [/DEF:test_preview_compute_config_hash:Function] - # #region test_preview_missing_datasource [TYPE Function] - # @BRIEF Verify error when job has no datasource configured. + # [DEF:test_preview_missing_datasource:Function] + # @PURPOSE: Verify error when job has no datasource configured. def test_preview_missing_datasource(self): """Preview should fail if job has no datasource.""" db = MagicMock() @@ -475,10 +477,10 @@ class TestTranslationPreview: preview_service = TranslationPreview(db, config_manager, "test-user") with pytest.raises(ValueError, match="source datasource"): preview_service.preview_rows(job_id="job-123") - # #endregion test_preview_missing_datasource + # [/DEF:test_preview_missing_datasource:Function] - # #region test_preview_missing_translation_column [TYPE Function] - # @BRIEF Verify error when job has no translation column. + # [DEF:test_preview_missing_translation_column:Function] + # @PURPOSE: Verify error when job has no translation column. def test_preview_missing_translation_column(self): """Preview should fail if job has no translation column.""" db = MagicMock() @@ -490,9 +492,9 @@ class TestTranslationPreview: preview_service = TranslationPreview(db, config_manager, "test-user") with pytest.raises(ValueError, match="translation column"): preview_service.preview_rows(job_id="job-123") - # #endregion test_preview_missing_translation_column + # [/DEF:test_preview_missing_translation_column:Function] -# #endregion TestTranslationPreview +# [/DEF:TestTranslationPreview:Class] -# #endregion TranslationPreviewTests +# [/DEF:TranslationPreviewTests:Module] diff --git a/backend/src/plugins/translate/__tests__/test_sql_generator.py b/backend/src/plugins/translate/__tests__/test_sql_generator.py index 7ed9e58b..a175ef3f 100644 --- a/backend/src/plugins/translate/__tests__/test_sql_generator.py +++ b/backend/src/plugins/translate/__tests__/test_sql_generator.py @@ -1,7 +1,9 @@ -# #region SQLGeneratorTests [C:3] [TYPE Module] [SEMANTICS test, translate, sql_generator] -# @BRIEF Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety. -# @LAYER Test -# @RELATION BINDS_TO -> [SQLGenerator:Module] +# [DEF:SQLGeneratorTests:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: test, translate, sql_generator +# @PURPOSE: Tests for SQLGenerator: PostgreSQL INSERT + UPSERT, ClickHouse INSERT, dialect quoting, NULL handling, injection safety. +# @LAYER: Test +# @RELATION: BINDS_TO -> [SQLGenerator:Module] # @TEST_CONTRACT: SQLGenerator.generate -> SQL string, row_count # @TEST_FIXTURE: sample_rows -> [{"col1": "val1", "col2": 42}, {"col1": "val2", "col2": 99}] # @TEST_EDGE: empty_rows -> raises ValueError @@ -9,14 +11,12 @@ # @TEST_EDGE: sql_injection -> values with single quotes are escaped import pytest -from datetime import datetime, timezone from typing import Any, Dict, List from src.plugins.translate.sql_generator import ( SQLGenerator, _quote_identifier, _encode_sql_value, - _normalize_timestamp_value, generate_insert_sql, generate_upsert_sql, ) @@ -28,6 +28,9 @@ def _normalize_sql(sql: str) -> str: return " ".join(sql.split()) +# [/DEF:_normalize_sql:Function] + + # [DEF:sample_rows:Function] # @PURPOSE: Sample rows fixture for SQL generation tests. @pytest.fixture @@ -38,8 +41,11 @@ def sample_rows() -> List[Dict[str, Any]]: ] -# #region TestQuoteIdentifier [TYPE Class] -# @BRIEF Tests for identifier quoting per dialect. +# [/DEF:sample_rows:Function] + + +# [DEF:TestQuoteIdentifier:Class] +# @PURPOSE: Tests for identifier quoting per dialect. class TestQuoteIdentifier: # [DEF:test_quote_postgresql:Function] # @PURPOSE: PostgreSQL uses double quotes. @@ -62,11 +68,16 @@ class TestQuoteIdentifier: def test_quote_schema_table(self) -> None: assert _quote_identifier("my_schema", "postgresql") == '"my_schema"' assert _quote_identifier("my_table", "postgresql") == '"my_table"' -# #endregion TestQuoteIdentifier + + # [/DEF:test_quote_postgresql:Function] + # [/DEF:test_quote_clickhouse:Function] + # [/DEF:test_quote_with_existing_quotes:Function] + # [/DEF:test_quote_schema_table:Function] +# [/DEF:TestQuoteIdentifier:Class] -# #region TestEncodeSqlValue [TYPE Class] -# @BRIEF Tests for SQL value encoding. +# [DEF:TestEncodeSqlValue:Class] +# @PURPOSE: Tests for SQL value encoding. class TestEncodeSqlValue: # [DEF:test_null:Function] # @PURPOSE: None encodes as NULL. @@ -104,71 +115,18 @@ class TestEncodeSqlValue: def test_special_chars(self) -> None: assert _encode_sql_value("line1\nline2") == "'line1\nline2'" - # [DEF:test_clickhouse_timestamp_millis:Function] - # @PURPOSE: ClickHouse dialect converts Unix timestamp millis to 'YYYY-MM-DD'. - def test_clickhouse_timestamp_millis(self) -> None: - # 1726358400000 millis = 2024-09-15 - result = _encode_sql_value("1726358400000.0", dialect="clickhouse") - assert result == "'2024-09-15'" - - # [DEF:test_clickhouse_timestamp_seconds:Function] - # @PURPOSE: ClickHouse dialect converts Unix timestamp seconds to 'YYYY-MM-DD'. - def test_clickhouse_timestamp_seconds(self) -> None: - # 1726358400 seconds = 2024-09-15 - result = _encode_sql_value("1726358400", dialect="clickhouse") - assert result == "'2024-09-15'" - - # [DEF:test_clickhouse_non_timestamp_string:Function] - # @PURPOSE: Non-timestamp strings are not affected by ClickHouse normalization. - def test_clickhouse_non_timestamp_string(self) -> None: - result = _encode_sql_value("hello world", dialect="clickhouse") - assert result == "'hello world'" - - # [DEF:test_postgresql_timestamp_not_normalized:Function] - # @PURPOSE: PostgreSQL dialect does NOT normalize timestamps (different handling). - def test_postgresql_timestamp_not_normalized(self) -> None: - result = _encode_sql_value("1726358400000.0", dialect="postgresql") - assert result == "'1726358400000.0'" -# #endregion TestEncodeSqlValue + # [/DEF:test_null:Function] + # [/DEF:test_string:Function] + # [/DEF:test_integer:Function] + # [/DEF:test_float:Function] + # [/DEF:test_boolean:Function] + # [/DEF:test_sql_injection:Function] + # [/DEF:test_special_chars:Function] +# [/DEF:TestEncodeSqlValue:Class] -# #region TestNormalizeTimestampValue [TYPE Class] -# @BRIEF Tests for Unix timestamp detection and conversion. -class TestNormalizeTimestampValue: - # [DEF:test_millis_timestamp:Function] - # @PURPOSE: Millisecond Unix timestamps convert to YYYY-MM-DD. - def test_millis_timestamp(self) -> None: - # 1726358400000 = 2024-09-15 00:00:00 UTC - assert _normalize_timestamp_value("1726358400000.0") == "2024-09-15" - assert _normalize_timestamp_value("1726358400000") == "2024-09-15" - - # [DEF:test_seconds_timestamp:Function] - # @PURPOSE: Second-precision Unix timestamps convert to YYYY-MM-DD. - def test_seconds_timestamp(self) -> None: - assert _normalize_timestamp_value("1726358400") == "2024-09-15" - - # [DEF:test_non_timestamp_values:Function] - # @PURPOSE: Regular strings and small numbers return None. - def test_non_timestamp_values(self) -> None: - assert _normalize_timestamp_value("hello") is None - assert _normalize_timestamp_value("12345") is None - assert _normalize_timestamp_value("") is None - assert _normalize_timestamp_value(None) is None - - # [DEF:test_float_millis:Function] - # @PURPOSE: Float millis timestamp converts correctly. - def test_float_millis(self) -> None: - assert _normalize_timestamp_value(1726358400000.0) == "2024-09-15" - - # [DEF:test_int_seconds:Function] - # @PURPOSE: Integer seconds timestamp converts correctly. - def test_int_seconds(self) -> None: - assert _normalize_timestamp_value(1726358400) == "2024-09-15" -# #endregion TestNormalizeTimestampValue - - -# #region TestGenerateInsertSql [TYPE Class] -# @BRIEF Tests for plain INSERT SQL generation. +# [DEF:TestGenerateInsertSql:Class] +# @PURPOSE: Tests for plain INSERT SQL generation. class TestGenerateInsertSql: # [DEF:test_basic_insert:Function] # @PURPOSE: Generates basic INSERT with VALUES. @@ -246,11 +204,18 @@ class TestGenerateInsertSql: # The single quote should be doubled, preventing injection assert "'';" in sql or "''" in sql assert "DROP TABLE" in sql # It's literal data, not a command -# #endregion TestGenerateInsertSql + + # [/DEF:test_basic_insert:Function] + # [/DEF:test_insert_with_schema:Function] + # [/DEF:test_empty_columns_raises:Function] + # [/DEF:test_empty_rows_raises:Function] + # [/DEF:test_null_handling:Function] + # [/DEF:test_injection_safety:Function] +# [/DEF:TestGenerateInsertSql:Class] -# #region TestGenerateUpsertSql [TYPE Class] -# @BRIEF Tests for PostgreSQL UPSERT SQL generation. +# [DEF:TestGenerateUpsertSql:Class] +# @PURPOSE: Tests for PostgreSQL UPSERT SQL generation. class TestGenerateUpsertSql: # [DEF:test_basic_upsert:Function] # @PURPOSE: Generates INSERT ... ON CONFLICT DO UPDATE. @@ -292,11 +257,15 @@ class TestGenerateUpsertSql: key_columns=[], rows=sample_rows, ) -# #endregion TestGenerateUpsertSql + + # [/DEF:test_basic_upsert:Function] + # [/DEF:test_upsert_all_keys:Function] + # [/DEF:test_upsert_empty_keys:Function] +# [/DEF:TestGenerateUpsertSql:Class] -# #region TestSQLGenerator [TYPE Class] -# @BRIEF Tests for the full SQLGenerator class. +# [DEF:TestSQLGenerator:Class] +# @PURPOSE: Tests for the full SQLGenerator class. class TestSQLGenerator: # [DEF:test_postgresql_insert:Function] # @PURPOSE: PostgreSQL dialect generates proper INSERT. @@ -388,5 +357,12 @@ class TestSQLGenerator: assert len(statements) > 1 # Should be split into multiple statements total_count = sum(count for _, count in statements) assert total_count == 10 -# #endregion TestSQLGenerator -# #endregion SQLGeneratorTests + + # [/DEF:test_postgresql_insert:Function] + # [/DEF:test_postgresql_upsert:Function] + # [/DEF:test_clickhouse_insert:Function] + # [/DEF:test_clickhouse_upsert_fallback:Function] + # [/DEF:test_empty_rows_via_generator:Function] + # [/DEF:test_generate_batch:Function] +# [/DEF:TestSQLGenerator:Class] +# [/DEF:SQLGeneratorTests:Module] diff --git a/backend/src/plugins/translate/_utils.py b/backend/src/plugins/translate/_utils.py index 0a86825b..bac016c9 100644 --- a/backend/src/plugins/translate/_utils.py +++ b/backend/src/plugins/translate/_utils.py @@ -1,12 +1,17 @@ -# #region DictionaryUtils [C:1] [TYPE Module] -# #endregion DictionaryUtils +# #region DictionaryUtils [C:1] [TYPE Module] [SEMANTICS dictionary, utils, normalize, delimiter] +# @BRIEF Utility helpers for dictionary management (term normalization and delimiter detection). +# @LAYER: Domain import re import unicodedata -# #region _normalize_term [C:2] [TYPE Function] [SEMANTICS dictionary,normalize] -# @BRIEF Normalize a term for case-insensitive unique constraint lookup using NFC normalization. +# #region _normalize_term [TYPE Function] +# @BRIEF Normalize a term for case-insensitive unique constraint lookup. +# @RATIONALE: NFC normalization is applied before lowercasing to ensure consistent +# comparison of Unicode characters (e.g. precomposed vs decomposed forms). +# @REJECTED: Lowercasing without NFC normalization — would cause duplicate entries +# for semantically identical Unicode strings in different normalization forms. def _normalize_term(term: str) -> str: """Normalize a term by NFC, lowercasing, and removing extra whitespace.""" if not term: @@ -16,7 +21,8 @@ def _normalize_term(term: str) -> str: # #endregion _normalize_term -# #region _detect_delimiter [C:1] [TYPE Function] [SEMANTICS dictionary,delimiter] +# #region _detect_delimiter [TYPE Function] +# @BRIEF Detect the delimiter used in a CSV/TSV header line. def _detect_delimiter(header_line: str) -> str: """Detect delimiter by counting tabs vs commas in the first line.""" if not header_line: @@ -25,3 +31,4 @@ def _detect_delimiter(header_line: str) -> str: comma_count = header_line.count(",") return "\t" if tab_count > comma_count else "," # #endregion _detect_delimiter +# #endregion DictionaryUtils diff --git a/backend/src/plugins/translate/dictionary.py b/backend/src/plugins/translate/dictionary.py index fc779981..c45bf934 100644 --- a/backend/src/plugins/translate/dictionary.py +++ b/backend/src/plugins/translate/dictionary.py @@ -1,15 +1,13 @@ -# #region DictionaryManagerModule [C:5] [TYPE Module] [SEMANTICS dictionary,manager,terminology,crud,import,filter] +# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS dictionary, manager, terminology, crud, import, filter] # @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [TerminologyDictionary] -# @RELATION DEPENDS_ON -> [DictionaryEntry] -# @RELATION DEPENDS_ON -> [TranslationJobDictionary] -# @RELATION DEPENDS_ON -> [TranslationJob] -# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary] -# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) prohibits duplicate entries within a dictionary. -# @RATIONALE C5 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. -# @REJECTED Pure C3 CRUD without state guards would allow orphaned job-dictionary links. -# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class] +# @RELATION DEPENDS_ON -> [DictionaryEntry:Class] +# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class] +# @RELATION DEPENDS_ON -> [TranslationJob:Class] +# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term. +# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links. +# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing. import csv import io @@ -17,8 +15,7 @@ import re from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session from sqlalchemy import func -from ...core.cot_logger import MarkerLogger -from ...core.logger import belief_scope +from ...core.logger import logger, belief_scope from ...models.translate import ( TerminologyDictionary, DictionaryEntry, @@ -27,20 +24,18 @@ from ...models.translate import ( ) from ._utils import _normalize_term, _detect_delimiter -log = MarkerLogger("DictionaryManager") - -# #region DictionaryManager [C:5] [TYPE Class] [SEMANTICS dictionary,manager,crud,import] -# @BRIEF Manages terminology dictionaries and their entries with referential integrity, import, and batch filtering. -# @PRE Database session is open and valid. -# @POST Dictionary and entry mutations are persisted with conflict detection. -# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. -# @DATA_CONTRACT Input[name, source_dialect, target_dialect] -> Output[TerminologyDictionary] -# @INVARIANT UniqueConstraint(dictionary_id, source_term_normalized) enforces no duplicates within a dictionary. +# #region DictionaryManager [C:4] [TYPE Class] +# @BRIEF Manages terminology dictionaries and their entries with referential integrity. +# @PRE: Database session is open and valid. +# @POST: Dictionary and entry mutations are persisted with conflict detection. +# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards. class DictionaryManager: - # #region DictionaryManager.create_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,create] - # @BRIEF Create a new terminology dictionary with name, source_dialect, and target_dialect. - # @RELATION DEPENDS_ON -> [TerminologyDictionary] + # [DEF:DictionaryManager.create_dictionary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Create a new terminology dictionary. + # @PRE: payload contains name, source_dialect, target_dialect. + # @POST: New TerminologyDictionary row is created and returned. @staticmethod def create_dictionary( db: Session, name: str, source_dialect: str, target_dialect: str, @@ -48,7 +43,7 @@ class DictionaryManager: is_active: bool = True, ) -> TerminologyDictionary: with belief_scope("DictionaryManager.create_dictionary"): - log.reason("Creating dictionary", payload={"name": name, "source": source_dialect, "target": target_dialect}) + logger.reason("Creating dictionary", {"name": name, "source": source_dialect, "target": target_dialect}) dictionary = TerminologyDictionary( name=name, description=description, @@ -60,13 +55,15 @@ class DictionaryManager: db.add(dictionary) db.commit() db.refresh(dictionary) - log.reflect("Dictionary created", payload={"id": dictionary.id}) + logger.reflect("Dictionary created", {"id": dictionary.id}) return dictionary - # #endregion DictionaryManager.create_dictionary + # [/DEF:DictionaryManager.create_dictionary:Function] - # #region DictionaryManager.update_dictionary [C:3] [TYPE Function] [SEMANTICS dictionary,update] - # @BRIEF Update an existing terminology dictionary's metadata fields. - # @RELATION DEPENDS_ON -> [TerminologyDictionary] + # [DEF:DictionaryManager.update_dictionary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Update an existing terminology dictionary. + # @PRE: dict_id exists in terminology_dictionaries table. + # @POST: Dictionary metadata is updated and returned. @staticmethod def update_dictionary( db: Session, dict_id: str, name: Optional[str] = None, @@ -77,7 +74,7 @@ class DictionaryManager: dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") - log.reason("Updating dictionary", payload={"id": dict_id}) + logger.reason("Updating dictionary", {"id": dict_id}) if name is not None: dictionary.name = name if description is not None: @@ -90,15 +87,16 @@ class DictionaryManager: dictionary.is_active = is_active db.commit() db.refresh(dictionary) - log.reflect("Dictionary updated", payload={"id": dictionary.id}) + logger.reflect("Dictionary updated", {"id": dictionary.id}) return dictionary - # #endregion DictionaryManager.update_dictionary + # [/DEF:DictionaryManager.update_dictionary:Function] - # #region DictionaryManager.delete_dictionary [C:4] [TYPE Function] [SEMANTICS dictionary,delete] - # @BRIEF Delete a dictionary, blocked if attached to active/scheduled jobs (referential integrity guard). - # @PRE dict_id exists. - # @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. - # @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows; checks job attachments first. + # [DEF:DictionaryManager.delete_dictionary:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs. + # @PRE: dict_id exists. + # @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs. + # @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows. @staticmethod def delete_dictionary(db: Session, dict_id: str) -> None: with belief_scope("DictionaryManager.delete_dictionary"): @@ -122,33 +120,39 @@ class DictionaryManager: ) if attached_jobs > 0: - log.explore("Delete blocked: dictionary attached to active jobs", payload={"dict_id": dict_id, "jobs": attached_jobs}, error=f"Dictionary attached to {attached_jobs} active job(s)") + logger.explore("Delete blocked: dictionary attached to active jobs", {"dict_id": dict_id, "jobs": attached_jobs}) raise ValueError( f"Cannot delete dictionary attached to {attached_jobs} active/scheduled job(s). " "Remove dictionary associations from jobs first." ) - log.reason("Deleting dictionary", payload={"id": dict_id}) + logger.reason("Deleting dictionary", {"id": dict_id}) # Delete entries and job-dictionary links first db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() db.query(TranslationJobDictionary).filter(TranslationJobDictionary.dictionary_id == dict_id).delete() db.delete(dictionary) db.commit() - log.reflect("Dictionary deleted", payload={"id": dict_id}) - # #endregion DictionaryManager.delete_dictionary + logger.reflect("Dictionary deleted", {"id": dict_id}) + # [/DEF:DictionaryManager.delete_dictionary:Function] - # #region DictionaryManager.get_dictionary [C:2] [TYPE Function] [SEMANTICS dictionary,get] - # @BRIEF Get a single dictionary by ID with entry count. + # [DEF:DictionaryManager.get_dictionary:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Get a single dictionary by ID with entry count. + # @PRE: dict_id exists. + # @POST: Returns dict with dictionary + entry_count or raises ValueError. @staticmethod def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary: dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") return dictionary - # #endregion DictionaryManager.get_dictionary + # [/DEF:DictionaryManager.get_dictionary:Function] - # #region DictionaryManager.list_dictionaries [C:2] [TYPE Function] [SEMANTICS dictionary,list] - # @BRIEF List dictionaries with pagination and total count. + # [DEF:DictionaryManager.list_dictionaries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List dictionaries with pagination and entry counts. + # @PRE: page >= 1, page_size between 1 and 100. + # @POST: Returns (list of dicts, total_count). @staticmethod def list_dictionaries( db: Session, page: int = 1, page_size: int = 20, @@ -162,11 +166,13 @@ class DictionaryManager: .all() ) return dictionaries, total - # #endregion DictionaryManager.list_dictionaries + # [/DEF:DictionaryManager.list_dictionaries:Function] - # #region DictionaryManager.add_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,add] - # @BRIEF Add an entry to a dictionary, enforcing unique source_term_normalized per dictionary. - # @RELATION DEPENDS_ON -> [DictionaryEntry] + # [DEF:DictionaryManager.add_entry:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Add an entry to a dictionary, enforcing unique source_term_normalized. + # @PRE: dict_id exists. source_term and target_term are non-empty. + # @POST: New DictionaryEntry row is created or raises on duplicate. @staticmethod def add_entry( db: Session, dict_id: str, source_term: str, target_term: str, @@ -188,7 +194,7 @@ class DictionaryManager: f"(id={existing.id}). Use overwrite or keep_existing conflict mode." ) - log.reason("Adding dictionary entry", payload={"dict_id": dict_id, "term": source_term}) + logger.reason("Adding dictionary entry", {"dict_id": dict_id, "term": source_term}) entry = DictionaryEntry( dictionary_id=dict_id, source_term=source_term.strip(), @@ -199,13 +205,15 @@ class DictionaryManager: db.add(entry) db.commit() db.refresh(entry) - log.reflect("Entry added", payload={"entry_id": entry.id}) + logger.reflect("Entry added", {"entry_id": entry.id}) return entry - # #endregion DictionaryManager.add_entry + # [/DEF:DictionaryManager.add_entry:Function] - # #region DictionaryManager.edit_entry [C:3] [TYPE Function] [SEMANTICS dictionary,entry,edit] - # @BRIEF Edit an existing dictionary entry with duplicate-aware normalization check. - # @RELATION DEPENDS_ON -> [DictionaryEntry] + # [DEF:DictionaryManager.edit_entry:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Edit an existing dictionary entry with duplicate-aware normalization. + # @PRE: entry_id exists. + # @POST: Entry fields are updated. @staticmethod def edit_entry( db: Session, entry_id: str, source_term: Optional[str] = None, @@ -216,7 +224,7 @@ class DictionaryManager: if not entry: raise ValueError(f"Entry not found: {entry_id}") - log.reason("Editing dictionary entry", payload={"entry_id": entry_id}) + logger.reason("Editing dictionary entry", {"entry_id": entry_id}) if source_term is not None: normalized = _normalize_term(source_term) # Check uniqueness within the same dictionary @@ -242,41 +250,50 @@ class DictionaryManager: entry.context_notes = context_notes.strip() if context_notes else None db.commit() db.refresh(entry) - log.reflect("Entry updated", payload={"entry_id": entry.id}) + logger.reflect("Entry updated", {"entry_id": entry.id}) return entry - # #endregion DictionaryManager.edit_entry + # [/DEF:DictionaryManager.edit_entry:Function] - # #region DictionaryManager.delete_entry [C:2] [TYPE Function] [SEMANTICS dictionary,entry,delete] - # @BRIEF Delete a single dictionary entry. + # [DEF:DictionaryManager.delete_entry:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Delete a single dictionary entry. + # @PRE: entry_id exists. + # @POST: Entry is deleted. @staticmethod def delete_entry(db: Session, entry_id: str) -> None: with belief_scope("DictionaryManager.delete_entry"): entry = db.query(DictionaryEntry).filter(DictionaryEntry.id == entry_id).first() if not entry: raise ValueError(f"Entry not found: {entry_id}") - log.reason("Deleting dictionary entry", payload={"entry_id": entry_id}) + logger.reason("Deleting dictionary entry", {"entry_id": entry_id}) db.delete(entry) db.commit() - log.reflect("Entry deleted", payload={"entry_id": entry_id}) - # #endregion DictionaryManager.delete_entry + logger.reflect("Entry deleted", {"entry_id": entry_id}) + # [/DEF:DictionaryManager.delete_entry:Function] - # #region DictionaryManager.clear_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,clear] - # @BRIEF Delete all entries for a dictionary. + # [DEF:DictionaryManager.clear_entries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Delete all entries for a dictionary. + # @PRE: dict_id exists. + # @POST: All entries for the dictionary are deleted. @staticmethod def clear_entries(db: Session, dict_id: str) -> int: with belief_scope("DictionaryManager.clear_entries"): dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first() if not dictionary: raise ValueError(f"Dictionary not found: {dict_id}") - log.reason("Clearing all entries", payload={"dict_id": dict_id}) + logger.reason("Clearing all entries", {"dict_id": dict_id}) deleted = db.query(DictionaryEntry).filter(DictionaryEntry.dictionary_id == dict_id).delete() db.commit() - log.reflect("Entries cleared", payload={"dict_id": dict_id, "count": deleted}) + logger.reflect("Entries cleared", {"dict_id": dict_id, "count": deleted}) return deleted - # #endregion DictionaryManager.clear_entries + # [/DEF:DictionaryManager.clear_entries:Function] - # #region DictionaryManager.list_entries [C:2] [TYPE Function] [SEMANTICS dictionary,entry,list] - # @BRIEF List entries for a dictionary with pagination. + # [DEF:DictionaryManager.list_entries:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List entries for a dictionary with pagination. + # @PRE: dict_id exists. + # @POST: Returns (list of entries, total_count). @staticmethod def list_entries( db: Session, dict_id: str, page: int = 1, page_size: int = 50, @@ -296,13 +313,14 @@ class DictionaryManager: .all() ) return entries, total - # #endregion DictionaryManager.list_entries + # [/DEF:DictionaryManager.list_entries:Function] - # #region DictionaryManager.import_entries [C:4] [TYPE Function] [SEMANTICS dictionary,import,csv] - # @BRIEF Import entries from CSV/TSV content with duplicate detection and conflict resolution (overwrite/keep_existing/cancel). - # @PRE content is valid CSV or TSV. dict_id exists. - # @POST Entries are created/updated/skipped per conflict mode. Returns result summary with preview support. - # @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows. + # [DEF:DictionaryManager.import_entries:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution. + # @PRE: content is valid CSV or TSV. dict_id exists. + # @POST: Entries are created/updated/skipped per conflict mode. Returns result summary. + # @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows. @staticmethod def import_entries( db: Session, dict_id: str, content: str, @@ -319,7 +337,7 @@ class DictionaryManager: # Detect delimiter if not specified if not delimiter: delimiter = _detect_delimiter(content) - log.reason("Detected delimiter", payload={"delimiter": repr(delimiter)}) + logger.reason("Detected delimiter", {"delimiter": repr(delimiter)}) if delimiter not in (",", "\t"): raise ValueError(f"Unsupported delimiter: {repr(delimiter)}. Use ',' or '\\t'.") @@ -426,7 +444,7 @@ class DictionaryManager: if not preview_only: db.commit() - log.reflect("Import complete", payload={ + logger.reflect("Import complete", { "dict_id": dict_id, "total": result["total"], "created": result["created"], @@ -435,13 +453,14 @@ class DictionaryManager: "errors": len(result["errors"]), }) return result - # #endregion DictionaryManager.import_entries + # [/DEF:DictionaryManager.import_entries:Function] - # #region DictionaryManager.filter_for_batch [C:4] [TYPE Function] [SEMANTICS dictionary,filter,batch] - # @BRIEF Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. - # @PRE job_id exists and source_texts is a list of strings. - # @POST Returns list of matched entries with match info, sorted by dictionary link priority. - # @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. + # [DEF:DictionaryManager.filter_for_batch:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job. + # @PRE: job_id exists and source_texts is a list of strings. + # @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries. + # @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables. @staticmethod def filter_for_batch( db: Session, source_texts: List[str], job_id: str, @@ -454,7 +473,7 @@ class DictionaryManager: .all() ) if not job_dict_links: - log.reason("No dictionaries attached to job", payload={"job_id": job_id}) + logger.reason("No dictionaries attached to job", {"job_id": job_id}) return [] dict_ids = [jd.dictionary_id for jd in job_dict_links] @@ -525,21 +544,22 @@ class DictionaryManager: dict_priority = {d_id: idx for idx, d_id in enumerate(dict_ids)} matched.sort(key=lambda m: (dict_priority.get(m["dictionary_id"], 999), m["text_index"])) - log.reflect("Batch filter match complete", payload={ + logger.reflect("Batch filter match complete", { "job_id": job_id, "source_texts": len(source_texts), "matches": len(matched), "dictionaries_used": len(dictionaries), }) return matched - # #endregion DictionaryManager.filter_for_batch + # [/DEF:DictionaryManager.filter_for_batch:Function] - # #region DictionaryManager.submit_correction [C:4] [TYPE Function] [SEMANTICS dictionary,correction,submit] - # @BRIEF Submit a term correction from a run result with conflict detection and origin tracking. - # @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. - # @POST Entry created or updated; origin tracking populated. Returns action + conflict info. - # @SIDE_EFFECT Creates/updates DictionaryEntry row. + # [DEF:DictionaryManager.submit_correction:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts. + # @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists. + # @POST: Entry created or updated; origin tracking populated. Returns action + conflict info. + # @SIDE_EFFECT: Creates/updates DictionaryEntry row. @staticmethod def submit_correction( db: Session, @@ -628,15 +648,16 @@ class DictionaryManager: result["message"] = f"Entry created for '{source_term}' -> '{corrected_target_term}'" db.commit() - log.reflect("Correction processed", payload=result) + logger.reflect("Correction processed", result) return result - # #endregion DictionaryManager.submit_correction + # [/DEF:DictionaryManager.submit_correction:Function] - # #region DictionaryManager.submit_bulk_corrections [C:4] [TYPE Function] [SEMANTICS dictionary,correction,bulk] - # @BRIEF Submit multiple term corrections atomically — all succeed or all fail on conflict. - # @PRE corrections list is non-empty. dict_id exists. - # @POST All corrections applied or none applied with conflict list (atomic). - # @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once. + # [DEF:DictionaryManager.submit_bulk_corrections:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail. + # @PRE: corrections list is non-empty. dict_id exists. + # @POST: All corrections applied or none applied with conflict list. + # @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once. @staticmethod def submit_bulk_corrections( db: Session, @@ -750,7 +771,7 @@ class DictionaryManager: "total": len(corrections), "applied": sum(1 for r in results if r["action"] in ("created", "updated")), } - # #endregion DictionaryManager.submit_bulk_corrections + # [/DEF:DictionaryManager.submit_bulk_corrections:Function] # #endregion DictionaryManager diff --git a/backend/src/plugins/translate/events.py b/backend/src/plugins/translate/events.py index e36fd3a5..9e876f09 100644 --- a/backend/src/plugins/translate/events.py +++ b/backend/src/plugins/translate/events.py @@ -1,27 +1,24 @@ -# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate,events,audit,logging] +# #region TranslationEventLog [C:5] [TYPE Module] [SEMANTICS translate, events, audit, logging] # @BRIEF Structured event logging for translation operations with terminal event invariant enforcement. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationEvent] # @RELATION DEPENDS_ON -> [MetricSnapshot] -# @PRE Database session is open and valid. -# @POST Events are persisted immutably; terminal events enforce exactly-one invariant per run. -# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. -# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] -# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id. -# @RATIONALE Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. -# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. -# @REJECTED Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. +# @PRE: Database session is open and valid. +# @POST: Events are persisted immutably; terminal events enforce exactly-one invariant per run. +# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events with MetricSnapshot before deletion. +# @DATA_CONTRACT: Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] +# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. +# @RATIONALE: Immutable event log with nullable run_id allows job-level events (no run context) and run-level events. +# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. +# @REJECTED: Separate event table per entity — single TranslationEvent table with nullable run_id is simpler for audit. from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session from datetime import datetime, timezone, timedelta import uuid -from ...core.cot_logger import MarkerLogger -from ...core.logger import belief_scope +from ...core.logger import logger, belief_scope from ...models.translate import TranslationEvent, MetricSnapshot -log = MarkerLogger("EventLog") - # Terminal events per run — exactly one of these must exist for a completed run. TERMINAL_EVENT_TYPES = {"RUN_COMPLETED", "RUN_FAILED", "RUN_CANCELLED"} # All valid event types for validation. @@ -39,48 +36,23 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | { DEFAULT_RETENTION_DAYS = 90 -# #region TranslationEventLog [C:5] [TYPE Class] [SEMANTICS translate,events,log,invariants] -# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement and pruning. -# @PRE Database session is available. -# @POST Events are written immutably; terminal events enforced per run; expired events pruned with MetricSnapshot. -# @SIDE_EFFECT Writes TranslationEvent rows; prunes expired events. -# @DATA_CONTRACT Input[run_id:Optional[str], job_id:str, event_type:str, payload:dict] -> Output[TranslationEvent] -# @INVARIANT Exactly one run_started + exactly one terminal event per non-null run_id. +# #region TranslationEventLog [C:5] [TYPE Class] +# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement. +# @PRE: Database session is available. +# @POST: Events are written immutably; terminal events enforced per run. +# @SIDE_EFFECT: Writes TranslationEvent rows; prunes expired events. +# @INVARIANT: Exactly one run_started + exactly one terminal event per non-null run_id. class TranslationEventLog: def __init__(self, db: Session): self.db = db - # #region _create_event_raw [C:3] [TYPE Function] [SEMANTICS translate,events,create] - # @BRIEF Create a TranslationEvent row without invariant checks. Internal helper for callers that manage invariants themselves. - # @RELATION DEPENDS_ON -> [TranslationEvent] - def _create_event_raw( - self, - job_id: str, - event_type: str, - payload: Optional[Dict[str, Any]] = None, - run_id: Optional[str] = None, - created_by: Optional[str] = None, - ) -> TranslationEvent: - event = TranslationEvent( - id=str(uuid.uuid4()), - job_id=job_id, - run_id=run_id, - event_type=event_type, - event_data=payload or {}, - created_by=created_by, - created_at=datetime.now(timezone.utc), - ) - self.db.add(event) - self.db.flush() - return event - # #endregion _create_event_raw - - # #region log_event [C:4] [TYPE Function] [SEMANTICS translate,events,log] - # @BRIEF Write an immutable event. Enforces terminal event invariant and run_started ordering for non-null run_id. - # @PRE event_type must be a known type. If run_id is not None, enforce terminal + start invariants. - # @POST TranslationEvent row is created; invariants checked before write. - # @SIDE_EFFECT DB write. + # [DEF:log_event:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Write an immutable event. Enforces terminal event invariant for non-null run_id. + # @PRE: event_type must be a known type. If run_id is not None, enforce terminal invariant. + # @POST: TranslationEvent row is created. + # @SIDE_EFFECT: DB write. def log_event( self, job_id: str, @@ -107,10 +79,9 @@ class TranslationEventLog: .first() ) if existing_terminal: - existing_id = existing_terminal[0] if isinstance(existing_terminal, (tuple, list)) else str(existing_terminal.id) raise ValueError( f"Run '{run_id}' already has a terminal event " - f"({existing_id}). Cannot add another terminal event." + f"({existing_terminal[0]}). Cannot add another terminal event." ) # Enforce run_started invariant — must exist before other run events @@ -129,24 +100,31 @@ class TranslationEventLog: f"RUN_STARTED event must precede other run events." ) - event = self._create_event_raw( + event = TranslationEvent( + id=str(uuid.uuid4()), job_id=job_id, - event_type=event_type, - payload=payload, run_id=run_id, + event_type=event_type, + event_data=payload or {}, created_by=created_by, + created_at=datetime.now(timezone.utc), ) - log.reason("Event logged", payload={ + self.db.add(event) + self.db.flush() + + logger.reason(f"Event logged: {event_type}", { "event_id": event.id, "job_id": job_id, "run_id": run_id, "event_type": event_type, }) return event - # #endregion log_event + # [/DEF:log_event:Function] - # #region query_events [C:2] [TYPE Function] [SEMANTICS translate,events,query] - # @BRIEF Query events with optional filters (job_id, run_id, event_type) and pagination. + # [DEF:query_events:Function] + # @PURPOSE: Query events with optional filters. + # @PRE: None. + # @POST: Returns list of TranslationEvent dicts matching filters. def query_events( self, job_id: Optional[str] = None, @@ -184,13 +162,13 @@ class TranslationEventLog: } for e in events ] - # #endregion query_events + # [/DEF:query_events:Function] - # #region prune_expired [C:4] [TYPE Function] [SEMANTICS translate,events,prune] - # @BRIEF Delete events older than retention_days. Creates MetricSnapshot before pruning for audit trail. - # @PRE None. - # @POST Expired events are deleted; MetricSnapshot is created before deletion in batch. - # @SIDE_EFFECT Creates MetricSnapshot row; deletes TranslationEvent rows in batches. + # [DEF:prune_expired:Function] + # @PURPOSE: Delete events older than retention_days. Persists MetricSnapshot before pruning. + # @PRE: None. + # @POST: Expired events are deleted; MetricSnapshot is created before deletion. + # @SIDE_EFFECT: Creates MetricSnapshot row; deletes TranslationEvent rows. def prune_expired( self, retention_days: int = DEFAULT_RETENTION_DAYS, @@ -198,7 +176,7 @@ class TranslationEventLog: ) -> Dict[str, Any]: with belief_scope("TranslationEventLog.prune_expired"): cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days) - log.reason("Pruning expired events", payload={ + logger.reason("Pruning expired events", { "cutoff": cutoff.isoformat(), "retention_days": retention_days, }) @@ -210,7 +188,7 @@ class TranslationEventLog: total_expired = expired_query.count() if total_expired == 0: - log.reflect("No expired events to prune", payload={}) + logger.reflect("No expired events to prune", {}) return {"pruned": 0, "snapshot_id": None} # Create MetricSnapshot before pruning @@ -243,20 +221,21 @@ class TranslationEventLog: ).delete(synchronize_session=False) self.db.flush() pruned += len(ids) - log.reason("Pruned batch", payload={"batch_size": len(ids), "total_pruned": pruned}) + logger.reason("Pruned batch", {"batch_size": len(ids), "total_pruned": pruned}) self.db.commit() - log.reflect("Pruning complete", payload={ + logger.reflect("Pruning complete", { "pruned": pruned, "snapshot_id": snapshot_id, }) return {"pruned": pruned, "snapshot_id": snapshot_id} - # #endregion prune_expired + # [/DEF:prune_expired:Function] - # #region get_run_event_summary [C:3] [TYPE Function] [SEMANTICS translate,events,summary] - # @BRIEF Get a summary of events for a run, including invariant validity check. - # @RELATION DEPENDS_ON -> [TranslationEvent] + # [DEF:get_run_event_summary:Function] + # @PURPOSE: Get a summary of events for a run, including invariant check. + # @PRE: run_id is not None. + # @POST: Returns dict with event list and invariant validity. def get_run_event_summary(self, run_id: str) -> Dict[str, Any]: with belief_scope("TranslationEventLog.get_run_event_summary"): events = self.query_events(run_id=run_id) @@ -273,34 +252,7 @@ class TranslationEventLog: "invariant_valid": has_started and len(terminal_events) <= 1, "events": events, } - # #endregion get_run_event_summary - - # #region get_run_event_invariants_lightweight [C:3] [TYPE Function] [SEMANTICS translate,events,invariants] - # @BRIEF Lightweight invariant check for a run — fetches only event_type column, avoids event_data for performance. - # @RELATION DEPENDS_ON -> [TranslationEvent] - def get_run_event_invariants_lightweight(self, run_id: str) -> Dict[str, Any]: - with belief_scope("TranslationEventLog.get_run_event_invariants_lightweight"): - # Lightweight query: only fetch event_type column, not event_data - event_types = [ - row[0] - for row in ( - self.db.query(TranslationEvent.event_type) - .filter(TranslationEvent.run_id == run_id) - .order_by(TranslationEvent.created_at.desc()) - .limit(100) - .all() - ) - ] - - has_run_started = "RUN_STARTED" in event_types - terminal_count = sum(1 for et in event_types if et in TERMINAL_EVENT_TYPES) - - return { - "has_run_started": has_run_started, - "terminal_event_count": terminal_count, - "invariant_valid": has_run_started and terminal_count <= 1, - } - # #endregion get_run_event_invariants_lightweight + # [/DEF:get_run_event_summary:Function] # #endregion TranslationEventLog diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index 40b30747..e9f39179 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -1,19 +1,17 @@ -# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS translate,executor,batch,llm] +# #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS translate, executor, batch, llm] # @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationBatch] # @RELATION DEPENDS_ON -> [TranslationRecord] # @RELATION DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [DictionaryManager] # @RELATION DEPENDS_ON -> [TranslationPreview] -# @PRE Valid TranslationRun with job configuration. DB session is available. -# @POST TranslationBatch and TranslationRecord rows are created. Run status is updated. -# @SIDE_EFFECT Calls LLM provider; creates DB rows; updates run statistics. -# @DATA_CONTRACT Input[TranslationRun, job] -> Output[TranslationRun with batches and records] -# @INVARIANT Batch processing with retry — independent batches allow partial recovery; MAX_RETRIES_PER_BATCH = 3. -# @RATIONALE Batch processing with retry — independent batches allow partial recovery. -# @REJECTED Single monolithic LLM call — would lose all progress on any failure. +# @PRE: Valid TranslationRun with job configuration. DB session is available. +# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated. +# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics. +# @RATIONALE: Batch processing with retry — independent batches allow partial recovery. +# @REJECTED: Single monolithic LLM call — would lose all progress on any failure. import json import time @@ -24,7 +22,6 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Callable from sqlalchemy.orm import Session from ...core.logger import logger, belief_scope -from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...models.translate import ( TranslationJob, @@ -36,24 +33,20 @@ from ...models.translate import ( ) from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt -from ...core.superset_client import SupersetClient from .dictionary import DictionaryManager from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE -log = MarkerLogger("TranslationExecutor") -# #region MAX_RETRIES_PER_BATCH [C:1] [TYPE Constant] [SEMANTICS translate,executor,retry] +# #region MAX_RETRIES_PER_BATCH [TYPE Constant] +# @BRIEF Maximum number of retries for a single batch before marking it failed. MAX_RETRIES_PER_BATCH = 3 -# #endregion MAX_RETRIES_PER_BATCH -# #region TranslationExecutor [C:5] [TYPE Class] [SEMANTICS translate,executor,batch] +# #region TranslationExecutor [C:4] [TYPE Class] # @BRIEF Process translation batches: fetch source rows, filter dict, call LLM, persist results. -# @PRE DB session and config manager available. -# @POST Batches and records created with status tracking; run statistics updated. -# @SIDE_EFFECT LLM API calls; DB writes. -# @DATA_CONTRACT Input[TranslationRun] -> Output[TranslationRun with batches, records, stats] -# @INVARIANT MAX_RETRIES_PER_BATCH limit enforced; partial batch success tracked via COMPLETED_WITH_ERRORS. +# @PRE: DB session and config manager available. +# @POST: Batches and records created with status tracking. +# @SIDE_EFFECT: LLM API calls; DB writes. class TranslationExecutor: def __init__( @@ -69,27 +62,25 @@ class TranslationExecutor: self.on_batch_progress = on_batch_progress self._current_run_id: Optional[str] = None - # #region execute_run [C:4] [TYPE Function] [SEMANTICS translate,run,execute] - # @BRIEF Run full translation execution for a TranslationRun: fetch rows, batch, call LLM, persist. - # @PRE run is in PENDING or RUNNING status with valid job config. - # @POST Run is populated with batches and records; run status updated to COMPLETED/FAILED. - # @SIDE_EFFECT LLM API calls; DB batch writes. + # [DEF:execute_run:Function] + # @PURPOSE: Run full translation execution for a TranslationRun. + # @PRE: run is in PENDING or RUNNING status with valid job config. + # @POST: Run is populated with batches and records. + # @SIDE_EFFECT: LLM API calls; DB batch writes. def execute_run( self, run: TranslationRun, llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None, - full_translation: bool = False, ) -> TranslationRun: with belief_scope("TranslationExecutor.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() if not job: raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'") - log.reason("Starting translation execution", payload={ + logger.reason("Starting translation execution", { "run_id": run.id, "job_id": job.id, "batch_size": job.batch_size, - "full_translation": full_translation, }) # Mark run as RUNNING @@ -97,15 +88,10 @@ class TranslationExecutor: run.started_at = datetime.now(timezone.utc) self.db.flush() - # Fetch source rows - if full_translation: - # Full translation: fetch ALL rows from Superset dataset - source_rows = self._fetch_all_rows_from_superset(job) - else: - # Preview-based: fetch rows from the accepted preview session - source_rows = self._fetch_source_rows(job.id, run.id) + # Fetch source rows from the accepted preview session + source_rows = self._fetch_source_rows(job.id, run.id) if not source_rows: - log.explore("No source rows to translate", payload={"run_id": run.id}, error="Preview produced 0 source rows for translation") + logger.explore("No source rows to translate", {"run_id": run.id}) run.status = "COMPLETED" run.completed_at = datetime.now(timezone.utc) self.db.flush() @@ -121,7 +107,7 @@ class TranslationExecutor: for i in range(0, total_rows, batch_size) ] - log.reason(f"Processing {len(batches)} batches", payload={ + logger.reason(f"Processing {len(batches)} batches", { "run_id": run.id, "total_rows": total_rows, "batch_size": batch_size, @@ -165,7 +151,7 @@ class TranslationExecutor: run.completed_at = datetime.now(timezone.utc) self.db.flush() - log.reflect("Translation execution complete", payload={ + logger.reflect("Translation execution complete", { "run_id": run.id, "status": run.status, "total": total_rows, @@ -175,13 +161,12 @@ class TranslationExecutor: }) return run - # #endregion execute_run + # [/DEF:execute_run:Function] - # #region _fetch_source_rows [C:4] [TYPE Function] [SEMANTICS translate,source,rows] - # @BRIEF Fetch source rows from the accepted preview session for this job. - # @PRE job_id exists. - # @POST Returns list of dicts with source data (row_index, source_text, approved_translation, etc). - # @SIDE_EFFECT Queries preview session and records from DB. + # [DEF:_fetch_source_rows:Function] + # @PURPOSE: Fetch source rows from the accepted preview session for this job. + # @PRE: job_id exists. + # @POST: Returns list of dicts with source data. def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]: with belief_scope("TranslationExecutor._fetch_source_rows"): # Get the latest APPLIED preview session @@ -195,7 +180,7 @@ class TranslationExecutor: .first() ) if not session: - log.explore("No accepted preview session found", error="Preview session has no accepted rows", payload={"job_id": job_id}) + logger.explore("No accepted preview session found", {"job_id": job_id}) return [] # Fetch APPROVED or all records from the session @@ -215,124 +200,20 @@ class TranslationExecutor: "source_text": rec.source_sql or "", "approved_translation": rec.target_sql if rec.status == "APPROVED" else None, "source_object_name": rec.source_object_name or "", - "source_data": rec.source_data or {}, }) - log.reason(f"Fetched {len(source_rows)} source rows from preview", payload={ + logger.reason(f"Fetched {len(source_rows)} source rows from preview", { "run_id": run_id, "session_id": session.id, }) return source_rows - # #endregion _fetch_source_rows + # [/DEF:_fetch_source_rows:Function] - # #region _fetch_all_rows_from_superset [C:4] [TYPE Function] [SEMANTICS translate,superset,rows] - # @BRIEF Fetch ALL rows from the Superset dataset for full translation (paginated). - # @PRE job has source_datasource_id and environment_id configured. - # @POST Returns list of row dicts with row_index, source_text, source_data. - # @SIDE_EFFECT Calls Superset chart data endpoint (paginated). - def _fetch_all_rows_from_superset(self, job: TranslationJob) -> List[Dict[str, Any]]: - with belief_scope("TranslationExecutor._fetch_all_rows_from_superset"): - env_id = job.environment_id or job.source_dialect or "" - environments = self.config_manager.get_environments() - env_config = next( - (e for e in environments if e.id == env_id), - None, - ) - if not env_config: - raise ValueError(f"Superset environment '{env_id}' not found") - - client = SupersetClient(env_config) - dataset_detail = client.get_dataset_detail(int(job.source_datasource_id)) - - # Build query context (same as preview, but with large row_limit) - query_context = client.build_dataset_preview_query_context( - dataset_id=int(job.source_datasource_id), - dataset_record=dataset_detail, - template_params={}, - effective_filters=[], - ) - - queries = query_context.get("queries", []) - if queries: - queries[0]["row_limit"] = 100000 # Superset max default - queries[0].pop("result_type", None) - queries[0].pop("columns", None) - queries[0]["metrics"] = [] - queries[0]["row_offset"] = 0 - query_context["result_type"] = "samples" - form_data = query_context.get("form_data", {}) - form_data.pop("query_mode", None) - - all_rows: List[Dict[str, Any]] = [] - batch_size_fetch = 10000 # Fetch 10K rows per pagination request - - while True: - # Set pagination for this batch - if queries: - queries[0]["row_limit"] = min(batch_size_fetch, 100000) - queries[0]["row_offset"] = len(all_rows) - - try: - response = client.network.request( - method="POST", - endpoint="/api/v1/chart/data", - data=json.dumps(query_context), - headers={"Content-Type": "application/json"}, - ) - except Exception as e: - log.explore("Chart data API failed during full fetch", error="Chart data API error", - payload={ - "offset": len(all_rows), - "error": str(e), - }) - if not all_rows: - raise ValueError(f"Failed to fetch data from Superset: {e}") - break # Return what we have - - from .preview import TranslationPreview - rows = TranslationPreview._extract_data_rows(response) - if not rows: - break # No more data - - all_rows.extend(rows) - log.reason(f"Fetched {len(rows)} rows (total: {len(all_rows)})", payload={ - "offset": len(all_rows) - len(rows), - }) - - if len(rows) < batch_size_fetch: - break # Last page - - # Convert to the format expected by _process_batch - source_rows = [] - for idx, row in enumerate(all_rows): - translation_value = str(row.get(job.translation_column, "") or "") - context_values = {} - if job.context_columns: - for col in job.context_columns: - context_values[col] = str(row.get(col, "") or "") - # Also include target_key_cols values in context_data - if job.target_key_cols: - for col in job.target_key_cols: - context_values[col] = str(row.get(col, "") or "") - source_rows.append({ - "row_index": str(idx), - "source_text": translation_value, - "source_data": context_values, - "source_object_name": f"Row {idx + 1}", - "approved_translation": None, - }) - - log.reason(f"Prepared {len(source_rows)} source rows for full translation", payload={ - "job_id": job.id, - }) - return source_rows - # #endregion _fetch_all_rows_from_superset - - # #region _process_batch [C:4] [TYPE Function] [SEMANTICS translate,batch,process] - # @BRIEF Process a single batch: filter dictionary, build prompt, call LLM, persist records. - # @PRE job and batch_rows are valid. - # @POST TranslationBatch and TranslationRecord rows are created. - # @SIDE_EFFECT LLM API call; DB writes. + # [DEF:_process_batch:Function] + # @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records. + # @PRE: job and batch_rows are valid. + # @POST: TranslationBatch and TranslationRecord rows are created. + # @SIDE_EFFECT: LLM API call. def _process_batch( self, job: TranslationJob, @@ -391,7 +272,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="SUCCESS", ) self.db.add(record) @@ -419,20 +299,20 @@ class TranslationExecutor: self.db.flush() batch_latency = int((time.monotonic() - batch_start) * 1000) - log.reason(f"Batch {batch_index} complete", payload={ + logger.reason(f"Batch {batch_index} complete", { "batch_id": batch_id, "latency_ms": batch_latency, **result, }) return result - # #endregion _process_batch + # [/DEF:_process_batch:Function] - # #region _call_llm_for_batch [C:4] [TYPE Function] [SEMANTICS translate,llm,batch] - # @BRIEF Call LLM for a batch of rows requiring translation. Parse structured JSON response. - # @PRE job has valid provider_id. batch_rows is non-empty. - # @POST Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. - # @SIDE_EFFECT HTTP call to LLM provider. + # [DEF:_call_llm_for_batch:Function] + # @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response. + # @PRE: job has valid provider_id. batch_rows is non-empty. + # @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows. + # @SIDE_EFFECT: HTTP call to LLM provider. def _call_llm_for_batch( self, job: TranslationJob, @@ -493,8 +373,7 @@ class TranslationExecutor: except Exception as e: last_error = str(e) retries += 1 - log.explore("LLM call failed", error="LLM call failed, retrying", - payload={ + logger.explore(f"LLM call failed (attempt {attempt})", { "batch_id": batch_id, "error": last_error, "attempt": attempt, @@ -502,8 +381,7 @@ class TranslationExecutor: if attempt < MAX_RETRIES_PER_BATCH: time.sleep(2 ** attempt) # Exponential backoff else: - log.explore("LLM call exhausted retries", error="LLM retries exhausted", - payload={ + logger.explore("LLM call exhausted retries", { "batch_id": batch_id, "last_error": last_error, }) @@ -520,7 +398,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="FAILED", error_message=f"LLM call failed after {retries} retries: {last_error}", ) @@ -532,8 +409,7 @@ class TranslationExecutor: translations = self._parse_llm_response(llm_response, len(batch_rows)) except ValueError as e: # Parse failure — mark all rows as SKIPPED - log.explore("LLM response parse failed", error="Failed to parse LLM JSON response", - payload={ + logger.explore("LLM response parse failed", { "batch_id": batch_id, "error": str(e), "response_preview": llm_response[:500] if llm_response else "", @@ -549,7 +425,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="SKIPPED", error_message=f"LLM parse failure: {e}", ) @@ -581,7 +456,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="SKIPPED", error_message="NULL translation returned by LLM", ) @@ -600,7 +474,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="SKIPPED", error_message="Empty translation returned by LLM", ) @@ -617,7 +490,6 @@ class TranslationExecutor: source_object_type="table_row", source_object_id=row.get("row_index"), source_object_name=row.get("source_object_name", ""), - source_data=row.get("source_data", {}), status="SUCCESS", ) self.db.add(record) @@ -628,13 +500,13 @@ class TranslationExecutor: "skipped": skipped, "retries": retries, } - # #endregion _call_llm_for_batch + # [/DEF:_call_llm_for_batch:Function] - # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call] - # @BRIEF Call the configured LLM provider with the batch prompt, routing by provider type. - # @PRE job has valid provider_id. - # @POST Returns raw LLM response string. - # @SIDE_EFFECT HTTP call to LLM provider. + # [DEF:_call_llm:Function] + # @PURPOSE: Call the configured LLM provider with the batch prompt. + # @PRE: job has valid provider_id. + # @POST: Returns raw LLM response string. + # @SIDE_EFFECT: HTTP call to LLM provider. def _call_llm(self, job: TranslationJob, prompt: str) -> str: with belief_scope("TranslationExecutor._call_llm"): if not job.provider_id: @@ -662,13 +534,13 @@ class TranslationExecutor: ) else: raise ValueError(f"Unsupported provider type '{provider_type}'") - # #endregion _call_llm + # [/DEF:_call_llm:Function] - # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai] - # @BRIEF Call OpenAI-compatible API for batch translation with retry and structured output support. - # @PRE Valid API endpoint, key, model, and prompt. - # @POST Returns response text. - # @SIDE_EFFECT HTTP POST to LLM API. + # [DEF:_call_openai_compatible:Function] + # @PURPOSE: Call OpenAI-compatible API for batch translation. + # @PRE: Valid API endpoint, key, model, and prompt. + # @POST: Returns response text. + # @SIDE_EFFECT: HTTP POST to LLM API. @staticmethod def _call_openai_compatible( base_url: str, @@ -699,7 +571,7 @@ class TranslationExecutor: if provider_type in ("openai", "openai_compatible"): payload["response_format"] = {"type": "json_object"} - log.reason( + logger.reason( f"LLM request model={payload.get('model')} " f"provider_type={provider_type} " f"response_format={'yes' if 'response_format' in payload else 'no'} " @@ -707,11 +579,11 @@ class TranslationExecutor: ) response = http_requests.post(url, headers=headers, json=payload, timeout=180) if not response.ok: - log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={ - "status_code": response.status_code, - "model": payload.get('model'), - "body": response.text[:2000], - }) + logger.explore( + f"LLM API error status={response.status_code} " + f"model={payload.get('model')} " + f"body={response.text[:2000]}" + ) response.raise_for_status() data = response.json() @@ -721,27 +593,15 @@ class TranslationExecutor: content = choices[0].get("message", {}).get("content", "") if not content: - # Log full response for diagnostics - finish_reason = choices[0].get("finish_reason", "unknown") - log.explore("LLM returned empty content", error="Empty response from LLM", - payload={ - "finish_reason": finish_reason, - "model": payload.get("model"), - "prompt_len": len(prompt), - "response_keys": list(data.keys()), - "choices_count": len(choices), - }) - raise ValueError( - f"LLM returned empty content (finish_reason={finish_reason}, " - f"model={payload.get('model')}, prompt_len={len(prompt)})" - ) + raise ValueError("LLM returned empty content") return content - # #endregion _call_openai_compatible + # [/DEF:_call_openai_compatible:Function] - # #region _parse_llm_response [C:3] [TYPE Function] [SEMANTICS translate,llm,parse] - # @BRIEF Parse LLM JSON response into dict of row_id -> translation text. - # @RELATION DEPENDS_ON -> [json] + # [DEF:_parse_llm_response:Function] + # @PURPOSE: Parse LLM JSON response into dict of row_id -> translation. + # @PRE: response_text is valid JSON with {"rows": [...]} structure. + # @POST: Returns dict mapping row_id to translation text. @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationExecutor._parse_llm_response"): @@ -774,8 +634,9 @@ class TranslationExecutor: translations[row_id] = str(translation) return translations - # #endregion _parse_llm_response + # [/DEF:_parse_llm_response:Function] # #endregion TranslationExecutor # #endregion TranslationExecutor +# #endregion TranslationExecutor diff --git a/backend/src/plugins/translate/metrics.py b/backend/src/plugins/translate/metrics.py index 50b53a2f..6f6fd847 100644 --- a/backend/src/plugins/translate/metrics.py +++ b/backend/src/plugins/translate/metrics.py @@ -1,10 +1,14 @@ -# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate,metrics,aggregation] +# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS translate, metrics, aggregation] # @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [TranslationEvent] -# @RELATION DEPENDS_ON -> [MetricSnapshot] -# @RELATION DEPENDS_ON -> [TranslationRun] -# @RELATION DEPENDS_ON -> [TranslationSchedule] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [TranslationEvent:Class] +# @RELATION DEPENDS_ON -> [MetricSnapshot:Class] +# @RELATION DEPENDS_ON -> [TranslationRun:Class] +# @RELATION DEPENDS_ON -> [TranslationSchedule:Class] +# @PRE: Database session is open. +# @POST: Metrics are aggregated and returned; no side effects. +# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture. +# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary. from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple @@ -20,18 +24,17 @@ from ...models.translate import ( ) -# #region TranslationMetrics [C:3] [TYPE Class] [SEMANTICS translate,metrics] +# #region TranslationMetrics [C:3] [TYPE Class] # @BRIEF Aggregate translation metrics from live events and MetricSnapshot. -# @RELATION DEPENDS_ON -> [TranslationRun] -# @RELATION DEPENDS_ON -> [MetricSnapshot] -# @RELATION DEPENDS_ON -> [TranslationEvent] class TranslationMetrics: def __init__(self, db: Session): self.db = db - # #region get_job_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,job] - # @BRIEF Get aggregated metrics for a specific job including run counts, record stats, and duration. + # [DEF:get_job_metrics:Function] + # @PURPOSE: Get aggregated metrics for a specific job. + # @PRE: job_id exists. + # @POST: Returns dict with metrics from events + latest snapshot. def get_job_metrics(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationMetrics.get_job_metrics"): # Run counts from TranslationRun @@ -145,10 +148,11 @@ class TranslationMetrics: "last_run_at": last_run.created_at.isoformat() if last_run else None, "next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None, } - # #endregion get_job_metrics + # [/DEF:get_job_metrics:Function] - # #region get_all_metrics [C:2] [TYPE Function] [SEMANTICS translate,metrics,all] - # @BRIEF Get aggregated metrics for all jobs. + # [DEF:get_all_metrics:Function] + # @PURPOSE: Get aggregated metrics for all jobs. + # @POST: Returns list of per-job metrics. def get_all_metrics(self) -> List[Dict[str, Any]]: with belief_scope("TranslationMetrics.get_all_metrics"): job_ids = ( @@ -157,7 +161,7 @@ class TranslationMetrics: .all() ) return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]] - # #endregion get_all_metrics + # [/DEF:get_all_metrics:Function] # #endregion TranslationMetrics diff --git a/backend/src/plugins/translate/orchestrator.py b/backend/src/plugins/translate/orchestrator.py index ae1e54a0..b77721b2 100644 --- a/backend/src/plugins/translate/orchestrator.py +++ b/backend/src/plugins/translate/orchestrator.py @@ -1,6 +1,6 @@ -# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate,orchestrator,lifecycle,run] +# #region TranslationOrchestrator [C:5] [TYPE Module] [SEMANTICS translate, orchestrator, lifecycle, run] # @BRIEF Run lifecycle coordination: validate preconditions, dispatch executor, generate SQL, submit to Superset, record events. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationRun] # @RELATION DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [TranslationExecutor] @@ -8,14 +8,14 @@ # @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor] # @RELATION DEPENDS_ON -> [TranslationEventLog] # @RELATION DEPENDS_ON -> [TranslationPreviewSession] -# @PRE Valid job and accepted preview (for manual runs). Superset and LLM are reachable. -# @POST Translation run is executed, SQL generated and submitted, events recorded. -# @SIDE_EFFECT Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. -# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun] -# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. -# @RATIONALE C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. -# @REJECTED Distributed actor model (Celery) — eventual-consistency challenges at current scale. -# @REJECTED stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. +# @PRE: Valid job and accepted preview (for manual runs). Superset and LLM are reachable. +# @POST: Translation run is executed, SQL generated and submitted, events recorded. +# @SIDE_EFFECT: Creates TranslationRun; creates batches and records; generates SQL; submits to Superset; records events. +# @DATA_CONTRACT: Input[db, config_manager, current_user] -> Output[TranslationRun] +# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. Exactly one terminal event per run. +# @RATIONALE: C5 orchestrator because preview, execution, event logging, and retry share state within single transaction boundary. +# @REJECTED: Distributed actor model (Celery) — eventual-consistency challenges at current scale. +# @REJECTED: stdout-only logging — lacks structured payload integrity; cannot enforce terminal-event invariant. import json import time @@ -26,7 +26,6 @@ from typing import Any, Dict, List, Optional, Callable, Tuple from sqlalchemy.orm import Session from ...core.logger import logger, belief_scope -from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...models.translate import ( TranslationJob, @@ -42,15 +41,13 @@ from .superset_executor import SupersetSqlLabExecutor from .events import TranslationEventLog from ..translate.service import TranslateJobService -log = MarkerLogger("TranslationOrchestrator") -# #region TranslationOrchestrator [C:5] [TYPE Class] [SEMANTICS translate,orchestrator] +# #region TranslationOrchestrator [C:5] [TYPE Class] # @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging. -# @PRE DB session and config manager are available. -# @POST Runs are created, executed, and finalized with event records. -# @SIDE_EFFECT Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. -# @INVARIANT State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. -# @DATA_CONTRACT Input[db, config_manager, current_user] -> Output[TranslationRun] +# @PRE: DB session and config manager are available. +# @POST: Runs are created, executed, and finalized with event records. +# @SIDE_EFFECT: Creates/modifies TranslationRun, TranslationBatch, TranslationRecord, TranslationEvent rows. +# @INVARIANT: State transitions: PENDING -> RUNNING -> COMPLETED|FAILED|CANCELLED. class TranslationOrchestrator: def __init__( @@ -65,11 +62,12 @@ class TranslationOrchestrator: self.event_log = TranslationEventLog(db) self._job: Optional[TranslationJob] = None - # #region start_run [C:5] [TYPE Function] [SEMANTICS translate,run,create] - # @BRIEF Start a new translation run for a job with config snapshot and hash computation. - # @PRE job_id exists. For manual runs, there must be an accepted preview session. - # @POST TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. - # @SIDE_EFFECT DB writes; event logging. + # [DEF:start_run:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Start a new translation run for a job. + # @PRE: job_id exists. For manual runs, there must be an accepted preview session. + # @POST: TranslationRun is created in PENDING status with hash fields and config snapshot. Events are recorded. + # @SIDE_EFFECT: DB writes. def start_run( self, job_id: str, @@ -83,7 +81,7 @@ class TranslationOrchestrator: raise ValueError(f"Translation job '{job_id}' not found") self._job = job - log.reason("Starting translation run", payload={ + logger.reason("Starting translation run", { "job_id": job_id, "is_scheduled": is_scheduled, }) @@ -158,26 +156,25 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - log.reflect("Run created", payload={ + logger.reflect("Run created", { "run_id": run.id, "job_id": job_id, "status": run.status, "trigger_type": run.trigger_type, }) return run - # #endregion start_run + # [/DEF:start_run:Function] - # #region execute_run [C:5] [TYPE Function] [SEMANTICS translate,run,execute] - # @BRIEF Execute a translation run: dispatch executor, generate SQL, submit to Superset. - # @PRE run is in PENDING status. - # @POST Run is executed, SQL generated, Superset submission attempted. - # @SIDE_EFFECT LLM calls, DB writes, Superset API calls. + # [DEF:execute_run:Function] + # @PURPOSE: Execute a translation run: dispatch executor, generate SQL, submit to Superset. + # @PRE: run is in PENDING status. + # @POST: Run is executed, SQL generated, Superset submission attempted. + # @SIDE_EFFECT: LLM calls, DB writes, Superset API calls. def execute_run( self, run: TranslationRun, on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None, skip_insert: bool = False, - full_translation: bool = False, ) -> TranslationRun: with belief_scope("TranslationOrchestrator.execute_run"): job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first() @@ -191,7 +188,7 @@ class TranslationOrchestrator: f"Run must be in PENDING status." ) - log.reason("Executing run", payload={ + logger.reason("Executing run", { "run_id": run.id, "job_id": job.id, "skip_insert": skip_insert, @@ -212,10 +209,11 @@ class TranslationOrchestrator: on_batch_progress=on_batch_progress, ) try: - run = executor.execute_run(run, llm_progress_callback=None, full_translation=full_translation) + run = executor.execute_run(run, llm_progress_callback=None) except Exception as e: - log.explore("Translation execution failed", error=str(e), payload={ + logger.explore("Translation execution failed", { "run_id": run.id, + "error": str(e), }) run.status = "FAILED" run.error_message = f"Translation execution failed: {e}" @@ -292,19 +290,19 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - log.reflect("Run execution complete", payload={ + logger.reflect("Run execution complete", { "run_id": run.id, "status": run.status, "insert_status": run.insert_status, }) return run - # #endregion execute_run + # [/DEF:execute_run:Function] - # #region _generate_and_insert_sql [C:5] [TYPE Function] [SEMANTICS translate,sql,insert] - # @BRIEF Generate INSERT SQL from successful records and submit to Superset SQL Lab. - # @PRE job has target table configured. run has successful records. - # @POST SQL is generated and submitted. Returns execution result. - # @SIDE_EFFECT Superset API call; event logging. + # [DEF:_generate_and_insert_sql:Function] + # @PURPOSE: Generate INSERT SQL from successful records and submit to Superset SQL Lab. + # @PRE: job has target table configured. run has successful records. + # @POST: SQL is generated and submitted. Returns execution result. + # @SIDE_EFFECT: Superset API call. def _generate_and_insert_sql( self, job: TranslationJob, @@ -323,21 +321,20 @@ class TranslationOrchestrator: ) if not records: - log.reason("No successful records to insert", payload={"run_id": run.id}) + logger.reason("No successful records to insert", {"run_id": run.id}) return {"status": "skipped", "reason": "no_records", "query_id": None} - log.reason(f"Generating SQL for {len(records)} records", payload={ + logger.reason(f"Generating SQL for {len(records)} records", { "run_id": run.id, "dialect": job.database_dialect or job.target_dialect, }) # Build rows for SQL generation - # Only include the translation column — context columns are for LLM only - columns = [] - if job.translation_column: + columns = job.context_columns or [] + if job.translation_column and job.translation_column not in columns: columns.append(job.translation_column) - # Include key columns if present (for UPSERT matching) + # Also include key columns if used for upsert if job.target_key_cols: for k in job.target_key_cols: if k not in columns: @@ -346,6 +343,9 @@ class TranslationOrchestrator: rows_for_sql = [] for rec in records: row_data = {} + if job.context_columns: + for col in job.context_columns: + row_data[col] = "" if job.translation_column: row_data[job.translation_column] = rec.target_sql or "" if job.target_key_cols: @@ -372,20 +372,10 @@ class TranslationOrchestrator: upsert_strategy=job.upsert_strategy or "MERGE", ) except ValueError as e: - log.explore("SQL generation failed", error=str(e)) + logger.explore("SQL generation failed", {"error": str(e)}) return {"status": "failed", "error_message": str(e), "query_id": None} # Log insert phase start - log.reason("Insert SQL generated", payload={ - "run_id": run.id, - "sql_preview": sql[:500], - "sql_length": len(sql), - "row_count": row_count, - "dialect": job.database_dialect or job.target_dialect or "postgresql", - "columns": columns, - "has_key_cols": bool(job.target_key_cols), - }) - self.event_log.log_event( job_id=job.id, run_id=run.id, @@ -397,26 +387,16 @@ class TranslationOrchestrator: # Submit to Superset try: env_id = job.environment_id or job.source_dialect or "" - # Resolve database_id: explicit target_database_id from job (int or UUID), or resolve from environment - target_db_id = None - if job.target_database_id: - try: - target_db_id = int(job.target_database_id) - except (ValueError, TypeError): - # Could be a UUID — pass as-is, executor will resolve it - target_db_id = job.target_database_id - log.reason("target_database_id is not an integer, will resolve as UUID", payload={ - "target_database_id": job.target_database_id, - }) - executor = SupersetSqlLabExecutor(self.config_manager, env_id, database_id=target_db_id) + executor = SupersetSqlLabExecutor(self.config_manager, env_id) result = executor.execute_and_poll( sql=sql, max_polls=30, poll_interval_seconds=2.0, ) except Exception as e: - log.explore("Superset SQL submission failed", error=str(e), payload={ + logger.explore("Superset SQL submission failed", { "run_id": run.id, + "error": str(e), }) result = {"status": "failed", "error_message": str(e), "query_id": None} @@ -430,13 +410,13 @@ class TranslationOrchestrator: ) return result - # #endregion _generate_and_insert_sql + # [/DEF:_generate_and_insert_sql:Function] - # #region _validate_preconditions [C:5] [TYPE Function] [SEMANTICS translate,validation] - # @BRIEF Validate preconditions before starting a translation run. - # @PRE None. - # @POST Raises ValueError if preconditions are not met. - # @SIDE_EFFECT None. + # [DEF:_validate_preconditions:Function] + # @PURPOSE: Validate preconditions before starting a run. + # @PRE: None. + # @POST: Raises ValueError if preconditions are not met. + # @SIDE_EFFECT: None. def _validate_preconditions( self, job: TranslationJob, @@ -488,17 +468,17 @@ class TranslationOrchestrator: "Run and accept a preview before executing a manual translation run." ) - log.reason("Preconditions validated", payload={ + logger.reason("Preconditions validated", { "job_id": job.id, "is_scheduled": is_scheduled, }) - # #endregion _validate_preconditions + # [/DEF:_validate_preconditions:Function] - # #region retry_failed_batches [C:5] [TYPE Function] [SEMANTICS translate,retry,batch] - # @BRIEF Retry failed batches in a run by re-processing failed records. - # @PRE run exists and has failed batches. - # @POST Failed batches are re-processed; run status and stats updated. - # @SIDE_EFFECT LLM calls; DB writes; event logging. + # [DEF:retry_failed_batches:Function] + # @PURPOSE: Retry failed batches in a run. + # @PRE: run exists and has failed batches. + # @POST: Failed batches are re-processed. + # @SIDE_EFFECT: LLM calls, DB writes. def retry_failed_batches( self, run_id: str, @@ -526,7 +506,7 @@ class TranslationOrchestrator: if not failed_batches: raise ValueError(f"No failed batches found for run '{run_id}'") - log.reason("Retrying failed batches", payload={ + logger.reason("Retrying failed batches", { "run_id": run_id, "batch_count": len(failed_batches), }) @@ -602,18 +582,18 @@ class TranslationOrchestrator: ) self.db.commit() - log.reflect("Retry complete", payload={ + logger.reflect("Retry complete", { "run_id": run_id, "status": run.status, }) return run - # #endregion retry_failed_batches + # [/DEF:retry_failed_batches:Function] - # #region retry_insert [C:5] [TYPE Function] [SEMANTICS translate,retry,insert] - # @BRIEF Retry the SQL insert phase for a completed run. - # @PRE run exists and has successful records. - # @POST SQL is regenerated and re-submitted to Superset. - # @SIDE_EFFECT Superset API call; event logging. + # [DEF:retry_insert:Function] + # @PURPOSE: Retry the SQL insert phase for a completed run. + # @PRE: run exists and has successful records. + # @POST: SQL is regenerated and re-submitted to Superset. + # @SIDE_EFFECT: Superset API call. def retry_insert( self, run_id: str, @@ -628,7 +608,7 @@ class TranslationOrchestrator: raise ValueError(f"Job '{run.job_id}' not found") self._job = job - log.reason("Retrying insert phase", payload={ + logger.reason("Retrying insert phase", { "run_id": run_id, }) @@ -652,18 +632,18 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - log.reflect("Insert retry complete", payload={ + logger.reflect("Insert retry complete", { "run_id": run_id, "insert_status": run.insert_status, }) return run - # #endregion retry_insert + # [/DEF:retry_insert:Function] - # #region cancel_run [C:5] [TYPE Function] [SEMANTICS translate,run,cancel] - # @BRIEF Cancel a running translation run. - # @PRE run is in PENDING or RUNNING status. - # @POST Run status is set to CANCELLED; event recorded. - # @SIDE_EFFECT DB write; records event. + # [DEF:cancel_run:Function] + # @PURPOSE: Cancel a running translation. + # @PRE: run is in PENDING or RUNNING status. + # @POST: Run status is set to CANCELLED. + # @SIDE_EFFECT: DB write; records event. def cancel_run(self, run_id: str) -> TranslationRun: with belief_scope("TranslationOrchestrator.cancel_run"): run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() @@ -680,8 +660,7 @@ class TranslationOrchestrator: run.completed_at = datetime.now(timezone.utc) self.db.flush() - # Record terminal event directly — we are the source of the terminal event - self.event_log._create_event_raw( + self.event_log.log_event( job_id=run.job_id, run_id=run.id, event_type="RUN_CANCELLED", @@ -692,16 +671,16 @@ class TranslationOrchestrator: self.db.commit() self.db.refresh(run) - log.reflect("Run cancelled", payload={ + logger.reflect("Run cancelled", { "run_id": run_id, }) return run - # #endregion cancel_run + # [/DEF:cancel_run:Function] - # #region get_run_status [C:5] [TYPE Function] [SEMANTICS translate,run,status] - # @BRIEF Get run status with statistics and event invariant checks. - # @PRE run_id exists. - # @POST Returns dict with run details. + # [DEF:get_run_status:Function] + # @PURPOSE: Get run status with statistics. + # @PRE: run_id exists. + # @POST: Returns dict with run details. def get_run_status(self, run_id: str) -> Dict[str, Any]: with belief_scope("TranslationOrchestrator.get_run_status"): run = self.db.query(TranslationRun).filter(TranslationRun.id == run_id).first() @@ -715,8 +694,8 @@ class TranslationOrchestrator: .count() ) - # Get event invariants (lightweight — only fetches event_type column) - invariants = self.event_log.get_run_event_invariants_lightweight(run_id) + # Get event summary + event_summary = self.event_log.get_run_event_summary(run_id) return { "id": run.id, @@ -732,16 +711,20 @@ class TranslationOrchestrator: "insert_status": run.insert_status, "superset_execution_id": run.superset_execution_id, "batch_count": batch_count, - "event_invariants": invariants, + "event_invariants": { + "has_run_started": event_summary["has_run_started"], + "terminal_event_count": event_summary["terminal_event_count"], + "invariant_valid": event_summary["invariant_valid"], + }, "created_by": run.created_by, "created_at": run.created_at.isoformat() if run.created_at else None, } - # #endregion get_run_status + # [/DEF:get_run_status:Function] - # #region get_run_records [C:5] [TYPE Function] [SEMANTICS translate,run,records] - # @BRIEF Get paginated records for a run with optional status filter. - # @PRE run_id exists. - # @POST Returns dict with records and pagination info. + # [DEF:get_run_records:Function] + # @PURPOSE: Get paginated records for a run. + # @PRE: run_id exists. + # @POST: Returns dict with records and pagination info. def get_run_records( self, run_id: str, @@ -787,12 +770,12 @@ class TranslationOrchestrator: "page_size": page_size, "status_filter": status_filter, } - # #endregion get_run_records + # [/DEF:get_run_records:Function] - # #region get_run_history [C:5] [TYPE Function] [SEMANTICS translate,run,history] - # @BRIEF Get paginated run history for a job. - # @PRE job_id exists. - # @POST Returns tuple of (total_count, list_of_runs). + # [DEF:get_run_history:Function] + # @PURPOSE: Get run history for a job. + # @PRE: job_id exists. + # @POST: Returns list of runs. def get_run_history( self, job_id: str, @@ -830,10 +813,10 @@ class TranslationOrchestrator: } for r in runs ] - # #endregion get_run_history + # [/DEF:get_run_history:Function] - # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] - # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison. + # [DEF:_compute_config_hash:Function] + # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. @staticmethod def _compute_config_hash(job: TranslationJob) -> str: import hashlib @@ -849,10 +832,10 @@ class TranslationOrchestrator: "upsert_strategy": job.upsert_strategy, }, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] - # #endregion _compute_config_hash + # [/DEF:_compute_config_hash:Function] - # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] - # @BRIEF Compute a SHA-256 hash of dictionary state for snapshot comparison. + # [DEF:_compute_dict_snapshot_hash:Function] + # @PURPOSE: Compute a hash of dictionary state for snapshot comparison. def _compute_dict_snapshot_hash(self, job_id: str) -> str: import hashlib from ...models.translate import TranslationJobDictionary @@ -864,7 +847,7 @@ class TranslationOrchestrator: dict_ids = sorted([dl.dictionary_id for dl in dict_links]) hash_input = ",".join(dict_ids) return hashlib.sha256(hash_input.encode()).hexdigest()[:16] - # #endregion _compute_dict_snapshot_hash + # [/DEF:_compute_dict_snapshot_hash:Function] # #endregion TranslationOrchestrator diff --git a/backend/src/plugins/translate/plugin.py b/backend/src/plugins/translate/plugin.py index 940468e2..a446183b 100644 --- a/backend/src/plugins/translate/plugin.py +++ b/backend/src/plugins/translate/plugin.py @@ -1,15 +1,17 @@ -# #region TranslatePlugin [C:3] [TYPE Module] [SEMANTICS plugin,translate,llm,sql,dashboard] +# #region TranslatePlugin [C:2] [TYPE Module] [SEMANTICS plugin, translate, llm, sql, dashboard] # @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation. -# @LAYER Domain +# @LAYER: Domain # @RELATION INHERITS -> [PluginBase] +# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines). +# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains. from typing import Dict, Any, Optional from ...core.plugin_base import PluginBase -# #region TranslatePlugin [C:3] [TYPE Class] [SEMANTICS plugin,translate] +# #region TranslatePlugin [TYPE Class] # @BRIEF Plugin for translating SQL queries and dashboard definitions across database dialects. -# @RELATION IMPLEMENTS -> [PluginBase] +# @RELATION IMPLEMENTS -> backend.src.core.plugin_base.PluginBase class TranslatePlugin(PluginBase): @property def id(self) -> str: diff --git a/backend/src/plugins/translate/preview.py b/backend/src/plugins/translate/preview.py index ece466b7..17a4e6c1 100644 --- a/backend/src/plugins/translate/preview.py +++ b/backend/src/plugins/translate/preview.py @@ -1,21 +1,19 @@ -# #region TranslationPreview [C:5] [TYPE Module] [SEMANTICS translate,preview,llm,session] +# #region TranslationPreview [C:4] [TYPE Module] [SEMANTICS translate, preview, llm, session] # @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [TranslationJob] -# @RELATION DEPENDS_ON -> [TranslationPreviewSession] -# @RELATION DEPENDS_ON -> [TranslationPreviewRecord] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [TranslationJob:Class] +# @RELATION DEPENDS_ON -> [TranslationPreviewSession:Class] +# @RELATION DEPENDS_ON -> [TranslationPreviewRecord:Class] # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [DictionaryManager] # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [render_prompt] # @RELATION DEPENDS_ON -> [ConfigManager] -# @PRE Database session and config manager are available. -# @POST Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. -# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows. -# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate] -# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time. -# @RATIONALE C5 because preview is stateful with approve/edit/reject lifecycle and LLM API calls requiring structured error handling. -# @REJECTED Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. +# @PRE: Database session and config manager are available. +# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. +# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls. +# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably. from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session @@ -25,8 +23,7 @@ import json import hashlib import re -from ...core.cot_logger import MarkerLogger -from ...core.logger import belief_scope +from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient from ...models.translate import ( @@ -39,9 +36,8 @@ from ...services.llm_provider import LLMProviderService from ...services.llm_prompt_templates import render_prompt from .dictionary import DictionaryManager -log = MarkerLogger("TranslationPreview") - -# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt] +# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant] +# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster). DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( "Translate the following database content from {source_language} to {target_language}.\n\n" "Source dialect: {source_dialect}\n" @@ -57,7 +53,8 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = ( # #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE -# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [C:1] [TYPE Constant] [SEMANTICS translate,prompt] +# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant] +# @BRIEF Default prompt template for LLM translation preview. DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( "Translate the following database content from {source_language} to {target_language}.\n\n" "Source dialect: {source_dialect}\n" @@ -74,8 +71,10 @@ DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = ( # #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE -# #region TokenEstimator [C:2] [TYPE Class] [SEMANTICS translate,tokens,estimate] -# @BRIEF Estimate token counts and costs for LLM translation operations using heuristic chars/token ratio. +# #region TokenEstimator [TYPE Class] +# @BRIEF Estimate token counts and costs for LLM translation operations. +# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model. +# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only. class TokenEstimator: """Estimate token counts and costs for LLM operations.""" @@ -83,38 +82,45 @@ class TokenEstimator: OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50 TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens - # #region estimate_prompt_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens] + # [DEF:estimate_prompt_tokens:Function] + # @PURPOSE: Estimate token count for a prompt string. + # @PRE: prompt is a non-empty string. + # @POST: Returns estimated token count (integer). @staticmethod def estimate_prompt_tokens(prompt: str) -> int: if not prompt: return 0 return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE)) - # #endregion estimate_prompt_tokens + # [/DEF:estimate_prompt_tokens:Function] - # #region estimate_output_tokens [C:1] [TYPE Function] [SEMANTICS translate,tokens] + # [DEF:estimate_output_tokens:Function] + # @PURPOSE: Estimate output token count for translating N rows. + # @PRE: row_count >= 0. + # @POST: Returns estimated output token count. @staticmethod def estimate_output_tokens(row_count: int) -> int: return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE - # #endregion estimate_output_tokens + # [/DEF:estimate_output_tokens:Function] - # #region estimate_cost [C:1] [TYPE Function] [SEMANTICS translate,cost] + # [DEF:estimate_cost:Function] + # @PURPOSE: Estimate cost for a given number of tokens. + # @PRE: total_tokens >= 0. + # @POST: Returns estimated cost in USD. @staticmethod def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float: rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K return round((total_tokens / 1000) * rate, 6) - # #endregion estimate_cost + # [/DEF:estimate_cost:Function] # #endregion TokenEstimator -# #region TranslationPreview [C:5] [TYPE Class] [SEMANTICS translate,preview,lifecycle] +# #region TranslationPreview [C:4] [TYPE Class] # @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate. -# @PRE Database session and config manager are available. -# @POST Preview sessions created with persisted records; full execution gates on accepted session. -# @SIDE_EFFECT Fetches sample data from Superset; calls LLM provider; creates DB rows. -# @DATA_CONTRACT Input[job_id, sample_size] -> Output[Dict with session, records, cost_estimate] -# @INVARIANT Preview sessions expire after 24 hours; exactly one ACTIVE session per job at a time. +# @PRE: Database session and config manager are available. +# @POST: Preview sessions created with persisted records; full execution gates on accepted session. +# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows. class TranslationPreview: def __init__( @@ -127,11 +133,12 @@ class TranslationPreview: self.config_manager = config_manager self.current_user = current_user - # #region preview_rows [C:4] [TYPE Function] [SEMANTICS translate,preview,rows] - # @BRIEF Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. - # @PRE job_id exists and job has source_datasource_id, translation_column configured. - # @POST Returns TranslationPreviewResponse with records, cost estimation, and persistent session. - # @SIDE_EFFECT Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. + # [DEF:preview_rows:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records. + # @PRE: job_id exists and job has source_datasource_id, translation_column configured. + # @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session. + # @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows. def preview_rows( self, job_id: str, @@ -140,7 +147,7 @@ class TranslationPreview: env_id: Optional[str] = None, ) -> Dict[str, Any]: with belief_scope("TranslationPreview.preview_rows"): - log.reason("Starting preview for job", payload={"job_id": job_id, "sample_size": sample_size}) + logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size}) # 1. Load job job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() @@ -156,7 +163,7 @@ class TranslationPreview: dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id) # 3. Fetch sample rows from Superset - log.reason("Fetching sample rows from Superset", payload={ + logger.reason("Fetching sample rows from Superset", { "datasource_id": job.source_datasource_id, "sample_size": sample_size, "translation_column": job.translation_column, @@ -170,12 +177,12 @@ class TranslationPreview: raise ValueError("No rows returned from datasource for preview") actual_row_count = len(source_rows) - log.reason(f"Fetched {actual_row_count} sample row(s)") + logger.reason(f"Fetched {actual_row_count} sample row(s)") # Debug: log first row keys and translation column value if source_rows: first_row = source_rows[0] - log.reason( + logger.reason( f"First source row keys={list(first_row.keys())} " f"translation_col={job.translation_column} " f"val='{first_row.get(job.translation_column, '')}'" @@ -249,7 +256,7 @@ class TranslationPreview: total_est_cost = TokenEstimator.estimate_cost(total_est_tokens) # 8. Call LLM - log.reason("Calling LLM for preview translation", payload={ + logger.reason("Calling LLM for preview translation", { "provider_id": job.provider_id, "row_count": actual_row_count, "estimated_tokens": sample_total_tokens, @@ -293,7 +300,6 @@ class TranslationPreview: source_object_name=f"Row {idx + 1}", status=status, feedback=feedback, - source_data=meta.get("context_data"), created_at=datetime.now(timezone.utc), ) self.db.add(record) @@ -333,19 +339,18 @@ class TranslationPreview: "dict_snapshot_hash": dict_snapshot_hash, } - log.reflect("Preview completed", payload={ + logger.reflect("Preview completed", { "session_id": session.id, "row_count": actual_row_count, "sample_cost": sample_cost, }) return result - # #endregion preview_rows + # [/DEF:preview_rows:Function] - # #region update_preview_row [C:4] [TYPE Function] [SEMANTICS translate,preview,row,update] - # @BRIEF Approve, edit, or reject an individual preview row. - # @PRE session_id and row_id exist, session is ACTIVE. - # @POST PreviewRecord status is updated (APPROVED, REJECTED, or APPROVED with edited translation). - # @SIDE_EFFECT DB write. + # [DEF:update_preview_row:Function] + # @PURPOSE: Approve, edit, or reject an individual preview row. + # @PRE: session_id and row_id exist, session is ACTIVE. + # @POST: PreviewRecord status is updated. def update_preview_row( self, job_id: str, @@ -396,7 +401,7 @@ class TranslationPreview: self.db.commit() self.db.refresh(record) - log.reason(f"Preview row {action}d", payload={ + logger.reason(f"Preview row {action}d", { "row_id": row_id, "session_id": session.id, "status": record.status, @@ -409,13 +414,13 @@ class TranslationPreview: "status": record.status, "feedback": record.feedback, } - # #endregion update_preview_row + # [/DEF:update_preview_row:Function] - # #region accept_preview_session [C:4] [TYPE Function] [SEMANTICS translate,preview,accept] - # @BRIEF Mark a preview session as accepted (APPLIED), which gates full execution. - # @PRE job_id has an ACTIVE preview session. - # @POST Session status changes to APPLIED; future full execution calls check for accepted session. - # @SIDE_EFFECT DB write. + # [DEF:accept_preview_session:Function] + # @PURPOSE: Mark a preview session as accepted, which gates full execution. + # @PRE: job_id has an ACTIVE preview session. + # @POST: Session status changes to APPLIED. + # @SIDE_EFFECT: Future full execution calls will check for accepted session. def accept_preview_session(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationPreview.accept_preview_session"): session = ( @@ -434,7 +439,7 @@ class TranslationPreview: self.db.commit() self.db.refresh(session) - log.reason("Preview session accepted", payload={ + logger.reason("Preview session accepted", { "session_id": session.id, "job_id": job_id, }) @@ -464,11 +469,12 @@ class TranslationPreview: for r in records ], } - # #endregion accept_preview_session + # [/DEF:accept_preview_session:Function] - # #region get_preview_session [C:3] [TYPE Function] [SEMANTICS translate,preview,get] - # @BRIEF Get the latest preview session for a job with its records. - # @RELATION DEPENDS_ON -> [TranslationPreviewSession] + # [DEF:get_preview_session:Function] + # @PURPOSE: Get the latest preview session for a job with its records. + # @PRE: job_id exists. + # @POST: Returns session data with records or raises ValueError. def get_preview_session(self, job_id: str) -> Dict[str, Any]: with belief_scope("TranslationPreview.get_preview_session"): session = ( @@ -507,13 +513,14 @@ class TranslationPreview: for r in records ], } - # #endregion get_preview_session + # [/DEF:get_preview_session:Function] - # #region _fetch_sample_rows [C:4] [TYPE Function] [SEMANTICS translate,superset,sample] - # @BRIEF Fetch sample rows from the Superset dataset for preview. - # @PRE job has source_datasource_id and translation_column. - # @POST Returns list of dicts with row data from Superset chart data API. - # @SIDE_EFFECT Calls Superset chart data endpoint with pagination. + # [DEF:_fetch_sample_rows:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Fetch sample rows from the Superset dataset for preview. + # @PRE: job has source_datasource_id and translation_column. + # @POST: Returns list of dicts with row data. + # @SIDE_EFFECT: Calls Superset chart data endpoint. def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]: with belief_scope("TranslationPreview._fetch_sample_rows"): # Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy) @@ -524,15 +531,13 @@ class TranslationPreview: None, ) if not env_config: - log.explore("Could not find environment for datasource", error="Environment not found for datasource", - payload={ + logger.explore("Could not find environment for datasource", { "env_id": target_env_id, }) # Fallback: try first environment if environments: env_config = environments[0] - log.explore("Falling back to first available environment", error="Environment fallback used", - payload={ + logger.explore("Falling back to first available environment", { "env_id": env_config.id, }) else: @@ -577,28 +582,29 @@ class TranslationPreview: headers={"Content-Type": "application/json"}, ) except Exception as e: - log.explore("Chart data API failed", error=str(e)) + logger.explore("Chart data API failed", {"error": str(e)}) raise ValueError(f"Failed to fetch sample data from Superset: {e}") # Parse response rows = self._extract_data_rows(response) - log.reason(f"Extracted {len(rows)} data row(s)") + logger.reason(f"Extracted {len(rows)} data row(s)") # Debug: log first row keys and translation column value if rows: first_row = rows[0] - log.reason( + logger.reason( f"Row keys={list(first_row.keys())} " f"target_col={job.translation_column} " f"val='{first_row.get(job.translation_column, '')}'" ) return rows - # #endregion _fetch_sample_rows + # [/DEF:_fetch_sample_rows:Function] - # #region _extract_data_rows [C:3] [TYPE Function] [SEMANTICS translate,superset,data] - # @BRIEF Extract data rows from Superset chart data response, trying multiple response formats. - # @RELATION DEPENDS_ON -> [SupersetClient] + # [DEF:_extract_data_rows:Function] + # @PURPOSE: Extract data rows from Superset chart data response. + # @PRE: response is a dict from Superset API. + # @POST: Returns list of row dicts. @staticmethod def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]: with belief_scope("TranslationPreview._extract_data_rows"): @@ -627,13 +633,14 @@ class TranslationPreview: return result return [] - # #endregion _extract_data_rows + # [/DEF:_extract_data_rows:Function] - # #region _call_llm [C:4] [TYPE Function] [SEMANTICS translate,llm,call] - # @BRIEF Call the configured LLM provider with the preview prompt. - # @PRE job has a valid provider_id. - # @POST Returns raw LLM response string. - # @SIDE_EFFECT Makes HTTP call to LLM provider. + # [DEF:_call_llm:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Call the configured LLM provider with the preview prompt. + # @PRE: job has a valid provider_id. + # @POST: Returns raw LLM response string. + # @SIDE_EFFECT: Makes HTTP call to LLM provider. def _call_llm(self, job: TranslationJob, prompt: str) -> str: with belief_scope("TranslationPreview._call_llm"): if not job.provider_id: @@ -663,19 +670,19 @@ class TranslationPreview: else: raise ValueError(f"Unsupported provider type '{provider_type}' for preview") - log.reason("LLM call completed", payload={ + logger.reason("LLM call completed", { "provider_id": job.provider_id, "model": model, "response_length": len(response_text), }) return response_text - # #endregion _call_llm + # [/DEF:_call_llm:Function] - # #region _call_openai_compatible [C:4] [TYPE Function] [SEMANTICS translate,llm,openai] - # @BRIEF Call an OpenAI-compatible API for translation with structured output support. - # @PRE base_url, api_key, model, prompt are valid. - # @POST Returns response text. - # @SIDE_EFFECT Makes HTTP POST to LLM API. + # [DEF:_call_openai_compatible:Function] + # @PURPOSE: Call an OpenAI-compatible API for translation. + # @PRE: base_url, api_key, model, prompt are valid. + # @POST: Returns response text. + # @SIDE_EFFECT: Makes HTTP POST to LLM API. @staticmethod def _call_openai_compatible( base_url: str, @@ -706,7 +713,7 @@ class TranslationPreview: if provider_type in ("openai", "openai_compatible"): payload["response_format"] = {"type": "json_object"} - log.reason( + logger.reason( f"LLM request model={payload.get('model')} " f"provider_type={provider_type} " f"response_format={'yes' if 'response_format' in payload else 'no'} " @@ -714,11 +721,11 @@ class TranslationPreview: ) response = http_requests.post(url, headers=headers, json=payload, timeout=120) if not response.ok: - log.explore("LLM API error", error=f"LLM API returned status {response.status_code}", payload={ - "status_code": response.status_code, - "model": payload.get('model'), - "body": response.text[:2000], - }) + logger.explore( + f"LLM API error status={response.status_code} " + f"model={payload.get('model')} " + f"body={response.text[:2000]}" + ) response.raise_for_status() data = response.json() @@ -731,17 +738,16 @@ class TranslationPreview: raise ValueError("LLM returned empty content") return content - # #endregion _call_openai_compatible + # [/DEF:_call_openai_compatible:Function] - # #region _parse_llm_response [C:4] [TYPE Function] [SEMANTICS translate,llm,parse] - # @BRIEF Parse the LLM JSON response into a dict of row_id -> translation, with markdown code block fallback. - # @PRE response_text is valid JSON with {"rows": [...]} structure. - # @POST Returns dict mapping string row_id to translation text; warns if fewer translations than expected. - # @SIDE_EFFECT Logs warnings for short responses. + # [DEF:_parse_llm_response:Function] + # @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation. + # @PRE: response_text is valid JSON with {"rows": [...]} structure. + # @POST: Returns dict mapping string row_id to translation text. @staticmethod def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]: with belief_scope("TranslationPreview._parse_llm_response"): - log.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") + logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}") try: data = json.loads(response_text) @@ -759,7 +765,7 @@ class TranslationPreview: rows = data.get("rows", []) if not isinstance(rows, list): - log.explore("LLM response has no 'rows' array", error="LLM response missing 'rows' key") + logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}") raise ValueError("LLM response missing 'rows' array") translations: Dict[str, str] = {} @@ -770,17 +776,16 @@ class TranslationPreview: translations[row_id] = translation if len(translations) < expected_count: - log.explore("LLM returned fewer translations than expected", error=f"Expected {expected_count} translations, got {len(translations)}", payload={ - "expected_count": expected_count, - "actual_count": len(translations), - "response_preview": response_text[:600], - }) + logger.explore( + f"LLM returned fewer translations expected={expected_count} " + f"got={len(translations)} response_preview={response_text[:600]}" + ) return translations - # #endregion _parse_llm_response + # [/DEF:_parse_llm_response:Function] - # #region _compute_config_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] - # @BRIEF Compute a SHA-256 hash of the job's current configuration for snapshot comparison. + # [DEF:_compute_config_hash:Function] + # @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison. @staticmethod def _compute_config_hash(job: TranslationJob) -> str: config_str = json.dumps({ @@ -795,10 +800,10 @@ class TranslationPreview: "upsert_strategy": job.upsert_strategy, }, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] - # #endregion _compute_config_hash + # [/DEF:_compute_config_hash:Function] - # #region _compute_dict_snapshot_hash [C:2] [TYPE Function] [SEMANTICS translate,hash] - # @BRIEF Compute a SHA-256 hash of the dictionary state for snapshot comparison. + # [DEF:_compute_dict_snapshot_hash:Function] + # @PURPOSE: Compute a hash of the dictionary state for snapshot comparison. def _compute_dict_snapshot_hash(self, job_id: str) -> str: dict_links = ( self.db.query(TranslationJobDictionary) @@ -808,8 +813,9 @@ class TranslationPreview: dict_ids = sorted([dl.dictionary_id for dl in dict_links]) hash_input = ",".join(dict_ids) return hashlib.sha256(hash_input.encode()).hexdigest()[:16] - # #endregion _compute_dict_snapshot_hash + # [/DEF:_compute_dict_snapshot_hash:Function] # #endregion TranslationPreview + # #endregion TranslationPreview diff --git a/backend/src/plugins/translate/scheduler.py b/backend/src/plugins/translate/scheduler.py index 87268916..0d2f3689 100644 --- a/backend/src/plugins/translate/scheduler.py +++ b/backend/src/plugins/translate/scheduler.py @@ -1,18 +1,16 @@ -# #region TranslationScheduler [C:5] [TYPE Module] [SEMANTICS translate,scheduler,apscheduler,cron] -# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService for cron-based execution. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [TranslationSchedule] -# @RELATION DEPENDS_ON -> [SchedulerService] -# @RELATION DEPENDS_ON -> [TranslationOrchestrator] -# @RELATION DEPENDS_ON -> [TranslationEventLog] -# @PRE Database session and SchedulerService are available. -# @POST TranslationSchedule CRUD persisted; translation runs triggered on schedule. -# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events. -# @DATA_CONTRACT Input[job_id, cron_expression] -> Output[TranslationSchedule] -# @INVARIANT Concurrency guard: max 1 pending/running run per job at scheduled trigger time. -# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. -# @REJECTED Separate scheduler instance would create resource contention. -# @REJECTED Polling-based approach — event-driven APScheduler is more precise. +# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS translate, scheduler, apscheduler, cron] +# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService. +# @LAYER: Domain +# @RELATION DEPENDS_ON -> [TranslationSchedule:Class] +# @RELATION DEPENDS_ON -> [SchedulerService:Class] +# @RELATION DEPENDS_ON -> [TranslationOrchestrator:Class] +# @RELATION DEPENDS_ON -> [TranslationEventLog:Class] +# @PRE: Database session and SchedulerService are available. +# @POST: TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed. +# @SIDE_EFFECT: Registers APScheduler jobs; runs translations on trigger; creates events. +# @RATIONALE: Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. +# @REJECTED: Separate scheduler instance would create resource contention. +# @REJECTED: Polling-based approach — event-driven APScheduler is more precise. import uuid from datetime import datetime, timezone, timedelta @@ -20,22 +18,14 @@ from typing import Any, Dict, List, Optional from sqlalchemy.orm import Session -from ...core.cot_logger import MarkerLogger -from ...core.logger import belief_scope +from ...core.logger import logger, belief_scope from ...core.config_manager import ConfigManager from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun from .events import TranslationEventLog -log = MarkerLogger("TranslationScheduler") - -# #region TranslationScheduler [C:5] [TYPE Class] [SEMANTICS translate,scheduler,crud] +# #region TranslationScheduler [C:4] [TYPE Class] # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. -# @PRE Database session and ConfigManager are available. -# @POST Schedule rows created/updated/deleted with event logging. -# @SIDE_EFFECT Writes TranslationSchedule rows; logs SCHEDULE_CREATED/UPDATED/DELETED events. -# @DATA_CONTRACT Input[job_id, cron_expression, timezone] -> Output[TranslationSchedule] -# @INVARIANT Exactly one schedule per job (upsert pattern). class TranslationScheduler: def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): @@ -44,11 +34,10 @@ class TranslationScheduler: self.current_user = current_user self.event_log = TranslationEventLog(db) - # #region create_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,create] - # @BRIEF Create a new schedule for a job with cron expression and event logging. - # @PRE job_id exists. cron_expression is valid. - # @POST TranslationSchedule row created; SCHEDULE_CREATED event logged. - # @SIDE_EFFECT DB write; event logging. + # [DEF:create_schedule:Function] + # @PURPOSE: Create a new schedule for a job. + # @PRE: job_id exists. cron_expression is valid. + # @POST: TranslationSchedule row created. def create_schedule( self, job_id: str, @@ -85,15 +74,14 @@ class TranslationScheduler: created_by=self.current_user, ) - log.reflect("Schedule created", payload={"schedule_id": schedule.id, "job_id": job_id}) + logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) return schedule - # #endregion create_schedule + # [/DEF:create_schedule:Function] - # #region update_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,update] - # @BRIEF Update an existing schedule's cron expression, timezone, or active state. - # @PRE job_id has an existing schedule. - # @POST Schedule updated; SCHEDULE_UPDATED event logged. - # @SIDE_EFFECT DB write; event logging. + # [DEF:update_schedule:Function] + # @PURPOSE: Update an existing schedule. + # @PRE: job_id has an existing schedule. + # @POST: Schedule updated. def update_schedule( self, job_id: str, @@ -130,15 +118,14 @@ class TranslationScheduler: created_by=self.current_user, ) - log.reflect("Schedule updated", payload={"schedule_id": schedule.id, "job_id": job_id}) + logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) return schedule - # #endregion update_schedule + # [/DEF:update_schedule:Function] - # #region delete_schedule [C:4] [TYPE Function] [SEMANTICS translate,schedule,delete] - # @BRIEF Delete a schedule for a job with event logging. - # @PRE job_id has an existing schedule. - # @POST Schedule deleted; SCHEDULE_DELETED event logged. - # @SIDE_EFFECT DB write; event logging. + # [DEF:delete_schedule:Function] + # @PURPOSE: Delete a schedule for a job. + # @PRE: job_id has an existing schedule. + # @POST: Schedule deleted. def delete_schedule(self, job_id: str) -> None: with belief_scope("TranslationScheduler.delete_schedule"): schedule = self.db.query(TranslationSchedule).filter( @@ -158,14 +145,13 @@ class TranslationScheduler: created_by=self.current_user, ) - log.reflect("Schedule deleted", payload={"schedule_id": schedule_id, "job_id": job_id}) - # #endregion delete_schedule + logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) + # [/DEF:delete_schedule:Function] - # #region set_schedule_active [C:4] [TYPE Function] [SEMANTICS translate,schedule,activate] - # @BRIEF Enable or disable a schedule by setting is_active flag. - # @PRE job_id has an existing schedule. - # @POST Schedule is_active updated. - # @SIDE_EFFECT DB write. + # [DEF:enable_disable_schedule:Function] + # @PURPOSE: Enable or disable a schedule. + # @PRE: job_id has an existing schedule. + # @POST: Schedule is_active updated. def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule: with belief_scope("TranslationScheduler.set_schedule_active"): schedule = self.db.query(TranslationSchedule).filter( @@ -179,16 +165,18 @@ class TranslationScheduler: self.db.commit() self.db.refresh(schedule) - log.reflect("Schedule active state set", payload={ + logger.reflect("Schedule active state set", { "schedule_id": schedule.id, "job_id": job_id, "is_active": is_active, }) return schedule - # #endregion set_schedule_active + # [/DEF:enable_disable_schedule:Function] - # #region get_schedule [C:2] [TYPE Function] [SEMANTICS translate,schedule,get] - # @BRIEF Get the schedule for a job by job_id. + # [DEF:get_schedule:Function] + # @PURPOSE: Get schedule for a job. + # @PRE: job_id exists. + # @POST: Returns TranslationSchedule or raises ValueError. def get_schedule(self, job_id: str) -> TranslationSchedule: with belief_scope("TranslationScheduler.get_schedule"): schedule = self.db.query(TranslationSchedule).filter( @@ -197,10 +185,11 @@ class TranslationScheduler: if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") return schedule - # #endregion get_schedule + # [/DEF:get_schedule:Function] - # #region list_active_schedules [C:2] [TYPE Function] [SEMANTICS translate,schedule,list] - # @BRIEF List all active schedules. + # [DEF:list_active_schedules:Function] + # @PURPOSE: List all active schedules. + # @POST: Returns list of active TranslationSchedule rows. @staticmethod def list_active_schedules(db: Session) -> List[TranslationSchedule]: return ( @@ -208,10 +197,13 @@ class TranslationScheduler: .filter(TranslationSchedule.is_active == True) .all() ) - # #endregion list_active_schedules + # [/DEF:list_active_schedules:Function] - # #region get_next_executions [C:2] [TYPE Function] [SEMANTICS translate,schedule,next] - # @BRIEF Compute next N execution times from a cron expression using APScheduler's CronTrigger. + # [DEF:get_next_executions:Function] + # @PURPOSE: Compute next N execution times from cron expression. + # @PRE: cron_expression is valid. + # @POST: Returns list of ISO datetime strings. + @staticmethod def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> List[str]: from zoneinfo import ZoneInfo from apscheduler.triggers.cron import CronTrigger @@ -221,7 +213,7 @@ class TranslationScheduler: tz = ZoneInfo(timezone_str) trigger = CronTrigger.from_crontab(cron_expression, timezone=tz) except (ValueError, KeyError) as e: - log.explore("Invalid cron", error=str(e)) + logger.warning(f"[get_next_executions] Invalid cron: {e}") return [] now = datetime.now(tz) @@ -236,20 +228,18 @@ class TranslationScheduler: prev = ft next_time = ft return results - # #endregion get_next_executions + # [/DEF:get_next_executions:Function] # #endregion TranslationScheduler -# #region execute_scheduled_translation [C:5] [TYPE Function] [SEMANTICS translate,schedule,execute] -# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and baseline expiry detection. -# @PRE schedule_id is valid and job exists. -# @POST Translation run created and executed if no concurrent run exists. -# @SIDE_EFFECT DB writes; LLM calls; Superset API calls; event logging. -# @DATA_CONTRACT Input[schedule_id, job_id, db_session_maker, config_manager] -> Output[None] -# @INVARIANT Concurrency guard: max 1 pending/running run per job at trigger time. -# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. +# #region execute_scheduled_translation [TYPE Function] +# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. +# @PRE: schedule_id is valid. +# @POST: Translation run created and executed if no concurrent run exists. +# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls. +# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. def execute_scheduled_translation( schedule_id: str, job_id: str, @@ -266,10 +256,10 @@ def execute_scheduled_translation( TranslationSchedule.job_id == job_id, ).first() if not schedule or not schedule.is_active: - log.reason("Schedule inactive/missing", payload={"schedule_id": schedule_id}) + logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}") return - log.reason("Scheduled translation triggered", payload={ + logger.reason("Scheduled translation triggered", { "schedule_id": schedule_id, "job_id": job_id, }) @@ -284,8 +274,7 @@ def execute_scheduled_translation( .first() ) if active_run: - log.explore("Skipping scheduled run — concurrent run in progress", error="Concurrent run detected", - payload={ + logger.explore("Skipping scheduled run — concurrent run in progress", { "job_id": job_id, "existing_run_id": active_run.id, }) @@ -312,7 +301,7 @@ def execute_scheduled_translation( age = datetime.now(timezone.utc) - most_recent.created_at if age > timedelta(days=90): baseline_expired = True - log.reason("Baseline expired — full translation", payload={ + logger.reason("Baseline expired — full translation", { "job_id": job_id, "last_run": most_recent.created_at.isoformat(), "age_days": age.days, @@ -343,7 +332,10 @@ def execute_scheduled_translation( orch.execute_run(run) run.insert_status = run.insert_status or "succeeded" except Exception as exec_err: - log.explore("Scheduled translation execution failed", error=str(exec_err)) + logger.explore("Scheduled translation execution failed", { + "run_id": run.id, + "error": str(exec_err), + }) # Leave schedule enabled — schedule continues on failure run.status = "FAILED" run.error_message = str(exec_err) @@ -353,13 +345,13 @@ def execute_scheduled_translation( schedule.last_run_at = datetime.now(timezone.utc) db.commit() - log.reflect("Scheduled translation complete", payload={ + logger.reflect("Scheduled translation complete", { "schedule_id": schedule_id, "run_id": run.id, "status": run.status, }) except Exception as e: - log.explore("Unexpected error", error=str(e)) + logger.error(f"[scheduled_translation] Unexpected error: {e}") finally: db.close() # #endregion execute_scheduled_translation diff --git a/backend/src/plugins/translate/service.py b/backend/src/plugins/translate/service.py index a01d4a30..bfef8986 100644 --- a/backend/src/plugins/translate/service.py +++ b/backend/src/plugins/translate/service.py @@ -1,16 +1,14 @@ -# #region TranslateJobService [C:5] [TYPE Module] [SEMANTICS translate,service,crud,validation] +# #region TranslateJobService [C:4] [TYPE Module] [SEMANTICS translate, service, crud, validation] # @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [ConfigManager] -# @PRE Database session and config manager are available. -# @POST Translation jobs are created/updated/deleted with column validation and dialect caching. -# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time. -# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob/TranslateJobResponse] -# @INVARIANT Snapshot isolation: in-progress runs use config snapshot; config edits affect future runs only. -# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. -# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity. +# @PRE: Database session and config manager are available. +# @POST: Translation jobs are created/updated/deleted with column validation and dialect caching. +# @SIDE_EFFECT: Queries Superset for column metadata and database dialect at save time. +# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only. +# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity. from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.orm import Session @@ -18,7 +16,6 @@ from datetime import datetime, timezone import uuid from ...core.logger import logger -from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient from ...models.translate import TranslationJob, TranslationJobDictionary @@ -30,8 +27,6 @@ from ...schemas.translate import ( DatasourceColumnResponse, ) -log = MarkerLogger("TranslateJobService") - # Supported database dialects for translation SUPPORTED_DIALECTS = { "postgresql", "mysql", "clickhouse", "sqlite", "mssql", @@ -40,16 +35,12 @@ SUPPORTED_DIALECTS = { } -# #region get_dialect_from_database [C:4] [TYPE Function] [SEMANTICS translate,dialect,validation] -# @BRIEF Extract normalized dialect string from a Superset database record and validate it is supported. -# @PRE database_record is a dict from Superset API with 'backend' or 'engine' key. -# @POST Returns normalized dialect string or raises ValueError if unsupported. -# @SIDE_EFFECT None — pure data extraction. +# #region get_dialect_from_database [TYPE Function] +# @BRIEF Extract normalized dialect string from a Superset database record. +# @PRE: database_record is a dict from Superset API. +# @POST: Returns normalized dialect string or raises ValueError if unsupported. def get_dialect_from_database(database_record: Dict[str, Any]) -> str: """Extract and validate dialect from Superset database record.""" - log.reason("Extracting dialect from database record", payload={ - "has_backend": bool(database_record.get("backend") or database_record.get("engine")), - }) backend = ( database_record.get("backend") or database_record.get("engine") @@ -57,9 +48,6 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: ).lower().strip() if not backend: - log.explore("Could not determine database dialect", error="No backend/engine field in database record", payload={ - "record_keys": list(database_record.keys()), - }) raise ValueError("Could not determine database dialect from connection") # Map Superset backend names to normalized dialect @@ -85,36 +73,28 @@ def get_dialect_from_database(database_record: Dict[str, Any]) -> str: normalized = dialect_map.get(backend, backend) if normalized not in SUPPORTED_DIALECTS: - log.explore("Unsupported dialect", payload={"backend": backend, "normalized": normalized}, error=f"Unsupported database dialect: {backend}") raise ValueError( f"Unsupported database dialect: '{backend}'. " f"Supported dialects: {', '.join(sorted(SUPPORTED_DIALECTS))}" ) - log.reflect("Dialect validated", payload={"dialect": normalized}) return normalized # #endregion get_dialect_from_database -# #region fetch_datasource_metadata [C:4] [TYPE Function] [SEMANTICS translate,datasource,metadata] -# @BRIEF Fetch datasource columns and database dialect from Superset for a given dataset. -# @PRE dataset_id is a valid Superset dataset ID and environment has valid credentials. -# @POST Returns (columns_list, dialect_string) or raises on failure. -# @SIDE_EFFECT Queries Superset API for dataset detail and database info. +# #region fetch_datasource_metadata [TYPE Function] +# @BRIEF Fetch datasource columns and database dialect from Superset. +# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials. +# @POST: Returns (columns_list, dialect_string) or raises on failure. def fetch_datasource_metadata( dataset_id: int, env_id: str, config_manager: ConfigManager, ) -> Tuple[List[Dict[str, Any]], str]: """Fetch column metadata and database dialect for a datasource from Superset.""" - log.reason("Fetching datasource metadata", payload={ - "dataset_id": dataset_id, - "env_id": env_id, - }) # Find environment config environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: - log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured") raise ValueError(f"Superset environment '{env_id}' not found in configuration") # Create Superset client and fetch dataset detail @@ -151,29 +131,25 @@ def fetch_datasource_metadata( else: raise ValueError("No database information available for this datasource") except Exception as e: - log.explore("Could not fetch database dialect", error=str(e)) + logger.warning(f"[translate_service] Could not fetch database dialect: {e}") raise ValueError(f"Could not determine database dialect: {e}") - log.reflect("Metadata fetched", payload={"columns": len(columns), "dialect": dialect}) return columns, dialect # #endregion fetch_datasource_metadata -# #region detect_virtual_columns [C:2] [TYPE Function] [SEMANTICS translate,columns] -# @BRIEF Identify virtual (calculated) columns from column metadata by filtering non-physical entries. +# #region detect_virtual_columns [TYPE Function] +# @BRIEF Identify virtual (calculated) columns from column metadata. +# @PRE: columns is a list of dicts with 'is_physical' key. +# @POST: Returns list of virtual column names. def detect_virtual_columns(columns: List[Dict[str, Any]]) -> List[str]: """Return names of columns that are virtual (not physical).""" return [col["name"] for col in columns if not col.get("is_physical", True)] # #endregion detect_virtual_columns -# #region TranslateJobService [C:5] [TYPE Class] [SEMANTICS translate,service,crud] +# #region TranslateJobService [TYPE Class] # @BRIEF Service for translation job CRUD with validation and Superset integration. -# @PRE Database session and config manager are available. -# @POST Translation jobs are managed with full lifecycle (create/update/delete/duplicate). -# @SIDE_EFFECT Queries Superset for dialect detection and column validation. -# @DATA_CONTRACT Input[TranslateJobCreate/Update] -> Output[TranslationJob] -# @INVARIANT Snapshot isolation: in-progress runs unaffected by config edits. class TranslateJobService: def __init__(self, db: Session, config_manager: ConfigManager, current_user: Optional[str] = None): @@ -181,8 +157,9 @@ class TranslateJobService: self.config_manager = config_manager self.current_user = current_user - # #region list_jobs [C:2] [TYPE Function] [SEMANTICS translate,jobs,query] - # @BRIEF List translation jobs with optional status filter and pagination. + # [DEF:list_jobs:Function] + # @PURPOSE: List translation jobs with optional status filter and pagination. + # @POST: Returns tuple of (total_count, list_of_jobs). def list_jobs( self, page: int = 1, @@ -199,30 +176,29 @@ class TranslateJobService: jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all() return total, jobs - # #endregion list_jobs + # [/DEF:list_jobs:Function] - # #region get_job [C:2] [TYPE Function] [SEMANTICS translate,jobs,query] - # @BRIEF Get a single translation job by ID, raising ValueError if not found. + # [DEF:get_job:Function] + # @PURPOSE: Get a single translation job by ID. + # @POST: Returns TranslationJob or raises ValueError. def get_job(self, job_id: str) -> TranslationJob: job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() if not job: raise ValueError(f"Translation job '{job_id}' not found") return job - # #endregion get_job + # [/DEF:get_job:Function] - # #region create_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,create] - # @BRIEF Create a new translation job with column validation and dialect detection. - # @PRE payload contains valid job configuration (name, upsert_strategy, etc). - # @POST Returns the created TranslationJob with database_dialect cached. - # @SIDE_EFFECT Validates columns via SupersetClient if source_datasource_id is provided; caches database_dialect. + # [DEF:create_job:Function] + # @PURPOSE: Create a new translation job with column validation. + # @PRE: payload contains valid job configuration. + # @POST: Returns the created TranslationJob with database_dialect cached. + # @SIDE_EFFECT: Validates columns via SupersetClient if source_datasource_id is provided. + # @SIDE_EFFECT: Caches database_dialect from Superset connection. def create_job(self, payload: TranslateJobCreate) -> TranslationJob: - log.reason("Creating translation job", payload={"name": payload.name}) + logger.info(f"[TranslateJobService] Creating job '{payload.name}'") # Validate: must have a translation column if datasource is configured if payload.source_datasource_id and not payload.translation_column: - log.explore("Missing translation column", error="translation_column required when source_datasource_id is set", payload={ - "source_datasource_id": payload.source_datasource_id, - }) raise ValueError("A translation column is required when a datasource is selected") # Validate upsert strategy @@ -247,7 +223,7 @@ class TranslateJobService: ) dialect = detected_dialect except Exception as e: - log.explore("Dialect detection failed", error=str(e)) + logger.warning(f"[TranslateJobService] Dialect detection failed: {e}") dialect = payload.source_dialect # Build job instance @@ -271,7 +247,6 @@ class TranslateJobService: batch_size=payload.batch_size, upsert_strategy=payload.upsert_strategy, environment_id=payload.environment_id, - target_database_id=payload.target_database_id, status="DRAFT", created_by=self.current_user, ) @@ -290,17 +265,17 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - log.reflect("Job created", payload={"job_id": job.id}) + logger.info(f"[TranslateJobService] Created job '{job.id}'") return job - # #endregion create_job + # [/DEF:create_job:Function] - # #region update_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,update] - # @BRIEF Update an existing translation job with optional dialect re-detection. - # @PRE payload contains fields to update; job_id exists. - # @POST Returns the updated TranslationJob with re-detected dialect if datasource changed. - # @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed. + # [DEF:update_job:Function] + # @PURPOSE: Update an existing translation job. + # @PRE: payload contains fields to update. + # @POST: Returns the updated TranslationJob. + # @SIDE_EFFECT: Re-detects database_dialect if source_datasource_id changed. def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob: - log.reason("Updating translation job", payload={"job_id": job_id}) + logger.info(f"[TranslateJobService] Updating job '{job_id}'") job = self.get_job(job_id) update_data = payload.model_dump(exclude_unset=True) @@ -321,7 +296,7 @@ class TranslateJobService: ) job.database_dialect = detected_dialect except Exception as e: - log.explore("Dialect re-detection failed", error=str(e)) + logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}") job.updated_at = datetime.now(timezone.utc) @@ -340,17 +315,16 @@ class TranslateJobService: self.db.commit() self.db.refresh(job) - log.reflect("Job updated", payload={"job_id": job_id}) + logger.info(f"[TranslateJobService] Updated job '{job_id}'") return job - # #endregion update_job + # [/DEF:update_job:Function] - # #region delete_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,delete] - # @BRIEF Delete a translation job and its dictionary associations. - # @PRE job_id must exist. - # @POST Job and all related TranslationJobDictionary records are deleted. - # @SIDE_EFFECT Deletes TranslationJob and TranslationJobDictionary rows. + # [DEF:delete_job:Function] + # @PURPOSE: Delete a translation job and its associations. + # @PRE: job_id must exist. + # @POST: Job and all related records are deleted. def delete_job(self, job_id: str) -> None: - log.reason("Deleting translation job", payload={"job_id": job_id}) + logger.info(f"[TranslateJobService] Deleting job '{job_id}'") job = self.get_job(job_id) # Delete dictionary associations @@ -360,16 +334,15 @@ class TranslateJobService: self.db.delete(job) self.db.commit() - log.reflect("Job deleted", payload={"job_id": job_id}) - # #endregion delete_job + logger.info(f"[TranslateJobService] Deleted job '{job_id}'") + # [/DEF:delete_job:Function] - # #region duplicate_job [C:4] [TYPE Function] [SEMANTICS translate,jobs,duplicate] - # @BRIEF Duplicate a translation job with a new name, copying all fields and dictionary associations. - # @PRE job_id must exist. - # @POST Returns the duplicated TranslationJob with status DRAFT. - # @SIDE_EFFECT Creates new TranslationJob and TranslationJobDictionary rows. + # [DEF:duplicate_job:Function] + # @PURPOSE: Duplicate a translation job with a new name. + # @PRE: job_id must exist. + # @POST: Returns the duplicated TranslationJob with status DRAFT. def duplicate_job(self, job_id: str, new_name: Optional[str] = None) -> TranslationJob: - log.reason("Duplicating translation job", payload={"job_id": job_id, "new_name": new_name}) + logger.info(f"[TranslateJobService] Duplicating job '{job_id}'") source = self.get_job(job_id) # Copy all fields except id, created_at, updated_at, status @@ -392,8 +365,6 @@ class TranslateJobService: provider_id=source.provider_id, batch_size=source.batch_size, upsert_strategy=source.upsert_strategy, - environment_id=source.environment_id, - target_database_id=source.target_database_id, status="DRAFT", created_by=self.current_user, ) @@ -414,20 +385,22 @@ class TranslateJobService: self.db.commit() self.db.refresh(new_job) - log.reflect("Job duplicated", payload={"new_job_id": new_job.id, "source_job_id": job_id}) + logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'") return new_job - # #endregion duplicate_job + # [/DEF:duplicate_job:Function] - # #region get_job_dictionary_ids [C:1] [TYPE Function] [SEMANTICS translate,jobs,dictionaries] + # [DEF:get_job_dictionary_ids:Function] + # @PURPOSE: Get dictionary IDs attached to a job. + # @POST: Returns list of dictionary IDs. def get_job_dictionary_ids(self, job_id: str) -> List[str]: assocs = self.db.query(TranslationJobDictionary).filter( TranslationJobDictionary.job_id == job_id ).all() return [a.dictionary_id for a in assocs] - # #endregion get_job_dictionary_ids + # [/DEF:get_job_dictionary_ids:Function] - # #region fetch_available_datasources [C:2] [TYPE Function] [SEMANTICS translate,datasources,query] - # @BRIEF List available Superset datasets for translation job creation with optional search filter. + # [DEF:fetch_available_datasources:Function] + # @PURPOSE: List available Superset datasets for translation job creation. def fetch_available_datasources(self, env_id: str, search: Optional[str] = None) -> list: """List Superset datasets available for translation.""" from ...core.superset_client import SupersetClient @@ -453,11 +426,12 @@ class TranslateJobService: "description": ds.get("description", ""), }) return result - # #endregion fetch_available_datasources + # [/DEF:fetch_available_datasources:Function] # #endregion TranslateJobService -# #region _extract_dialect [C:1] [TYPE Function] [SEMANTICS translate,dialect] +# #region _extract_dialect [TYPE Function] +# @BRIEF Extract database dialect from Superset backend URI. def _extract_dialect(backend: str) -> str: """Extract dialect name from a Superset database backend URI (e.g. 'postgresql+psycopg2://...').""" if not backend: @@ -468,10 +442,9 @@ def _extract_dialect(backend: str) -> str: return dialect.lower() except Exception: return "unknown" -# #endregion _extract_dialect -# #region job_to_response [C:2] [TYPE Function] [SEMANTICS translate,serialization] +# #region job_to_response [TYPE Function] # @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids. def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) -> TranslateJobResponse: return TranslateJobResponse( @@ -499,29 +472,27 @@ def job_to_response(job: TranslationJob, dict_ids: Optional[List[str]] = None) - updated_at=job.updated_at, dictionary_ids=dict_ids or [], environment_id=job.environment_id, - target_database_id=job.target_database_id, ) # #endregion job_to_response -# #region get_datasource_columns [C:4] [TYPE Function] [SEMANTICS translate,datasource,columns] -# @BRIEF Fetch datasource column metadata from Superset and return structured DatasourceColumnsResponse. -# @PRE datasource_id is a valid Superset dataset ID. -# @POST Returns DatasourceColumnsResponse with column metadata and database dialect. -# @SIDE_EFFECT Queries Superset API for dataset detail and database info. +# #region DatasourceColumnsService [TYPE Function] +# @BRIEF Fetch datasource column metadata from Superset and return structured response. +# @PRE: datasource_id is a valid Superset dataset ID. +# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect. +# @SIDE_EFFECT: Queries Superset API for dataset detail and database info. def get_datasource_columns( datasource_id: int, env_id: str, config_manager: ConfigManager, ) -> DatasourceColumnsResponse: """Fetch and return column metadata for a given Superset datasource.""" - log.reason("Fetching datasource columns", payload={"datasource_id": datasource_id, "env_id": env_id}) + logger.info(f"[get_datasource_columns] Fetching columns for datasource {datasource_id}") # Find environment config environments = config_manager.get_environments() env_config = next((e for e in environments if e.id == env_id), None) if not env_config: - log.explore("Environment not found", payload={"env_id": env_id}, error=f"Environment {env_id} not configured") raise ValueError(f"Superset environment '{env_id}' not found") # Create Superset client @@ -543,7 +514,6 @@ def get_datasource_columns( db_result = db_record.get("result", db_record) if isinstance(db_record, dict) else db_record dialect = get_dialect_from_database(db_result) else: - log.explore("Could not determine dialect", payload={"datasource_id": datasource_id}, error=f"Could not determine dialect for datasource {datasource_id}") raise ValueError("Could not determine database dialect for this datasource") # Extract columns @@ -561,19 +531,14 @@ def get_datasource_columns( description=col.get("description", ""), )) - result = DatasourceColumnsResponse( + return DatasourceColumnsResponse( datasource_id=datasource_id, datasource_name=dataset_detail.get("table_name"), schema_name=dataset_detail.get("schema"), database_dialect=dialect, columns=columns, ) - log.reflect("Datasource columns fetched", payload={ - "datasource_id": datasource_id, - "columns_count": len(columns), - "dialect": dialect, - }) - return result -# #endregion get_datasource_columns +# #endregion DatasourceColumnsService # #endregion TranslateJobService +# #endregion TranslateJobService diff --git a/backend/src/plugins/translate/sql_generator.py b/backend/src/plugins/translate/sql_generator.py index 15ca71af..c5b3ef60 100644 --- a/backend/src/plugins/translate/sql_generator.py +++ b/backend/src/plugins/translate/sql_generator.py @@ -1,18 +1,18 @@ -# #region SQLGenerator [C:4] [TYPE Module] [SEMANTICS translate,sql,generator,dialect] -# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations with identifier quoting and value encoding. -# @LAYER Domain +# #region SQLGenerator [C:3] [TYPE Module] [SEMANTICS translate, sql, generator, dialect] +# @BRIEF Dialect-aware safe SQL generation for INSERT/UPSERT operations. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TranslationJob] # @RELATION DEPENDS_ON -> [TranslationRun] -# @PRE Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS. -# @POST Returns safe SQL strings for the target dialect. +# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS. +# @POST: Returns safe SQL strings for the target dialect. +# @SIDE_EFFECT: None — pure code generation. +# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations. +# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case. +# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail. -from typing import Any, Dict, List, Optional, Tuple from datetime import datetime, timezone -from ...core.cot_logger import MarkerLogger - -from ...core.logger import belief_scope - -log = MarkerLogger("SqlGenerator") +from typing import Any, Dict, List, Optional, Tuple +from ...core.logger import logger, belief_scope # PostgreSQL/Greenplum dialects that support ON CONFLICT POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"} @@ -57,8 +57,10 @@ def _normalize_timestamp_value(value: Any) -> Optional[str]: # #endregion _normalize_timestamp_value -# #region _quote_identifier [C:2] [TYPE Function] [SEMANTICS translate,sql,quoting] -# @BRIEF Quote a SQL identifier per dialect rules — double quotes for PostgreSQL, backticks for ClickHouse. +# #region _quote_identifier [TYPE Function] +# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks. +# @PRE: identifier is a non-empty string. +# @POST: Returns safely quoted identifier. def _quote_identifier(identifier: str, dialect: str) -> str: """Quote a SQL identifier per dialect rules.""" if not identifier: @@ -155,17 +157,16 @@ def generate_insert_sql( # #endregion generate_insert_sql -# #region generate_upsert_sql [C:3] [TYPE Function] [SEMANTICS translate,sql,upsert] -# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE / DO NOTHING. -# @RELATION DEPENDS_ON -> [_quote_identifier] -# @RELATION DEPENDS_ON -> [_build_values_clause] +# #region generate_upsert_sql [TYPE Function] +# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE. +# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty. +# @POST: Returns UPSERT SQL string or raises ValueError. def generate_upsert_sql( target_schema: Optional[str], target_table: str, columns: List[str], key_columns: List[str], rows: List[Dict[str, Any]], - dialect: Optional[str] = None, ) -> str: """Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL.""" with belief_scope("generate_upsert_sql"): @@ -180,7 +181,7 @@ def generate_upsert_sql( col_list = ", ".join(columns) key_list = ", ".join(key_columns) - values = _build_values_clause(columns, rows, dialect=dialect) + values = _build_values_clause(columns, rows) table_ref = target_table if target_schema: @@ -205,20 +206,17 @@ def generate_upsert_sql( # #endregion generate_upsert_sql -# #region SQLGenerator [C:4] [TYPE Class] [SEMANTICS translate,sql,generator] -# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements with batching support. -# @PRE Job has target_schema, target_table, key columns configured. -# @POST Returns generated SQL string for the target dialect. -# @SIDE_EFFECT None — pure SQL generation. -# @RELATION DEPENDS_ON -> [generate_insert_sql] -# @RELATION DEPENDS_ON -> [generate_upsert_sql] +# #region SQLGenerator [C:3] [TYPE Class] +# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements. +# @PRE: Job has target_schema, target_table, key columns configured. +# @POST: Returns generated SQL string for the target dialect. class SQLGenerator: - # #region SQLGenerator.generate [C:4] [TYPE Function] [SEMANTICS translate,sql,generate] - # @BRIEF Generate SQL for a set of rows, detecting dialect from the job configuration. - # @PRE dialect is a supported database dialect. columns list is non-empty. rows is non-empty. - # @POST Returns tuple of (sql_string, statement_count). - # @SIDE_EFFECT None — pure SQL generation. + # [DEF:SQLGenerator.generate:Function] + # @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration. + # @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty. + # @POST: Returns tuple of (sql_string, statement_count). + # @SIDE_EFFECT: None — pure SQL generation. @staticmethod def generate( dialect: str, @@ -244,7 +242,7 @@ class SQLGenerator: Tuple of (sql_string, row_count). """ with belief_scope("SQLGenerator.generate"): - log.reason("Generating SQL", payload={ + logger.reason("Generating SQL", { "dialect": dialect, "schema": target_schema, "table": target_table, @@ -290,7 +288,6 @@ class SQLGenerator: columns=quoted_columns, key_columns=quoted_key_columns, rows=rows, - dialect=dialect, ) else: sql = generate_insert_sql( @@ -310,7 +307,7 @@ class SQLGenerator: dialect=dialect, ) if use_upsert: - log.reason("ClickHouse UPSERT not supported; using plain INSERT", payload={ + logger.reason("ClickHouse UPSERT not supported; using plain INSERT", { "note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.", }) else: @@ -323,17 +320,18 @@ class SQLGenerator: dialect=dialect, ) - log.reflect("SQL generated", payload={ + logger.reflect("SQL generated", { "dialect": dialect, "row_count": len(rows), "sql_length": len(sql), }) return sql, len(rows) - # #endregion SQLGenerator.generate + # [/DEF:SQLGenerator.generate:Function] - # #region SQLGenerator.generate_batch [C:3] [TYPE Function] [SEMANTICS translate,sql,batch] - # @BRIEF Generate separate INSERT statements for each row chunk (batch-safe version). - # @RELATION DEPENDS_ON -> [SQLGenerator.generate] + # [DEF:SQLGenerator.generate_batch:Function] + # @PURPOSE: Generate separate INSERT statements for each row (batch-safe version). + # @PRE: Same as generate(). + # @POST: Returns list of (sql_string, row_index) tuples. @staticmethod def generate_batch( dialect: str, @@ -369,7 +367,7 @@ class SQLGenerator: statements.append((sql, count)) return statements - # #endregion SQLGenerator.generate_batch + # [/DEF:SQLGenerator.generate_batch:Function] # #endregion SQLGenerator diff --git a/backend/src/plugins/translate/superset_executor.py b/backend/src/plugins/translate/superset_executor.py index cafc2fd4..275e978f 100644 --- a/backend/src/plugins/translate/superset_executor.py +++ b/backend/src/plugins/translate/superset_executor.py @@ -1,49 +1,43 @@ -# #region SupersetSqlLabExecutor [C:5] [TYPE Module] [SEMANTICS translate,superset,sqllab,execute] -# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status with retry. -# @LAYER Infra +# #region SupersetSqlLabExecutor [C:4] [TYPE Module] [SEMANTICS translate, superset, sqllab, execute] +# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status. +# @LAYER: Infra # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [ConfigManager] -# @PRE Valid Superset environment configuration and authenticated client. -# @POST SQL is submitted to Superset SQL Lab; execution reference is returned. -# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. -# @DATA_CONTRACT Input[sql:str, database_id:Optional] -> Output[Dict with query_id, status, error_message] -# @INVARIANT Polling respects max_polls limit; timeout returns structured error. -# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. -# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. +# @PRE: Valid Superset environment configuration and authenticated client. +# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned. +# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status. +# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset. +# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple from datetime import datetime, timezone import json import time import uuid from ...core.logger import logger, belief_scope -from ...core.cot_logger import MarkerLogger from ...core.config_manager import ConfigManager from ...core.superset_client import SupersetClient -log = MarkerLogger("SupersetExecutor") -# #region SupersetSqlLabExecutor [C:5] [TYPE Class] [SEMANTICS translate,superset,sqllab] +# #region SupersetSqlLabExecutor [C:4] [TYPE Class] # @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking. -# @PRE Valid environment ID and ConfigManager. -# @POST SQL submitted to Superset; execution reference recorded; polling completes. -# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. -# @DATA_CONTRACT Input[env_id:str, database_id:Optional] -> Output[Dict with query_id, status] -# @INVARIANT database_id resolved lazily; polling capped at max_polls. +# @PRE: Valid environment ID and ConfigManager. +# @POST: SQL submitted to Superset; execution reference recorded. +# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status. class SupersetSqlLabExecutor: - def __init__(self, config_manager: ConfigManager, env_id: str, database_id: Optional[int] = None): + def __init__(self, config_manager: ConfigManager, env_id: str): self.config_manager = config_manager self.env_id = env_id self._client: Optional[SupersetClient] = None - self._database_id: Optional[int] = database_id + self._database_id: Optional[int] = None - # #region _get_client [C:4] [TYPE Function] [SEMANTICS translate,superset,client] - # @BRIEF Lazy-initialize SupersetClient for the configured environment. - # @PRE env_id must correspond to a valid environment config. - # @POST Returns authenticated SupersetClient. - # @SIDE_EFFECT Authenticates against Superset API on first call. + # [DEF:_get_client:Function] + # @PURPOSE: Lazy-initialize SupersetClient for the configured environment. + # @PRE: env_id must correspond to a valid environment config. + # @POST: Returns authenticated SupersetClient. + # @SIDE_EFFECT: Authenticates against Superset API. def _get_client(self) -> SupersetClient: with belief_scope("SupersetSqlLabExecutor._get_client"): if self._client is not None: @@ -56,13 +50,13 @@ class SupersetSqlLabExecutor: self._client = SupersetClient(env_config) return self._client - # #endregion _get_client + # [/DEF:_get_client:Function] - # #region resolve_database_id [C:4] [TYPE Function] [SEMANTICS translate,database,resolve] - # @BRIEF Resolve the target database ID from the environment by name or default. - # @PRE Superset environment is reachable and has databases. - # @POST Returns database_id integer or raises ValueError. - # @SIDE_EFFECT Fetches databases list from Superset API. + # [DEF:resolve_database_id:Function] + # @PURPOSE: Resolve the target database ID from the environment. + # @PRE: database_name or database_id should be known. + # @POST: Returns database_id integer or raises ValueError. + # @SIDE_EFFECT: Fetches databases list from Superset. def resolve_database_id(self, database_name: Optional[str] = None) -> int: with belief_scope("SupersetSqlLabExecutor.resolve_database_id"): client = self._get_client() @@ -76,7 +70,7 @@ class SupersetSqlLabExecutor: for db in databases: if db.get("database_name", "").lower() == database_name.lower(): self._database_id = db["id"] - log.reason("Resolved database ID by name", payload={ + logger.reason("Resolved database ID by name", { "database_name": database_name, "database_id": self._database_id, }) @@ -87,76 +81,42 @@ class SupersetSqlLabExecutor: # Default: use first database self._database_id = databases[0]["id"] - log.reason("Using default database", payload={ + logger.reason("Using default database", { "database_name": databases[0].get("database_name"), "database_id": self._database_id, }) return self._database_id - # #endregion resolve_database_id + # [/DEF:resolve_database_id:Function] - # #region resolve_database_uuid [C:4] [TYPE Function] [SEMANTICS translate,database,uuid] - # @BRIEF Resolve a database UUID to a numeric database ID from Superset. - # @PRE uuid_str is a valid Superset database UUID. - # @POST Returns numeric database_id or raises ValueError if not found. - # @SIDE_EFFECT Fetches databases list from Superset API. - def resolve_database_uuid(self, uuid_str: str) -> int: - with belief_scope("SupersetSqlLabExecutor.resolve_database_uuid"): - client = self._get_client() - _, databases = client.get_databases( - query={"columns": ["id", "uuid", "database_name"]} - ) - if not databases: - raise ValueError("No databases found in Superset environment") - - for db in databases: - db_uuid = db.get("uuid") or "" - if db_uuid.lower() == uuid_str.lower(): - db_id = db["id"] - self._database_id = db_id - log.reason("Resolved database ID by UUID", payload={ - "uuid": uuid_str, - "database_id": db_id, - "database_name": db.get("database_name"), - }) - return db_id - - raise ValueError( - f"Database with UUID '{uuid_str}' not found in Superset environment" - ) - # #endregion resolve_database_uuid - - # #region execute_sql [C:4] [TYPE Function] [SEMANTICS translate,sql,execute] - # @BRIEF Submit SQL to Superset SQL Lab and return execution tracking info. - # @PRE sql is valid SQL string. database_id is a valid Superset DB ID. - # @POST Returns execution result dict with query_id and status. - # @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/. + # [DEF:execute_sql:Function] + # @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info. + # @PRE: sql is valid SQL string. database_id is a valid Superset DB ID. + # @POST: Returns execution result dict with query_id and status. + # @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/. def execute_sql( self, sql: str, - database_id: Optional[Union[int, str]] = None, + database_id: Optional[int] = None, + run_async: bool = True, ) -> Dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.execute_sql"): client = self._get_client() db_id = database_id or self._database_id - if db_id is None: + if not db_id: db_id = self.resolve_database_id() - # Handle UUID string passed as database_id — resolve to numeric ID - if isinstance(db_id, str): - try: - db_id = int(db_id) - except (ValueError, TypeError): - db_id = self.resolve_database_uuid(db_id) - log.reason("Submitting SQL to Superset SQL Lab", payload={ + logger.reason("Submitting SQL to Superset SQL Lab", { "database_id": db_id, "sql_length": len(sql), + "run_async": run_async, }) payload = { "database_id": db_id, "sql": sql, - "client_id": uuid.uuid4().hex[:10], + "runAsync": run_async, "schema": None, + "tab": "translation-insert", } try: @@ -167,25 +127,17 @@ class SupersetSqlLabExecutor: headers={"Content-Type": "application/json"}, ) except Exception as e: - log.explore("SQL Lab execute failed", error=str(e)) + logger.explore("SQL Lab execute failed", {"error": str(e)}) raise ValueError(f"Superset SQL Lab execute failed: {e}") - # Parse response — try multiple formats + # Parse response result = response if isinstance(response, dict) else {} - query_id = (result.get("query_id") - or result.get("id") - or result.get("sql_lab_session_ref") - or (result.get("result") or {}).get("query_id") - or (result.get("result") or {}).get("id")) + query_id = result.get("query_id") or result.get("id") status = result.get("status", "unknown") - log.reason("SQL Lab execute response", payload={ + logger.reason("SQL Lab execute response", { "query_id": query_id, "status": status, - "has_query_id": query_id is not None, - "has_errors": "errors" in result or "error" in result, - "response_keys": list(result.keys()) if isinstance(result, dict) else type(result).__name__, - "response_preview": str(result)[:1000] if isinstance(result, dict) else str(result)[:1000], }) return { @@ -194,13 +146,13 @@ class SupersetSqlLabExecutor: "raw_response": result, "database_id": db_id, } - # #endregion execute_sql + # [/DEF:execute_sql:Function] - # #region poll_execution_status [C:4] [TYPE Function] [SEMANTICS translate,poll,status] - # @BRIEF Poll Superset for SQL execution status until completion or timeout. - # @PRE query_id is a valid Superset query ID. - # @POST Returns final execution status dict with success/failure/timeout. - # @SIDE_EFFECT Makes HTTP GET requests to Superset API. + # [DEF:poll_execution_status:Function] + # @PURPOSE: Poll Superset for SQL execution status until completion or timeout. + # @PRE: query_id is a valid Superset query ID. + # @POST: Returns final execution status dict. + # @SIDE_EFFECT: Makes HTTP GET requests to Superset. def poll_execution_status( self, query_id: str, @@ -210,7 +162,7 @@ class SupersetSqlLabExecutor: with belief_scope("SupersetSqlLabExecutor.poll_execution_status"): client = self._get_client() - log.reason("Polling SQL execution status", payload={ + logger.reason("Polling SQL execution status", { "query_id": query_id, "max_polls": max_polls, "interval": poll_interval_seconds, @@ -228,7 +180,7 @@ class SupersetSqlLabExecutor: # Terminal states if state in ("success", "finished", "completed"): - log.reason("SQL execution completed", payload={ + logger.reason("SQL execution completed", { "query_id": query_id, "attempt": attempt + 1, "rows_affected": result.get("rows"), @@ -245,8 +197,7 @@ class SupersetSqlLabExecutor: if state in ("failed", "error", "stopped"): error_msg = result.get("error_message", "Unknown error") - log.explore("SQL execution failed", error="SQL query execution returned failed/error state", - payload={ + logger.explore("SQL execution failed", { "query_id": query_id, "state": state, "error": error_msg, @@ -267,8 +218,7 @@ class SupersetSqlLabExecutor: time.sleep(poll_interval_seconds) except Exception as e: - log.explore("Polling error, retrying", error="Polling encountered error, will retry", - payload={ + logger.explore("Polling error, retrying", { "query_id": query_id, "attempt": attempt + 1, "error": str(e), @@ -277,7 +227,7 @@ class SupersetSqlLabExecutor: continue # Timeout - log.explore("SQL execution polling timed out", error="Polling timed out", payload={ + logger.explore("SQL execution polling timed out", { "query_id": query_id, "max_polls": max_polls, }) @@ -287,17 +237,17 @@ class SupersetSqlLabExecutor: "error_message": f"Polling timed out after {max_polls} attempts", "results": None, } - # #endregion poll_execution_status + # [/DEF:poll_execution_status:Function] - # #region execute_and_poll [C:4] [TYPE Function] [SEMANTICS translate,sql,execute,poll] - # @BRIEF Execute SQL and wait for completion. One-shot convenience method. - # @PRE sql is valid SQL. - # @POST Returns final execution result. - # @SIDE_EFFECT Makes HTTP calls to Superset API. + # [DEF:execute_and_poll:Function] + # @PURPOSE: Execute SQL and wait for completion. One-shot convenience method. + # @PRE: sql is valid SQL. + # @POST: Returns final execution result. + # @SIDE_EFFECT: Makes HTTP calls to Superset API. def execute_and_poll( self, sql: str, - database_id: Optional[Union[int, str]] = None, + database_id: Optional[int] = None, max_polls: int = 60, poll_interval_seconds: float = 2.0, ) -> Dict[str, Any]: @@ -305,11 +255,12 @@ class SupersetSqlLabExecutor: exec_result = self.execute_sql( sql=sql, database_id=database_id, + run_async=True, ) query_id = exec_result.get("query_id") if not query_id: - log.explore("No query_id from SQL Lab execute", error="No query_id returned", payload={ + logger.explore("No query_id from SQL Lab execute", { "raw_response": exec_result.get("raw_response"), }) return { @@ -323,13 +274,13 @@ class SupersetSqlLabExecutor: max_polls=max_polls, poll_interval_seconds=poll_interval_seconds, ) - # #endregion execute_and_poll + # [/DEF:execute_and_poll:Function] - # #region get_query_results [C:4] [TYPE Function] [SEMANTICS translate,query,results] - # @BRIEF Fetch the results of a completed query from Superset. - # @PRE query_id is a valid Superset query ID. - # @POST Returns query results if available, or error dict. - # @SIDE_EFFECT Makes HTTP GET to Superset API. + # [DEF:get_query_results:Function] + # @PURPOSE: Fetch the results of a completed query. + # @PRE: query_id is a valid Superset query ID. + # @POST: Returns query results if available. + # @SIDE_EFFECT: Makes HTTP GET to Superset. def get_query_results(self, query_id: str) -> Dict[str, Any]: with belief_scope("SupersetSqlLabExecutor.get_query_results"): client = self._get_client() @@ -345,8 +296,7 @@ class SupersetSqlLabExecutor: "results": result.get("results"), } except Exception as e: - log.explore("Failed to fetch query results", error="Failed to fetch query results from Superset", - payload={ + logger.explore("Failed to fetch query results", { "query_id": query_id, "error": str(e), }) @@ -356,7 +306,7 @@ class SupersetSqlLabExecutor: "error_message": str(e), "results": None, } - # #endregion get_query_results + # [/DEF:get_query_results:Function] # #endregion SupersetSqlLabExecutor diff --git a/backend/src/schemas/__init__.py b/backend/src/schemas/__init__.py index 1fc9bb5f..82262ade 100644 --- a/backend/src/schemas/__init__.py +++ b/backend/src/schemas/__init__.py @@ -1,2 +1,3 @@ -# #region SchemasPackage [C:1] [TYPE Package] +# #region SchemasPackage [TYPE Package] +# @BRIEF API schema package root. # #endregion SchemasPackage diff --git a/backend/src/schemas/__tests__/test_settings_and_health_schemas.py b/backend/src/schemas/__tests__/test_settings_and_health_schemas.py index 212703f0..5aeff6aa 100644 --- a/backend/src/schemas/__tests__/test_settings_and_health_schemas.py +++ b/backend/src/schemas/__tests__/test_settings_and_health_schemas.py @@ -1,6 +1,7 @@ -# #region TestSettingsAndHealthSchemas [C:3] [TYPE Module] -# @BRIEF Regression tests for settings and health schema contracts updated in 026 fix batch. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:TestSettingsAndHealthSchemas:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @PURPOSE: Regression tests for settings and health schema contracts updated in 026 fix batch. import pytest from pydantic import ValidationError @@ -9,9 +10,9 @@ from src.schemas.health import DashboardHealthItem from src.schemas.settings import ValidationPolicyCreate -# #region test_validation_policy_create_accepts_structured_custom_channels [TYPE Function] -# @BRIEF Ensure policy schema accepts structured custom channel objects with type/target fields. -# @RELATION BINDS_TO -> [TestSettingsAndHealthSchemas] +# [DEF:test_validation_policy_create_accepts_structured_custom_channels:Function] +# @RELATION: BINDS_TO -> TestSettingsAndHealthSchemas +# @PURPOSE: Ensure policy schema accepts structured custom channel objects with type/target fields. def test_validation_policy_create_accepts_structured_custom_channels(): payload = { "name": "Daily Health", @@ -31,12 +32,12 @@ def test_validation_policy_create_accepts_structured_custom_channels(): assert len(policy.custom_channels) == 1 assert policy.custom_channels[0].type == "SLACK" assert policy.custom_channels[0].target == "#alerts" -# #endregion test_validation_policy_create_accepts_structured_custom_channels +# [/DEF:test_validation_policy_create_accepts_structured_custom_channels:Function] -# #region test_validation_policy_create_rejects_legacy_string_custom_channels [TYPE Function] -# @BRIEF Ensure legacy list[str] custom channel payload is rejected by typed channel contract. -# @RELATION BINDS_TO -> [TestSettingsAndHealthSchemas] +# [DEF:test_validation_policy_create_rejects_legacy_string_custom_channels:Function] +# @RELATION: BINDS_TO -> TestSettingsAndHealthSchemas +# @PURPOSE: Ensure legacy list[str] custom channel payload is rejected by typed channel contract. def test_validation_policy_create_rejects_legacy_string_custom_channels(): payload = { "name": "Daily Health", @@ -51,12 +52,12 @@ def test_validation_policy_create_rejects_legacy_string_custom_channels(): with pytest.raises(ValidationError): ValidationPolicyCreate(**payload) -# #endregion test_validation_policy_create_rejects_legacy_string_custom_channels +# [/DEF:test_validation_policy_create_rejects_legacy_string_custom_channels:Function] -# #region test_dashboard_health_item_status_accepts_only_whitelisted_values [TYPE Function] -# @BRIEF Verify strict grouped regex only accepts PASS/WARN/FAIL/UNKNOWN exact statuses. -# @RELATION BINDS_TO -> [TestSettingsAndHealthSchemas] +# [DEF:test_dashboard_health_item_status_accepts_only_whitelisted_values:Function] +# @RELATION: BINDS_TO -> TestSettingsAndHealthSchemas +# @PURPOSE: Verify strict grouped regex only accepts PASS/WARN/FAIL/UNKNOWN exact statuses. def test_dashboard_health_item_status_accepts_only_whitelisted_values(): valid = DashboardHealthItem( dashboard_id="dash-1", @@ -81,7 +82,7 @@ def test_dashboard_health_item_status_accepts_only_whitelisted_values(): status="FAIL ", last_check="2026-03-10T10:00:00", ) -# #endregion test_dashboard_health_item_status_accepts_only_whitelisted_values +# [/DEF:test_dashboard_health_item_status_accepts_only_whitelisted_values:Function] -# #endregion TestSettingsAndHealthSchemas +# [/DEF:TestSettingsAndHealthSchemas:Module] \ No newline at end of file diff --git a/backend/src/schemas/auth.py b/backend/src/schemas/auth.py index cd6b91b0..3fc5a876 100644 --- a/backend/src/schemas/auth.py +++ b/backend/src/schemas/auth.py @@ -1,16 +1,14 @@ # #region AuthSchemas [C:3] [TYPE Module] [SEMANTICS auth, schemas, pydantic, user, token] +# # @BRIEF Pydantic schemas for authentication requests and responses. -# @LAYER API -# @INVARIANT Sensitive fields like password must not be included in response schemas. -# @RELATION DEPENDS_ON -> [pydantic] -# +# @LAYER: API +# @RELATION DEPENDS_ON -> pydantic # +# @INVARIANT: Sensitive fields like password must not be included in response schemas. -# [SECTION: IMPORTS] from typing import List, Optional from pydantic import BaseModel, EmailStr from datetime import datetime -# [/SECTION] # #region Token [C:1] [TYPE Class] diff --git a/backend/src/schemas/dataset_review.py b/backend/src/schemas/dataset_review.py index 2ac2a4f0..59820758 100644 --- a/backend/src/schemas/dataset_review.py +++ b/backend/src/schemas/dataset_review.py @@ -1,8 +1,8 @@ # #region DatasetReviewSchemas [C:2] [TYPE Module] [SEMANTICS dataset_review, schemas, pydantic, session, profile, findings] # @BRIEF Thin facade re-exporting all dataset review API schemas from decomposed sub-modules. -# @LAYER API -# @RATIONALE Original 419-line file exceeded INV_7 (400-line module limit). Decomposed into DTO and composite sub-modules. -# @REJECTED Keeping all schemas in a single file because it exceeded the fractal limit. +# @LAYER: API +# @RATIONALE: Original 419-line file exceeded INV_7 (400-line module limit). Decomposed into DTO and composite sub-modules. +# @REJECTED: Keeping all schemas in a single file because it exceeded the fractal limit. from src.schemas.dataset_review_pkg._dtos import ( # noqa: F401 SessionCollaboratorDto, diff --git a/backend/src/schemas/dataset_review_pkg/_composites.py b/backend/src/schemas/dataset_review_pkg/_composites.py index e36b5372..b0ddc992 100644 --- a/backend/src/schemas/dataset_review_pkg/_composites.py +++ b/backend/src/schemas/dataset_review_pkg/_composites.py @@ -1,6 +1,6 @@ # #region DatasetReviewSchemaComposites [C:2] [TYPE Module] # @BRIEF Composite Pydantic DTOs for clarification, preview, run context, and session summary/detail responses. -# @LAYER API +# @LAYER: API # @RELATION DEPENDS_ON -> [DatasetReviewSchemaDtos] from datetime import datetime diff --git a/backend/src/schemas/dataset_review_pkg/_dtos.py b/backend/src/schemas/dataset_review_pkg/_dtos.py index 1b2b4a78..73bebc0b 100644 --- a/backend/src/schemas/dataset_review_pkg/_dtos.py +++ b/backend/src/schemas/dataset_review_pkg/_dtos.py @@ -1,6 +1,6 @@ # #region DatasetReviewSchemaDtos [C:2] [TYPE Module] # @BRIEF Pydantic DTOs for session, profile, findings, collaborators, and semantic field API payloads. -# @LAYER API +# @LAYER: API # @RELATION DEPENDS_ON -> [DatasetReviewModels] from datetime import datetime diff --git a/backend/src/schemas/health.py b/backend/src/schemas/health.py index d4bfce19..97152b69 100644 --- a/backend/src/schemas/health.py +++ b/backend/src/schemas/health.py @@ -1,7 +1,7 @@ # #region HealthSchemas [C:3] [TYPE Module] [SEMANTICS health, schemas, pydantic] # @BRIEF Pydantic schemas for dashboard health summary. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [pydantic] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> pydantic from pydantic import BaseModel, Field from typing import List, Optional diff --git a/backend/src/schemas/profile.py b/backend/src/schemas/profile.py index db4e7436..624ea272 100644 --- a/backend/src/schemas/profile.py +++ b/backend/src/schemas/profile.py @@ -1,21 +1,19 @@ # #region ProfileSchemas [C:3] [TYPE Module] [SEMANTICS profile, schemas, pydantic, preferences, superset, lookup, security, git, ux] +# # @BRIEF Defines API schemas for profile preference persistence, security read-only snapshot, and Superset account lookup. -# @LAYER API -# @INVARIANT Schema shapes stay stable for profile UI states and backend preference contracts. -# @RELATION DEPENDS_ON -> [pydantic] -# +# @LAYER: API +# @RELATION DEPENDS_ON -> pydantic # +# @INVARIANT: Schema shapes stay stable for profile UI states and backend preference contracts. -# [SECTION: IMPORTS] from datetime import datetime from typing import List, Literal, Optional from pydantic import BaseModel, Field -# [/SECTION] # #region ProfilePermissionState [C:3] [TYPE Class] # @BRIEF Represents one permission badge state for profile read-only security view. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePermissionState(BaseModel): key: str allowed: bool @@ -26,7 +24,7 @@ class ProfilePermissionState(BaseModel): # #region ProfileSecuritySummary [C:3] [TYPE Class] # @BRIEF Read-only security and access snapshot for current user. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class ProfileSecuritySummary(BaseModel): read_only: bool = True auth_source: Optional[str] = None @@ -41,7 +39,7 @@ class ProfileSecuritySummary(BaseModel): # #region ProfilePreference [C:3] [TYPE Class] # @BRIEF Represents persisted profile preference for a single authenticated user. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreference(BaseModel): user_id: str superset_username: Optional[str] = None @@ -74,7 +72,7 @@ class ProfilePreference(BaseModel): # #region ProfilePreferenceUpdateRequest [C:3] [TYPE Class] # @BRIEF Request payload for updating current user's profile settings. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceUpdateRequest(BaseModel): superset_username: Optional[str] = Field( default=None, @@ -135,7 +133,7 @@ class ProfilePreferenceUpdateRequest(BaseModel): # #region ProfilePreferenceResponse [C:3] [TYPE Class] # @BRIEF Response envelope for profile preference read/update endpoints. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class ProfilePreferenceResponse(BaseModel): status: Literal["success", "error"] = "success" message: Optional[str] = None @@ -149,7 +147,7 @@ class ProfilePreferenceResponse(BaseModel): # #region SupersetAccountLookupRequest [C:3] [TYPE Class] # @BRIEF Query contract for Superset account lookup by selected environment. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountLookupRequest(BaseModel): environment_id: str search: Optional[str] = None @@ -164,7 +162,7 @@ class SupersetAccountLookupRequest(BaseModel): # #region SupersetAccountCandidate [C:3] [TYPE Class] # @BRIEF Canonical account candidate projected from Superset users payload. -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountCandidate(BaseModel): environment_id: str username: str @@ -178,7 +176,7 @@ class SupersetAccountCandidate(BaseModel): # #region SupersetAccountLookupResponse [C:3] [TYPE Class] # @BRIEF Response envelope for Superset account lookup (success or degraded mode). -# @RELATION DEPENDS_ON -> [ProfileSchemas] +# @RELATION DEPENDS_ON -> ProfileSchemas class SupersetAccountLookupResponse(BaseModel): status: Literal["success", "degraded"] environment_id: str diff --git a/backend/src/schemas/settings.py b/backend/src/schemas/settings.py index 02001cd5..1c4bb6a5 100644 --- a/backend/src/schemas/settings.py +++ b/backend/src/schemas/settings.py @@ -1,7 +1,7 @@ # #region SettingsSchemas [C:3] [TYPE Module] [SEMANTICS settings, schemas, pydantic, validation] # @BRIEF Pydantic schemas for application settings and automation policies. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [pydantic] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> pydantic from pydantic import BaseModel, Field from typing import List, Optional diff --git a/backend/src/schemas/translate.py b/backend/src/schemas/translate.py index 67ed6b2a..c1546588 100644 --- a/backend/src/schemas/translate.py +++ b/backend/src/schemas/translate.py @@ -1,6 +1,7 @@ -# #region TranslateSchemas [C:2] [TYPE Module] +# #region TranslateSchemas [C:2] [TYPE Module] [SEMANTICS translate, schemas, pydantic] # @BRIEF Pydantic v2 schemas for translation API request/response serialization. -# @LAYER Domain +# @LAYER: API +# @RELATION DEPENDS_ON -> pydantic from datetime import datetime from typing import Any, Dict, List, Optional @@ -8,7 +9,8 @@ from pydantic import BaseModel, Field import json -# #region TranslateJobCreate [C:1] [TYPE Class] +# #region TranslateJobCreate [TYPE Class] +# @BRIEF Schema for creating a new translation job. class TranslateJobCreate(BaseModel): name: str description: Optional[str] = None @@ -29,11 +31,11 @@ class TranslateJobCreate(BaseModel): upsert_strategy: str = Field("MERGE", description="UPSERT strategy: MERGE, INSERT, UPDATE") dictionary_ids: Optional[List[str]] = Field(default_factory=list, description="Associated terminology dictionary IDs") environment_id: Optional[str] = Field(None, description="Superset environment ID") - target_database_id: Optional[str] = Field(None, description="Superset database ID for INSERT via SQL Lab") # #endregion TranslateJobCreate -# #region TranslateJobUpdate [C:1] [TYPE Class] +# #region TranslateJobUpdate [TYPE Class] +# @BRIEF Schema for updating an existing translation job. class TranslateJobUpdate(BaseModel): name: Optional[str] = None description: Optional[str] = None @@ -55,11 +57,11 @@ class TranslateJobUpdate(BaseModel): status: Optional[str] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None - target_database_id: Optional[str] = None # #endregion TranslateJobUpdate -# #region TranslateJobResponse [C:1] [TYPE Class] +# #region TranslateJobResponse [TYPE Class] +# @BRIEF Schema for translation job API responses. class TranslateJobResponse(BaseModel): id: str name: str @@ -85,24 +87,25 @@ class TranslateJobResponse(BaseModel): updated_at: Optional[datetime] = None dictionary_ids: Optional[List[str]] = None environment_id: Optional[str] = None - target_database_id: Optional[str] = None class Config: from_attributes = True # #endregion TranslateJobResponse -# #region DatasourceColumnResponse [C:1] [TYPE Class] +# #region DatasourceColumnResponse [TYPE Class] +# @BRIEF Schema for datasource column metadata response. class DatasourceColumnResponse(BaseModel): name: str type: Optional[str] = None is_physical: bool = True is_dttm: bool = False description: Optional[str] = None + # #endregion DatasourceColumnResponse - -# #region DatasourceColumnsResponse [C:1] [TYPE Class] +# #region DatasourceColumnsResponse [TYPE Class] +# @BRIEF Schema for datasource columns endpoint response. class DatasourceColumnsResponse(BaseModel): datasource_id: int datasource_name: Optional[str] = None @@ -115,7 +118,8 @@ class DatasourceColumnsResponse(BaseModel): # #endregion DatasourceColumnsResponse -# #region DuplicateJobResponse [C:1] [TYPE Class] +# #region DuplicateJobResponse [TYPE Class] +# @BRIEF Schema for duplicate job response. class DuplicateJobResponse(BaseModel): id: str name: str @@ -126,7 +130,8 @@ class DuplicateJobResponse(BaseModel): # #endregion DuplicateJobResponse -# #region DictionaryCreate [C:1] [TYPE Class] +# #region DictionaryCreate [TYPE Class] +# @BRIEF Schema for creating a new terminology dictionary. class DictionaryCreate(BaseModel): name: str description: Optional[str] = None @@ -136,7 +141,8 @@ class DictionaryCreate(BaseModel): # #endregion DictionaryCreate -# #region DictionaryImport [C:1] [TYPE Class] +# #region DictionaryImport [TYPE Class] +# @BRIEF Schema for importing entries into a terminology dictionary. class DictionaryImport(BaseModel): content: str = Field(..., description="CSV or TSV content as raw string") delimiter: Optional[str] = Field(None, description="Detected or forced delimiter: ',' or '\\t'. Auto-detect if omitted.") @@ -145,7 +151,8 @@ class DictionaryImport(BaseModel): # #endregion DictionaryImport -# #region DictionaryResponse [C:1] [TYPE Class] +# #region DictionaryResponse [TYPE Class] +# @BRIEF Schema for terminology dictionary API responses. class DictionaryResponse(BaseModel): id: str name: str @@ -163,7 +170,8 @@ class DictionaryResponse(BaseModel): # #endregion DictionaryResponse -# #region DictionaryEntryCreate [C:1] [TYPE Class] +# #region DictionaryEntryCreate [TYPE Class] +# @BRIEF Schema for adding/editing a dictionary entry. class DictionaryEntryCreate(BaseModel): source_term: str = Field(..., description="Source term to translate") target_term: str = Field(..., description="Target/translated term") @@ -174,7 +182,8 @@ class DictionaryEntryCreate(BaseModel): # #endregion DictionaryEntryCreate -# #region DictionaryEntryResponse [C:1] [TYPE Class] +# #region DictionaryEntryResponse [TYPE Class] +# @BRIEF Schema for dictionary entry API responses. class DictionaryEntryResponse(BaseModel): id: str dictionary_id: str @@ -190,7 +199,8 @@ class DictionaryEntryResponse(BaseModel): # #endregion DictionaryEntryResponse -# #region DictionaryImportResult [C:1] [TYPE Class] +# #region DictionaryImportResult [TYPE Class] +# @BRIEF Schema for dictionary import result. class DictionaryImportResult(BaseModel): total: int = 0 created: int = 0 @@ -201,7 +211,8 @@ class DictionaryImportResult(BaseModel): # #endregion DictionaryImportResult -# #region PreviewRequest [C:1] [TYPE Class] +# #region PreviewRequest [TYPE Class] +# @BRIEF Schema for triggering a translation preview. class PreviewRequest(BaseModel): sample_size: int = Field(10, ge=1, le=100, description="Number of sample rows to preview") prompt_template: Optional[str] = Field(None, description="Optional custom prompt template") @@ -209,7 +220,8 @@ class PreviewRequest(BaseModel): # #endregion PreviewRequest -# #region PreviewRowUpdate [C:1] [TYPE Class] +# #region PreviewRowUpdate [TYPE Class] +# @BRIEF Schema for approving/editing/rejecting a preview row. class PreviewRowUpdate(BaseModel): action: str = Field(..., description="'approve', 'reject', or 'edit'") translation: Optional[str] = Field(None, description="Edited translation (required for 'edit' action)") @@ -217,7 +229,8 @@ class PreviewRowUpdate(BaseModel): # #endregion PreviewRowUpdate -# #region PreviewAcceptResponse [C:1] [TYPE Class] +# #region PreviewAcceptResponse [TYPE Class] +# @BRIEF Schema for preview accept response. class PreviewAcceptResponse(BaseModel): id: str job_id: str @@ -232,7 +245,8 @@ class PreviewAcceptResponse(BaseModel): # #endregion PreviewAcceptResponse -# #region CostEstimate [C:1] [TYPE Class] +# #region CostEstimate [TYPE Class] +# @BRIEF Schema for cost estimation in preview response. class CostEstimate(BaseModel): sample_size: int = 0 sample_prompt_tokens: int = 0 @@ -245,7 +259,8 @@ class CostEstimate(BaseModel): # #endregion CostEstimate -# #region TermCorrectionSubmit [C:1] [TYPE Class] +# #region TermCorrectionSubmit [TYPE Class] +# @BRIEF Schema for submitting a term correction in a translation preview. class TermCorrectionSubmit(BaseModel): source_term: str incorrect_target_term: str @@ -256,14 +271,16 @@ class TermCorrectionSubmit(BaseModel): # #endregion TermCorrectionSubmit -# #region TermCorrectionBulkSubmit [C:1] [TYPE Class] +# #region TermCorrectionBulkSubmit [TYPE Class] +# @BRIEF Schema for submitting multiple term corrections atomically. class TermCorrectionBulkSubmit(BaseModel): corrections: List[TermCorrectionSubmit] dictionary_id: str = Field(..., description="Target dictionary ID for all corrections") # #endregion TermCorrectionBulkSubmit -# #region CorrectionConflictResult [C:1] [TYPE Class] +# #region CorrectionConflictResult [TYPE Class] +# @BRIEF Schema for reporting conflicts in correction submission. class CorrectionConflictResult(BaseModel): source_term: str existing_target_term: str @@ -272,7 +289,8 @@ class CorrectionConflictResult(BaseModel): # #endregion CorrectionConflictResult -# #region CorrectionSubmitResponse [C:1] [TYPE Class] +# #region CorrectionSubmitResponse [TYPE Class] +# @BRIEF Schema for correction submission response. class CorrectionSubmitResponse(BaseModel): entry_id: Optional[str] = None action: str # "created", "updated", "conflict_detected", "skipped" @@ -283,7 +301,8 @@ class CorrectionSubmitResponse(BaseModel): # #endregion CorrectionSubmitResponse -# #region ScheduleConfig [C:1] [TYPE Class] +# #region ScheduleConfig [TYPE Class] +# @BRIEF Schema for configuring a recurring translation schedule. class ScheduleConfig(BaseModel): cron_expression: str = Field(..., description="Cron expression for scheduling (e.g. '0 2 * * *')") timezone: str = Field("UTC", description="Timezone for the cron schedule (e.g. 'UTC', 'Europe/Moscow')") @@ -291,7 +310,8 @@ class ScheduleConfig(BaseModel): # #endregion ScheduleConfig -# #region ScheduleResponse [C:1] [TYPE Class] +# #region ScheduleResponse [TYPE Class] +# @BRIEF Schema for schedule API responses. class ScheduleResponse(BaseModel): id: str job_id: str @@ -309,7 +329,8 @@ class ScheduleResponse(BaseModel): # #endregion ScheduleResponse -# #region NextExecutionResponse [C:1] [TYPE Class] +# #region NextExecutionResponse [TYPE Class] +# @BRIEF Schema for next execution preview. class NextExecutionResponse(BaseModel): job_id: str cron_expression: str @@ -318,7 +339,8 @@ class NextExecutionResponse(BaseModel): # #endregion NextExecutionResponse -# #region TranslationRunResponse [C:1] [TYPE Class] +# #region TranslationRunResponse [TYPE Class] +# @BRIEF Schema for translation run API responses. class TranslationRunResponse(BaseModel): id: str job_id: str @@ -345,7 +367,8 @@ class TranslationRunResponse(BaseModel): # #endregion TranslationRunResponse -# #region RunDetailResponse [C:1] [TYPE Class] +# #region RunDetailResponse [TYPE Class] +# @BRIEF Schema for detailed run response including records, events, and config snapshot. class RunDetailResponse(BaseModel): run: TranslationRunResponse records: List[Dict[str, Any]] = Field(default_factory=list, description="Paginated translation records") @@ -355,7 +378,8 @@ class RunDetailResponse(BaseModel): # #endregion RunDetailResponse -# #region RunHistoryFilter [C:1] [TYPE Class] +# #region RunHistoryFilter [TYPE Class] +# @BRIEF Schema for filtering run history. class RunHistoryFilter(BaseModel): job_id: Optional[str] = None status: Optional[str] = None @@ -368,7 +392,8 @@ class RunHistoryFilter(BaseModel): # #endregion RunHistoryFilter -# #region RunListResponse [C:1] [TYPE Class] +# #region RunListResponse [TYPE Class] +# @BRIEF Schema for paginated run list response. class RunListResponse(BaseModel): items: List[TranslationRunResponse] = [] total: int = 0 @@ -377,7 +402,8 @@ class RunListResponse(BaseModel): # #endregion RunListResponse -# #region AggregatedMetricsResponse [C:1] [TYPE Class] +# #region AggregatedMetricsResponse [TYPE Class] +# @BRIEF Schema for aggregated per-job metrics. class AggregatedMetricsResponse(BaseModel): job_id: str total_runs: int = 0 @@ -396,7 +422,8 @@ class AggregatedMetricsResponse(BaseModel): # #endregion AggregatedMetricsResponse -# #region PreviewRow [C:1] [TYPE Class] +# #region PreviewRow [TYPE Class] +# @BRIEF A single row in a translation preview showing original and translated content side by side. class PreviewRow(BaseModel): id: str source_sql: Optional[str] = None @@ -409,7 +436,8 @@ class PreviewRow(BaseModel): # #endregion PreviewRow -# #region TranslationPreviewResponse [C:1] [TYPE Class] +# #region TranslationPreviewResponse [TYPE Class] +# @BRIEF Schema for translation preview session API responses. class TranslationPreviewResponse(BaseModel): id: str job_id: str @@ -425,7 +453,8 @@ class TranslationPreviewResponse(BaseModel): # #endregion TranslationPreviewResponse -# #region TranslationBatchResponse [C:1] [TYPE Class] +# #region TranslationBatchResponse [TYPE Class] +# @BRIEF Schema for translation batch API responses. class TranslationBatchResponse(BaseModel): id: str run_id: str @@ -443,7 +472,8 @@ class TranslationBatchResponse(BaseModel): # #endregion TranslationBatchResponse -# #region MetricsResponse [C:1] [TYPE Class] +# #region MetricsResponse [TYPE Class] +# @BRIEF Schema for translation metrics API responses. class MetricsResponse(BaseModel): job_id: str snapshot_date: datetime diff --git a/backend/src/scripts/clean_release_cli.py b/backend/src/scripts/clean_release_cli.py index e17c107c..add91038 100644 --- a/backend/src/scripts/clean_release_cli.py +++ b/backend/src/scripts/clean_release_cli.py @@ -1,7 +1,7 @@ # #region CleanReleaseCliScript [C:3] [TYPE Module] [SEMANTICS cli, clean-release, candidate, artifacts, manifest] # @BRIEF Provide headless CLI commands for candidate registration, artifact import and manifest build. -# @LAYER Scripts -# @RELATION CALLS -> [ComplianceOrchestrator] +# @LAYER: Scripts +# @RELATION CALLS -> ComplianceOrchestrator from __future__ import annotations @@ -102,8 +102,8 @@ def build_parser() -> argparse.ArgumentParser: # #region run_candidate_register [TYPE Function] # @BRIEF Register candidate in repository via CLI command. -# @PRE Candidate ID must be unique. -# @POST Candidate is persisted in DRAFT status. +# @PRE: Candidate ID must be unique. +# @POST: Candidate is persisted in DRAFT status. def run_candidate_register(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -131,8 +131,8 @@ def run_candidate_register(args: argparse.Namespace) -> int: # #region run_artifact_import [TYPE Function] # @BRIEF Import single artifact for existing candidate. -# @PRE Candidate must exist. -# @POST Artifact is persisted for candidate. +# @PRE: Candidate must exist. +# @POST: Artifact is persisted for candidate. def run_artifact_import(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -164,8 +164,8 @@ def run_artifact_import(args: argparse.Namespace) -> int: # #region run_manifest_build [TYPE Function] # @BRIEF Build immutable manifest snapshot for candidate. -# @PRE Candidate must exist. -# @POST New manifest version is persisted. +# @PRE: Candidate must exist. +# @POST: New manifest version is persisted. def run_manifest_build(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository from ..services.clean_release.manifest_service import build_manifest_snapshot @@ -198,8 +198,8 @@ def run_manifest_build(args: argparse.Namespace) -> int: # #region run_compliance_run [TYPE Function] # @BRIEF Execute compliance run for candidate with optional manifest fallback. -# @PRE Candidate exists and trusted snapshots are configured. -# @POST Returns run payload and exit code 0 on success. +# @PRE: Candidate exists and trusted snapshots are configured. +# @POST: Returns run payload and exit code 0 on success. def run_compliance_run(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository, get_config_manager @@ -237,8 +237,8 @@ def run_compliance_run(args: argparse.Namespace) -> int: # #region run_compliance_status [TYPE Function] # @BRIEF Read run status by run id. -# @PRE Run exists. -# @POST Returns run status payload. +# @PRE: Run exists. +# @POST: Returns run status payload. def run_compliance_status(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -269,8 +269,8 @@ def run_compliance_status(args: argparse.Namespace) -> int: # #region _to_payload [TYPE Function] # @BRIEF Serialize domain models for CLI JSON output across SQLAlchemy/Pydantic variants. -# @PRE value is serializable model or primitive object. -# @POST Returns dictionary payload without mutating value. +# @PRE: value is serializable model or primitive object. +# @POST: Returns dictionary payload without mutating value. def _to_payload(value: Any) -> Dict[str, Any]: def _normalize(raw: Any) -> Any: if isinstance(raw, datetime): @@ -299,8 +299,8 @@ def _to_payload(value: Any) -> Dict[str, Any]: # #region run_compliance_report [TYPE Function] # @BRIEF Read immutable report by run id. -# @PRE Run and report exist. -# @POST Returns report payload. +# @PRE: Run and report exist. +# @POST: Returns report payload. def run_compliance_report(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -326,8 +326,8 @@ def run_compliance_report(args: argparse.Namespace) -> int: # #region run_compliance_violations [TYPE Function] # @BRIEF Read run violations by run id. -# @PRE Run exists. -# @POST Returns violations payload. +# @PRE: Run exists. +# @POST: Returns violations payload. def run_compliance_violations(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -351,8 +351,8 @@ def run_compliance_violations(args: argparse.Namespace) -> int: # #region run_approve [TYPE Function] # @BRIEF Approve candidate based on immutable PASSED report. -# @PRE Candidate and report exist; report is PASSED. -# @POST Persists APPROVED decision and returns success payload. +# @PRE: Candidate and report exist; report is PASSED. +# @POST: Persists APPROVED decision and returns success payload. def run_approve(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -382,8 +382,8 @@ def run_approve(args: argparse.Namespace) -> int: # #region run_reject [TYPE Function] # @BRIEF Reject candidate without mutating compliance evidence. -# @PRE Candidate and report exist. -# @POST Persists REJECTED decision and returns success payload. +# @PRE: Candidate and report exist. +# @POST: Persists REJECTED decision and returns success payload. def run_reject(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -413,8 +413,8 @@ def run_reject(args: argparse.Namespace) -> int: # #region run_publish [TYPE Function] # @BRIEF Publish approved candidate to target channel. -# @PRE Candidate is approved and report belongs to candidate. -# @POST Appends ACTIVE publication record and returns payload. +# @PRE: Candidate is approved and report belongs to candidate. +# @POST: Appends ACTIVE publication record and returns payload. def run_publish(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository @@ -441,8 +441,8 @@ def run_publish(args: argparse.Namespace) -> int: # #region run_revoke [TYPE Function] # @BRIEF Revoke active publication record. -# @PRE Publication id exists and is ACTIVE. -# @POST Publication record status becomes REVOKED. +# @PRE: Publication id exists and is ACTIVE. +# @POST: Publication record status becomes REVOKED. def run_revoke(args: argparse.Namespace) -> int: from ..dependencies import get_clean_release_repository diff --git a/backend/src/scripts/clean_release_tui.py b/backend/src/scripts/clean_release_tui.py index 21399984..5c081ef0 100644 --- a/backend/src/scripts/clean_release_tui.py +++ b/backend/src/scripts/clean_release_tui.py @@ -1,9 +1,9 @@ # #region CleanReleaseTuiScript [C:3] [TYPE Module] [SEMANTICS clean-release, tui, ncurses, interactive-validator] # @BRIEF Interactive terminal interface for Enterprise Clean Release compliance validation. -# @LAYER UI -# @INVARIANT TUI refuses startup in non-TTY environments; headless flow is CLI/API only. +# @LAYER: UI # @RELATION DEPENDS_ON -> [ComplianceExecutionService] # @RELATION DEPENDS_ON -> [CleanReleaseRepository] +# @INVARIANT: TUI refuses startup in non-TTY environments; headless flow is CLI/API only. import curses import json @@ -46,8 +46,8 @@ from src.services.clean_release.repository import CleanReleaseRepository # #region TuiFacadeAdapter [TYPE Class] # @BRIEF Thin TUI adapter that routes business mutations through application services. -# @PRE repository contains candidate and trusted policy/registry snapshots for execution. -# @POST Business actions return service results/errors without direct TUI-owned mutations. +# @PRE: repository contains candidate and trusted policy/registry snapshots for execution. +# @POST: Business actions return service results/errors without direct TUI-owned mutations. class TuiFacadeAdapter: def __init__(self, repository: CleanReleaseRepository): self.repository = repository @@ -506,10 +506,10 @@ class CleanReleaseTUI: self.stdscr.addstr(max_y - 1, 0, footer_text[:max_x]) self.stdscr.attroff(curses.color_pair(1)) - # #region run_checks [TYPE Function] - # @BRIEF Execute compliance run via facade adapter and update UI state. - # @PRE Candidate and policy snapshots are present in repository. - # @POST UI reflects final run/report/violation state from service result. + # [DEF:run_checks:Function] + # @PURPOSE: Execute compliance run via facade adapter and update UI state. + # @PRE: Candidate and policy snapshots are present in repository. + # @POST: UI reflects final run/report/violation state from service result. def run_checks(self): self.status = "RUNNING" self.report_id = None @@ -550,7 +550,7 @@ class CleanReleaseTUI: self.refresh_overview() self.refresh_screen() - # #endregion run_checks + # [/DEF:run_checks:Function] def build_manifest(self): try: diff --git a/backend/src/scripts/create_admin.py b/backend/src/scripts/create_admin.py index 33b3442c..9bff20ae 100644 --- a/backend/src/scripts/create_admin.py +++ b/backend/src/scripts/create_admin.py @@ -1,14 +1,13 @@ # #region CreateAdminScript [C:3] [TYPE Module] [SEMANTICS admin, setup, user, auth, cli] +# # @BRIEF CLI tool for creating the initial admin user. -# @LAYER Scripts -# @INVARIANT Admin user must have the "Admin" role. +# @LAYER: Scripts # @RELATION USES -> [AuthSecurityModule] # @RELATION USES -> [DatabaseModule] # @RELATION USES -> [AuthModels] # -# +# @INVARIANT: Admin user must have the "Admin" role. -# [SECTION: IMPORTS] import sys import argparse from pathlib import Path @@ -20,17 +19,13 @@ from src.core.database import AuthSessionLocal, init_db from src.core.auth.security import get_password_hash from src.models.auth import User, Role from src.core.logger import logger, belief_scope -# [/SECTION] # #region create_admin [TYPE Function] # @BRIEF Creates an admin user and necessary roles/permissions. -# @PRE username and password provided via CLI. -# @POST Admin user exists in auth.db. +# @PRE: username and password provided via CLI. +# @POST: Admin user exists in auth.db. # -# @PARAM: username (str) - Admin username. -# @PARAM: password (str) - Admin password. -# @PARAM: email (str | None) - Optional admin email. def create_admin(username, password, email=None): with belief_scope("create_admin"): db = AuthSessionLocal() diff --git a/backend/src/scripts/init_auth_db.py b/backend/src/scripts/init_auth_db.py index aa950bfb..33f77de4 100644 --- a/backend/src/scripts/init_auth_db.py +++ b/backend/src/scripts/init_auth_db.py @@ -1,14 +1,13 @@ # #region InitAuthDbScript [C:2] [TYPE Module] [SEMANTICS setup, database, auth, migration] +# # @BRIEF Initializes the auth database and creates the necessary tables. -# @LAYER Scripts -# @INVARIANT Safe to run multiple times (idempotent). -# @RELATION CALLS -> [init_db] -# @RELATION CALLS -> [ensure_encryption_key] -# @RELATION CALLS -> [seed_permissions] -# +# @LAYER: Scripts +# @RELATION CALLS -> init_db +# @RELATION CALLS -> ensure_encryption_key +# @RELATION CALLS -> seed_permissions # +# @INVARIANT: Safe to run multiple times (idempotent). -# [SECTION: IMPORTS] import sys from pathlib import Path @@ -19,15 +18,14 @@ from src.core.database import init_db from src.core.encryption_key import ensure_encryption_key from src.core.logger import logger, belief_scope from src.scripts.seed_permissions import seed_permissions -# [/SECTION] # #region run_init [C:3] [TYPE Function] # @BRIEF Main entry point for the initialization script. -# @POST auth.db is initialized with the correct schema and seeded permissions. -# @RELATION CALLS -> [ensure_encryption_key] -# @RELATION CALLS -> [init_db] -# @RELATION CALLS -> [seed_permissions] +# @POST: auth.db is initialized with the correct schema and seeded permissions. +# @RELATION CALLS -> ensure_encryption_key +# @RELATION CALLS -> init_db +# @RELATION CALLS -> seed_permissions def run_init(): with belief_scope("init_auth_db"): logger.info("Initializing authentication database...") diff --git a/backend/src/scripts/migrate_sqlite_to_postgres.py b/backend/src/scripts/migrate_sqlite_to_postgres.py index 034be60a..072acd78 100644 --- a/backend/src/scripts/migrate_sqlite_to_postgres.py +++ b/backend/src/scripts/migrate_sqlite_to_postgres.py @@ -1,16 +1,15 @@ # #region MigrateSqliteToPostgresScript [C:3] [TYPE Module] [SEMANTICS migration, sqlite, postgresql, config, task_logs, task_records] +# # @BRIEF Migrates legacy config and task history from SQLite/file storage to PostgreSQL. -# @LAYER Scripts -# @INVARIANT Script is idempotent for task_records and app_configurations. -# @RELATION READS_FROM -> [backend/tasks.db] -# @RELATION READS_FROM -> [backend/config.json] -# @RELATION WRITES_TO -> [postgresql.task_records] -# @RELATION WRITES_TO -> [postgresql.task_logs] -# @RELATION WRITES_TO -> [postgresql.app_configurations] -# +# @LAYER: Scripts +# @RELATION READS_FROM -> backend/tasks.db +# @RELATION READS_FROM -> backend/config.json +# @RELATION WRITES_TO -> postgresql.task_records +# @RELATION WRITES_TO -> postgresql.task_logs +# @RELATION WRITES_TO -> postgresql.app_configurations # +# @INVARIANT: Script is idempotent for task_records and app_configurations. -# [SECTION: IMPORTS] import argparse import json import os @@ -21,11 +20,7 @@ from typing import Any, Dict, Iterable, Optional from sqlalchemy import create_engine, text from sqlalchemy.exc import SQLAlchemyError -from src.core.logger import belief_scope -from src.core.cot_logger import MarkerLogger -log = MarkerLogger("MigrateSqliteToPostgres") - -# [/SECTION] +from src.core.logger import belief_scope, logger # #region Constants [TYPE Section] @@ -41,8 +36,8 @@ DEFAULT_TARGET_URL = os.getenv( # #region _json_load_if_needed [C:3] [TYPE Function] # @BRIEF Parses JSON-like values from SQLite TEXT/JSON columns to Python objects. -# @PRE value is scalar JSON/text/list/dict or None. -# @POST Returns normalized Python object or original scalar value. +# @PRE: value is scalar JSON/text/list/dict or None. +# @POST: Returns normalized Python object or original scalar value. def _json_load_if_needed(value: Any) -> Any: with belief_scope("_json_load_if_needed"): if value is None: @@ -169,7 +164,9 @@ def _ensure_target_schema(engine) -> None: def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int: with belief_scope("_migrate_config"): if legacy_config_path is None: - log.reason("No legacy config.json found, skipping migration") + logger.info( + "[_migrate_config][Action] No legacy config.json found, skipping" + ) return 0 payload = json.loads(legacy_config_path.read_text(encoding="utf-8")) @@ -330,7 +327,9 @@ def run_migration( sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path] ) -> Dict[str, int]: with belief_scope("run_migration"): - log.reason("Starting migration from SQLite to PostgreSQL", payload={"sqlite_path": str(sqlite_path), "target_url": target_url}) + logger.info( + "[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url + ) if not sqlite_path.exists(): raise FileNotFoundError(f"SQLite source not found: {sqlite_path}") diff --git a/backend/src/scripts/seed_permissions.py b/backend/src/scripts/seed_permissions.py index 91eb11d7..e81fa434 100644 --- a/backend/src/scripts/seed_permissions.py +++ b/backend/src/scripts/seed_permissions.py @@ -1,15 +1,14 @@ # #region SeedPermissionsScript [C:3] [TYPE Module] [SEMANTICS setup, database, auth, permissions, seeding] +# # @BRIEF Populates the auth database with initial system permissions. -# @LAYER Scripts -# @INVARIANT Safe to run multiple times (idempotent). -# @RELATION DEPENDS_ON -> [AuthSessionLocal] -# @RELATION DEPENDS_ON -> [Permission] -# @RELATION DEPENDS_ON -> [Role] -# @RELATION DEPENDS_ON -> [AuthRepository] -# +# @LAYER: Scripts +# @RELATION DEPENDS_ON -> AuthSessionLocal +# @RELATION DEPENDS_ON -> Permission +# @RELATION DEPENDS_ON -> Role +# @RELATION DEPENDS_ON -> AuthRepository # +# @INVARIANT: Safe to run multiple times (idempotent). -# [SECTION: IMPORTS] import sys from pathlib import Path @@ -20,11 +19,10 @@ from src.core.database import AuthSessionLocal from src.models.auth import Permission, Role from src.core.auth.repository import AuthRepository from src.core.logger import logger, belief_scope -# [/SECTION] # #region INITIAL_PERMISSIONS [C:3] [TYPE Constant] # @BRIEF Canonical bootstrap permission tuples seeded into auth storage. -# @RELATION DEPENDS_ON -> [SeedPermissionsScript] +# @RELATION DEPENDS_ON -> SeedPermissionsScript INITIAL_PERMISSIONS = [ # Admin Permissions {"resource": "admin:users", "action": "READ"}, @@ -61,12 +59,12 @@ INITIAL_PERMISSIONS = [ # #region seed_permissions [C:3] [TYPE Function] # @BRIEF Inserts missing permissions into the database. -# @POST All INITIAL_PERMISSIONS exist in the DB. -# @RELATION DEPENDS_ON -> [AuthSessionLocal] -# @RELATION DEPENDS_ON -> [Permission] -# @RELATION DEPENDS_ON -> [Role] -# @RELATION DEPENDS_ON -> [AuthRepository] -# @RELATION DEPENDS_ON -> [INITIAL_PERMISSIONS] +# @POST: All INITIAL_PERMISSIONS exist in the DB. +# @RELATION DEPENDS_ON -> AuthSessionLocal +# @RELATION DEPENDS_ON -> Permission +# @RELATION DEPENDS_ON -> Role +# @RELATION DEPENDS_ON -> AuthRepository +# @RELATION DEPENDS_ON -> INITIAL_PERMISSIONS def seed_permissions(): with belief_scope("seed_permissions"): db = AuthSessionLocal() diff --git a/backend/src/scripts/seed_superset_load_test.py b/backend/src/scripts/seed_superset_load_test.py index 2a976b85..b117d785 100644 --- a/backend/src/scripts/seed_superset_load_test.py +++ b/backend/src/scripts/seed_superset_load_test.py @@ -1,12 +1,11 @@ # #region SeedSupersetLoadTestScript [C:3] [TYPE Module] [SEMANTICS superset, load-test, charts, dashboards, seed, stress] +# # @BRIEF Creates randomized load-test data in Superset by cloning chart configurations and creating dashboards in target environments. -# @LAYER Scripts -# @INVARIANT Created chart and dashboard names are globally unique for one script run. +# @LAYER: Scripts # @RELATION USES -> [ConfigManager] # @RELATION USES -> [SupersetClient] -# +# @INVARIANT: Created chart and dashboard names are globally unique for one script run. -# [SECTION: IMPORTS] import argparse import json import random @@ -19,18 +18,14 @@ sys.path.append(str(Path(__file__).parent.parent.parent)) from src.core.config_manager import ConfigManager from src.core.config_models import Environment -from src.core.logger import belief_scope -from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope, logger from src.core.superset_client import SupersetClient -log = MarkerLogger("SeedSupersetLoadTest") - -# [/SECTION] # #region _parse_args [TYPE Function] # @BRIEF Parses CLI arguments for load-test data generation. -# @PRE Script is called from CLI. -# @POST Returns validated argument namespace. +# @PRE: Script is called from CLI. +# @POST: Returns validated argument namespace. def _parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Seed Superset with load-test charts and dashboards" @@ -73,8 +68,8 @@ def _parse_args() -> argparse.Namespace: # #region _extract_result_payload [TYPE Function] # @BRIEF Normalizes Superset API payloads that may be wrapped in `result`. -# @PRE payload is a JSON-decoded API response. -# @POST Returns the unwrapped object when present. +# @PRE: payload is a JSON-decoded API response. +# @POST: Returns the unwrapped object when present. def _extract_result_payload(payload: Dict) -> Dict: result = payload.get("result") if isinstance(result, dict): @@ -87,8 +82,8 @@ def _extract_result_payload(payload: Dict) -> Dict: # #region _extract_created_id [TYPE Function] # @BRIEF Extracts object ID from create/update API response. -# @PRE payload is a JSON-decoded API response. -# @POST Returns integer object ID or None if missing. +# @PRE: payload is a JSON-decoded API response. +# @POST: Returns integer object ID or None if missing. def _extract_created_id(payload: Dict) -> Optional[int]: direct_id = payload.get("id") if isinstance(direct_id, int): @@ -104,8 +99,8 @@ def _extract_created_id(payload: Dict) -> Optional[int]: # #region _generate_unique_name [TYPE Function] # @BRIEF Generates globally unique random names for charts/dashboards. -# @PRE used_names is mutable set for collision tracking. -# @POST Returns a unique string and stores it in used_names. +# @PRE: used_names is mutable set for collision tracking. +# @POST: Returns a unique string and stores it in used_names. def _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) -> str: adjectives = [ "amber", @@ -144,8 +139,8 @@ def _generate_unique_name(prefix: str, used_names: set[str], rng: random.Random) # #region _resolve_target_envs [TYPE Function] # @BRIEF Resolves requested environment IDs from configuration. -# @PRE env_ids is non-empty. -# @POST Returns mapping env_id -> configured environment object. +# @PRE: env_ids is non-empty. +# @POST: Returns mapping env_id -> configured environment object. def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: config_manager = ConfigManager() configured = {env.id: env for env in config_manager.get_environments()} @@ -162,7 +157,9 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: env = Environment(**row) configured[env.id] = env except Exception as exc: - log.explore("Failed loading environments from config", payload={"config_path": str(config_path)}, error=str(exc)) + logger.warning( + f"[REFLECT] Failed loading environments from {config_path}: {exc}" + ) for env_id in env_ids: env = configured.get(env_id) @@ -178,8 +175,8 @@ def _resolve_target_envs(env_ids: List[str]) -> Dict[str, Environment]: # #region _build_chart_template_pool [TYPE Function] # @BRIEF Builds a pool of source chart templates to clone in one environment. -# @PRE Client is authenticated. -# @POST Returns non-empty list of chart payload templates. +# @PRE: Client is authenticated. +# @POST: Returns non-empty list of chart payload templates. def _build_chart_template_pool( client: SupersetClient, pool_size: int, rng: random.Random ) -> List[Dict]: @@ -253,9 +250,9 @@ def _build_chart_template_pool( # #region seed_superset_load_data [TYPE Function] # @BRIEF Creates dashboards and cloned charts for load testing across target environments. -# @PRE Target environments must be reachable and authenticated. -# @POST Returns execution statistics dictionary. -# @SIDE_EFFECT Creates objects in Superset environments. +# @PRE: Target environments must be reachable and authenticated. +# @POST: Returns execution statistics dictionary. +# @SIDE_EFFECT: Creates objects in Superset environments. def seed_superset_load_data(args: argparse.Namespace) -> Dict: rng = random.Random(args.seed) env_map = _resolve_target_envs(args.envs) @@ -274,7 +271,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: templates_by_env[env_id] = _build_chart_template_pool( client, args.template_pool_size, rng ) - log.reason("Loaded chart templates for environment", payload={"env_id": env_id, "template_count": len(templates_by_env[env_id])}) + logger.info( + f"[REASON] Environment {env_id}: loaded {len(templates_by_env[env_id])} chart templates" + ) errors = 0 env_ids = list(env_map.keys()) @@ -286,7 +285,9 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: dashboard_title = _generate_unique_name("lt_dash", used_dashboard_names, rng) if args.dry_run: - log.reflect("Dry-run dashboard create", payload={"env_id": env_id, "title": dashboard_title}) + logger.info( + f"[REFLECT] Dry-run dashboard create: env={env_id}, title={dashboard_title}" + ) continue try: @@ -300,7 +301,7 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: created_dashboards[env_id].append(dashboard_id) except Exception as exc: errors += 1 - log.explore("Failed creating dashboard", payload={"env_id": env_id}, error=str(exc)) + logger.error(f"[EXPLORE] Failed creating dashboard in {env_id}: {exc}") if errors >= args.max_errors: raise RuntimeError( f"Stopping due to max errors reached ({errors})" @@ -350,10 +351,10 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: created_charts[env_id].append(chart_id) if (index + 1) % 500 == 0: - log.reason(f"Created {index + 1}/{args.charts} charts") + logger.info(f"[REASON] Created {index + 1}/{args.charts} charts") except Exception as exc: errors += 1 - log.explore("Failed creating chart", payload={"env_id": env_id}, error=str(exc)) + logger.error(f"[EXPLORE] Failed creating chart in {env_id}: {exc}") if errors >= args.max_errors: raise RuntimeError( f"Stopping due to max errors reached ({errors})" @@ -374,13 +375,15 @@ def seed_superset_load_data(args: argparse.Namespace) -> Dict: # #region main [TYPE Function] # @BRIEF CLI entrypoint for Superset load-test data seeding. -# @PRE Command line arguments are valid. -# @POST Prints summary and exits with non-zero status on failure. +# @PRE: Command line arguments are valid. +# @POST: Prints summary and exits with non-zero status on failure. def main() -> None: with belief_scope("seed_superset_load_test.main"): args = _parse_args() result = seed_superset_load_data(args) - log.reflect("Seed load test complete", payload={"summary": result}) + logger.info( + f"[COHERENCE:OK] Result summary: {json.dumps(result, ensure_ascii=True)}" + ) # #endregion main diff --git a/backend/src/services/__tests__/test_encryption_manager.py b/backend/src/services/__tests__/test_encryption_manager.py index 77d58364..8fc7ea41 100644 --- a/backend/src/services/__tests__/test_encryption_manager.py +++ b/backend/src/services/__tests__/test_encryption_manager.py @@ -1,8 +1,10 @@ -# #region test_encryption_manager [C:3] [TYPE Module] [SEMANTICS encryption, security, fernet, api-keys, tests] -# @BRIEF Unit tests for EncryptionManager encrypt/decrypt functionality. -# @LAYER Domain -# @INVARIANT Encrypt+decrypt roundtrip always returns original plaintext. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:test_encryption_manager:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: encryption, security, fernet, api-keys, tests +# @PURPOSE: Unit tests for EncryptionManager encrypt/decrypt functionality. +# @LAYER: Domain +# @INVARIANT: Encrypt+decrypt roundtrip always returns original plaintext. import sys from pathlib import Path @@ -13,11 +15,11 @@ from unittest.mock import patch from cryptography.fernet import Fernet, InvalidToken -# #region TestEncryptionManager [TYPE Class] -# @BRIEF Validate EncryptionManager encrypt/decrypt roundtrip, uniqueness, and error handling. -# @PRE cryptography package installed. -# @POST All encrypt/decrypt invariants verified. -# @RELATION BINDS_TO -> [test_encryption_manager] +# [DEF:TestEncryptionManager:Class] +# @RELATION: BINDS_TO -> test_encryption_manager +# @PURPOSE: Validate EncryptionManager encrypt/decrypt roundtrip, uniqueness, and error handling. +# @PRE: cryptography package installed. +# @POST: All encrypt/decrypt invariants verified. class TestEncryptionManager: """Tests for the EncryptionManager class.""" @@ -39,10 +41,10 @@ class TestEncryptionManager: return EncryptionManager() - # #region test_encrypt_decrypt_roundtrip [TYPE Function] - # @BRIEF Encrypt then decrypt returns original plaintext. - # @PRE Valid plaintext string. - # @POST Decrypted output equals original input. + # [DEF:test_encrypt_decrypt_roundtrip:Function] + # @PURPOSE: Encrypt then decrypt returns original plaintext. + # @PRE: Valid plaintext string. + # @POST: Decrypted output equals original input. def test_encrypt_decrypt_roundtrip(self): mgr = self._make_manager() original = "my-secret-api-key-12345" @@ -50,69 +52,69 @@ class TestEncryptionManager: assert encrypted != original decrypted = mgr.decrypt(encrypted) assert decrypted == original - # #endregion test_encrypt_decrypt_roundtrip + # [/DEF:test_encrypt_decrypt_roundtrip:Function] - # #region test_encrypt_produces_different_output [TYPE Function] - # @BRIEF Same plaintext produces different ciphertext (Fernet uses random IV). - # @PRE Two encrypt calls with same input. - # @POST Ciphertexts differ but both decrypt to same value. + # [DEF:test_encrypt_produces_different_output:Function] + # @PURPOSE: Same plaintext produces different ciphertext (Fernet uses random IV). + # @PRE: Two encrypt calls with same input. + # @POST: Ciphertexts differ but both decrypt to same value. def test_encrypt_produces_different_output(self): mgr = self._make_manager() ct1 = mgr.encrypt("same-key") ct2 = mgr.encrypt("same-key") assert ct1 != ct2 assert mgr.decrypt(ct1) == mgr.decrypt(ct2) == "same-key" - # #endregion test_encrypt_produces_different_output + # [/DEF:test_encrypt_produces_different_output:Function] - # #region test_different_inputs_yield_different_ciphertext [TYPE Function] - # @BRIEF Different inputs produce different ciphertexts. - # @PRE Two different plaintext values. - # @POST Encrypted outputs differ. + # [DEF:test_different_inputs_yield_different_ciphertext:Function] + # @PURPOSE: Different inputs produce different ciphertexts. + # @PRE: Two different plaintext values. + # @POST: Encrypted outputs differ. def test_different_inputs_yield_different_ciphertext(self): mgr = self._make_manager() ct1 = mgr.encrypt("key-one") ct2 = mgr.encrypt("key-two") assert ct1 != ct2 - # #endregion test_different_inputs_yield_different_ciphertext + # [/DEF:test_different_inputs_yield_different_ciphertext:Function] - # #region test_decrypt_invalid_data_raises [TYPE Function] - # @BRIEF Decrypting invalid data raises InvalidToken. - # @PRE Invalid ciphertext string. - # @POST Exception raised. + # [DEF:test_decrypt_invalid_data_raises:Function] + # @PURPOSE: Decrypting invalid data raises InvalidToken. + # @PRE: Invalid ciphertext string. + # @POST: Exception raised. def test_decrypt_invalid_data_raises(self): mgr = self._make_manager() with pytest.raises(Exception): mgr.decrypt("not-a-valid-fernet-token") - # #endregion test_decrypt_invalid_data_raises + # [/DEF:test_decrypt_invalid_data_raises:Function] - # #region test_encrypt_empty_string [TYPE Function] - # @BRIEF Encrypting and decrypting an empty string works. - # @PRE Empty string input. - # @POST Decrypted output equals empty string. + # [DEF:test_encrypt_empty_string:Function] + # @PURPOSE: Encrypting and decrypting an empty string works. + # @PRE: Empty string input. + # @POST: Decrypted output equals empty string. def test_encrypt_empty_string(self): mgr = self._make_manager() encrypted = mgr.encrypt("") assert encrypted decrypted = mgr.decrypt(encrypted) assert decrypted == "" - # #endregion test_encrypt_empty_string + # [/DEF:test_encrypt_empty_string:Function] - # #region test_missing_key_fails_fast [TYPE Function] - # @BRIEF Missing ENCRYPTION_KEY must abort initialization instead of using a fallback secret. - # @PRE ENCRYPTION_KEY is unset. - # @POST RuntimeError raised during EncryptionManager construction. + # [DEF:test_missing_key_fails_fast:Function] + # @PURPOSE: Missing ENCRYPTION_KEY must abort initialization instead of using a fallback secret. + # @PRE: ENCRYPTION_KEY is unset. + # @POST: RuntimeError raised during EncryptionManager construction. def test_missing_key_fails_fast(self): from src.services.llm_provider import EncryptionManager with patch.dict("os.environ", {}, clear=True): with pytest.raises(RuntimeError, match="ENCRYPTION_KEY must be set"): EncryptionManager() - # #endregion test_missing_key_fails_fast + # [/DEF:test_missing_key_fails_fast:Function] - # #region test_custom_key_roundtrip [TYPE Function] - # @BRIEF Custom Fernet key produces valid roundtrip. - # @PRE Generated Fernet key. - # @POST Encrypt/decrypt with custom key succeeds. + # [DEF:test_custom_key_roundtrip:Function] + # @PURPOSE: Custom Fernet key produces valid roundtrip. + # @PRE: Generated Fernet key. + # @POST: Encrypt/decrypt with custom key succeeds. def test_custom_key_roundtrip(self): custom_key = Fernet.generate_key() fernet = Fernet(custom_key) @@ -130,7 +132,7 @@ class TestEncryptionManager: encrypted = mgr.encrypt("test-with-custom-key") decrypted = mgr.decrypt(encrypted) assert decrypted == "test-with-custom-key" - # #endregion test_custom_key_roundtrip + # [/DEF:test_custom_key_roundtrip:Function] -# #endregion TestEncryptionManager -# #endregion test_encryption_manager +# [/DEF:TestEncryptionManager:Class] +# [/DEF:test_encryption_manager:Module] diff --git a/backend/src/services/__tests__/test_health_service.py b/backend/src/services/__tests__/test_health_service.py index a9f925b7..6903c552 100644 --- a/backend/src/services/__tests__/test_health_service.py +++ b/backend/src/services/__tests__/test_health_service.py @@ -4,9 +4,10 @@ from unittest.mock import MagicMock, patch from src.services.health_service import HealthService from src.models.llm import ValidationRecord -# #region test_health_service [C:3] [TYPE Module] -# @BRIEF Unit tests for HealthService aggregation logic. -# @RELATION VERIFIES -> [src.services.health_service.HealthService] +# [DEF:test_health_service:Module] +# @COMPLEXITY: 3 +# @PURPOSE: Unit tests for HealthService aggregation logic. +# @RELATION: VERIFIES ->[src.services.health_service.HealthService] @pytest.mark.asyncio @@ -161,9 +162,9 @@ async def test_get_health_summary_reuses_dashboard_metadata_cache_across_service HealthService._dashboard_summary_cache.clear() -# #region test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks [TYPE Function] -# @BRIEF Verify that deleting a validation report also removes dashboard scope and linked tasks. -# @RELATION BINDS_TO -> [test_health_service] +# [DEF:test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks:Function] +# @RELATION: BINDS_TO ->[test_health_service] +# @PURPOSE: Verify that deleting a validation report also removes dashboard scope and linked tasks. def test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks(): db = MagicMock() config_manager = MagicMock() @@ -234,12 +235,12 @@ def test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks(): assert "task-3" in task_manager.tasks -# #endregion test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks +# [/DEF:test_delete_validation_report_deletes_dashboard_scope_and_linked_tasks:Function] -# #region test_delete_validation_report_returns_false_for_unknown_record [TYPE Function] -# @BRIEF Verify delete returns False when validation record does not exist. -# @RELATION BINDS_TO -> [test_health_service] +# [DEF:test_delete_validation_report_returns_false_for_unknown_record:Function] +# @RELATION: BINDS_TO ->[test_health_service] +# @PURPOSE: Verify delete returns False when validation record does not exist. def test_delete_validation_report_returns_false_for_unknown_record(): db = MagicMock() db.query.return_value.filter.return_value.first.return_value = None @@ -249,12 +250,12 @@ def test_delete_validation_report_returns_false_for_unknown_record(): assert service.delete_validation_report("missing") is False -# #endregion test_delete_validation_report_returns_false_for_unknown_record +# [/DEF:test_delete_validation_report_returns_false_for_unknown_record:Function] -# #region test_delete_validation_report_swallows_linked_task_cleanup_failure [TYPE Function] -# @BRIEF Verify delete swallows exceptions when cleaning up linked tasks. -# @RELATION BINDS_TO -> [test_health_service] +# [DEF:test_delete_validation_report_swallows_linked_task_cleanup_failure:Function] +# @RELATION: BINDS_TO ->[test_health_service] +# @PURPOSE: Verify delete swallows exceptions when cleaning up linked tasks. def test_delete_validation_report_swallows_linked_task_cleanup_failure(): db = MagicMock() config_manager = MagicMock() @@ -304,5 +305,5 @@ def test_delete_validation_report_swallows_linked_task_cleanup_failure(): assert "task-1" not in task_manager.tasks -# #endregion test_delete_validation_report_swallows_linked_task_cleanup_failure -# #endregion test_health_service +# [/DEF:test_delete_validation_report_swallows_linked_task_cleanup_failure:Function] +# [/DEF:test_health_service:Module] diff --git a/backend/src/services/__tests__/test_llm_plugin_persistence.py b/backend/src/services/__tests__/test_llm_plugin_persistence.py index a0b9a838..1050a312 100644 --- a/backend/src/services/__tests__/test_llm_plugin_persistence.py +++ b/backend/src/services/__tests__/test_llm_plugin_persistence.py @@ -1,6 +1,7 @@ -# #region test_llm_plugin_persistence [C:3] [TYPE Module] -# @BRIEF Regression test for ValidationRecord persistence fields populated from task context. -# @RELATION VERIFIES -> [DashboardValidationPlugin:Class] +# [DEF:test_llm_plugin_persistence:Module] +# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] +# @COMPLEXITY: 3 +# @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context. import types import pytest @@ -8,10 +9,11 @@ import pytest from src.plugins.llm_analysis import plugin as plugin_module -# #region _DummyLogger [C:1] [TYPE Class] -# @BRIEF Minimal logger shim for TaskContext-like objects used in tests. -# @INVARIANT Logging methods are no-ops and must not mutate test state. -# @RELATION BINDS_TO -> [test_llm_plugin_persistence:Module] +# [DEF:_DummyLogger:Class] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] +# @COMPLEXITY: 1 +# @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests. +# @INVARIANT: Logging methods are no-ops and must not mutate test state. class _DummyLogger: def with_source(self, _source: str): return self @@ -29,13 +31,14 @@ class _DummyLogger: return None -# #endregion _DummyLogger +# [/DEF:_DummyLogger:Class] -# #region _FakeDBSession [C:2] [TYPE Class] -# @BRIEF Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin. -# @INVARIANT add/commit/close provide only persistence signals asserted by this test. -# @RELATION BINDS_TO -> [test_llm_plugin_persistence:Module] +# [DEF:_FakeDBSession:Class] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin. +# @INVARIANT: add/commit/close provide only persistence signals asserted by this test. class _FakeDBSession: def __init__(self): self.added = None @@ -52,14 +55,15 @@ class _FakeDBSession: self.closed = True -# #endregion _FakeDBSession +# [/DEF:_FakeDBSession:Class] -# #region test_dashboard_validation_plugin_persists_task_and_environment_ids [C:2] [TYPE Function] -# @BRIEF Ensure db ValidationRecord includes context.task_id and params.environment_id. -# @INVARIANT Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals. -# @RELATION BINDS_TO -> [test_llm_plugin_persistence:Module] -# @RELATION VERIFIES -> [DashboardValidationPlugin:Class] +# [DEF:test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] +# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module] +# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class] +# @COMPLEXITY: 2 +# @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id. +# @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals. @pytest.mark.asyncio async def test_dashboard_validation_plugin_persists_task_and_environment_ids( tmp_path, monkeypatch @@ -76,10 +80,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( is_active=True, ) - # #region _FakeProviderService [C:1] [TYPE Class] - # @BRIEF LLM provider service stub returning deterministic provider and decrypted API key for plugin tests. - # @INVARIANT Returns same provider and key regardless of provider_id argument; no lookup logic. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeProviderService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests. + # @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic. class _FakeProviderService: def __init__(self, _db): return None @@ -90,12 +95,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( def get_decrypted_api_key(self, _provider_id): return "a" * 32 - # #endregion _FakeProviderService + # [/DEF:_FakeProviderService:Class] - # #region _FakeScreenshotService [C:1] [TYPE Class] - # @BRIEF Screenshot service stub that accepts capture_dashboard calls without side effects. - # @INVARIANT capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeScreenshotService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects. + # @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values. class _FakeScreenshotService: def __init__(self, _env): return None @@ -103,12 +109,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( async def capture_dashboard(self, _dashboard_id, _screenshot_path): return None - # #endregion _FakeScreenshotService + # [/DEF:_FakeScreenshotService:Class] - # #region _FakeLLMClient [C:2] [TYPE Class] - # @BRIEF Deterministic LLM client double returning canonical analysis payload for persistence-path assertions. - # @INVARIANT analyze_dashboard is side-effect free and returns schema-compatible PASS result. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeLLMClient:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions. + # @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result. class _FakeLLMClient: """Fake LLM client for persistence tests. @@ -126,12 +133,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( "issues": [], } - # #endregion _FakeLLMClient + # [/DEF:_FakeLLMClient:Class] - # #region _FakeNotificationService [C:1] [TYPE Class] - # @BRIEF Notification service stub that accepts plugin dispatch_report payload without introducing side effects. - # @INVARIANT dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeNotificationService:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects. + # @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema. class _FakeNotificationService: def __init__(self, *_args, **_kwargs): return None @@ -139,12 +147,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( async def dispatch_report(self, **_kwargs): return None - # #endregion _FakeNotificationService + # [/DEF:_FakeNotificationService:Class] - # #region _FakeConfigManager [C:1] [TYPE Class] - # @BRIEF Config manager stub providing storage root path and minimal settings for plugin execution path. - # @INVARIANT Only storage.root_path and llm fields are safe to access; all other settings fields are absent. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeConfigManager:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path. + # @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent. class _FakeConfigManager: def get_environment(self, _env_id): return env @@ -157,19 +166,20 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( ) ) - # #endregion _FakeConfigManager + # [/DEF:_FakeConfigManager:Class] - # #region _FakeSupersetClient [C:1] [TYPE Class] - # @BRIEF Superset client stub exposing network.request as a lambda that returns empty result list. - # @INVARIANT network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency. - # @RELATION BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # [DEF:_FakeSupersetClient:Class] + # @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list. + # @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency. class _FakeSupersetClient: def __init__(self, _env): self.network = types.SimpleNamespace( request=lambda **_kwargs: {"result": []} ) - # #endregion _FakeSupersetClient + # [/DEF:_FakeSupersetClient:Class] monkeypatch.setattr(plugin_module, "SessionLocal", lambda: fake_db) monkeypatch.setattr(plugin_module, "LLMProviderService", _FakeProviderService) @@ -205,4 +215,7 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids( assert fake_db.added.environment_id == "env-42" -# #endregion test_dashboard_validation_plugin_persists_task_and_environment_ids +# [/DEF:test_dashboard_validation_plugin_persists_task_and_environment_ids:Function] + + +# [/DEF:test_llm_plugin_persistence:Module] diff --git a/backend/src/services/__tests__/test_llm_prompt_templates.py b/backend/src/services/__tests__/test_llm_prompt_templates.py index fc890c52..c90ef225 100644 --- a/backend/src/services/__tests__/test_llm_prompt_templates.py +++ b/backend/src/services/__tests__/test_llm_prompt_templates.py @@ -1,8 +1,10 @@ -# #region test_llm_prompt_templates [C:3] [TYPE Module] [SEMANTICS tests, llm, prompts, templates, settings] -# @BRIEF Validate normalization and rendering behavior for configurable LLM prompt templates. -# @LAYER Domain Tests -# @INVARIANT All required prompt keys remain available after normalization. -# @RELATION DEPENDS_ON -> [llm_prompt_templates] +# [DEF:test_llm_prompt_templates:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, llm, prompts, templates, settings +# @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates. +# @LAYER: Domain Tests +# @RELATION: DEPENDS_ON -> [llm_prompt_templates] +# @INVARIANT: All required prompt keys remain available after normalization. from src.services.llm_prompt_templates import ( DEFAULT_LLM_ASSISTANT_SETTINGS, @@ -15,11 +17,12 @@ from src.services.llm_prompt_templates import ( ) -# #region test_normalize_llm_settings_adds_default_prompts [C:3] [TYPE Function] -# @BRIEF Ensure legacy/partial llm settings are expanded with all prompt defaults. -# @PRE Input llm settings do not contain complete prompts object. -# @POST Returned structure includes required prompt templates with fallback defaults. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_normalize_llm_settings_adds_default_prompts:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 3 +# @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults. +# @PRE: Input llm settings do not contain complete prompts object. +# @POST: Returned structure includes required prompt templates with fallback defaults. def test_normalize_llm_settings_adds_default_prompts(): normalized = normalize_llm_settings({"default_provider": "x"}) @@ -35,14 +38,15 @@ def test_normalize_llm_settings_adds_default_prompts(): assert key in normalized -# #endregion test_normalize_llm_settings_adds_default_prompts +# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function] -# #region test_normalize_llm_settings_keeps_custom_prompt_values [C:3] [TYPE Function] -# @BRIEF Ensure user-customized prompt values are preserved during normalization. -# @PRE Input llm settings contain custom prompt override. -# @POST Custom prompt value remains unchanged in normalized output. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 3 +# @PURPOSE: Ensure user-customized prompt values are preserved during normalization. +# @PRE: Input llm settings contain custom prompt override. +# @POST: Custom prompt value remains unchanged in normalized output. def test_normalize_llm_settings_keeps_custom_prompt_values(): custom = "Doc for {dataset_name} using {columns_json}" normalized = normalize_llm_settings({"prompts": {"documentation_prompt": custom}}) @@ -50,14 +54,15 @@ def test_normalize_llm_settings_keeps_custom_prompt_values(): assert normalized["prompts"]["documentation_prompt"] == custom -# #endregion test_normalize_llm_settings_keeps_custom_prompt_values +# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] -# #region test_render_prompt_replaces_known_placeholders [C:3] [TYPE Function] -# @BRIEF Ensure template placeholders are deterministically replaced. -# @PRE Template contains placeholders matching provided variables. -# @POST Rendered prompt string contains substituted values. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_render_prompt_replaces_known_placeholders:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 3 +# @PURPOSE: Ensure template placeholders are deterministically replaced. +# @PRE: Template contains placeholders matching provided variables. +# @POST: Rendered prompt string contains substituted values. def test_render_prompt_replaces_known_placeholders(): rendered = render_prompt( "Hello {name}, diff={diff}", @@ -67,12 +72,13 @@ def test_render_prompt_replaces_known_placeholders(): assert rendered == "Hello bot, diff=A->B" -# #endregion test_render_prompt_replaces_known_placeholders +# [/DEF:test_render_prompt_replaces_known_placeholders:Function] -# #region test_is_multimodal_model_detects_known_vision_models [C:2] [TYPE Function] -# @BRIEF Ensure multimodal model detection recognizes common vision-capable model names. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_is_multimodal_model_detects_known_vision_models:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 2 +# @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names. def test_is_multimodal_model_detects_known_vision_models(): assert is_multimodal_model("gpt-4o") is True assert is_multimodal_model("claude-3-5-sonnet") is True @@ -80,12 +86,13 @@ def test_is_multimodal_model_detects_known_vision_models(): assert is_multimodal_model("text-only-model") is False -# #endregion test_is_multimodal_model_detects_known_vision_models +# [/DEF:test_is_multimodal_model_detects_known_vision_models:Function] -# #region test_resolve_bound_provider_id_prefers_binding_then_default [C:2] [TYPE Function] -# @BRIEF Verify provider binding resolution priority. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_resolve_bound_provider_id_prefers_binding_then_default:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 2 +# @PURPOSE: Verify provider binding resolution priority. def test_resolve_bound_provider_id_prefers_binding_then_default(): settings = { "default_provider": "default-1", @@ -95,12 +102,13 @@ def test_resolve_bound_provider_id_prefers_binding_then_default(): assert resolve_bound_provider_id(settings, "documentation") == "default-1" -# #endregion test_resolve_bound_provider_id_prefers_binding_then_default +# [/DEF:test_resolve_bound_provider_id_prefers_binding_then_default:Function] -# #region test_normalize_llm_settings_keeps_assistant_planner_settings [C:2] [TYPE Function] -# @BRIEF Ensure assistant planner provider/model fields are preserved and normalized. -# @RELATION BINDS_TO -> [test_llm_prompt_templates] +# [DEF:test_normalize_llm_settings_keeps_assistant_planner_settings:Function] +# @RELATION: BINDS_TO -> test_llm_prompt_templates +# @COMPLEXITY: 2 +# @PURPOSE: Ensure assistant planner provider/model fields are preserved and normalized. def test_normalize_llm_settings_keeps_assistant_planner_settings(): normalized = normalize_llm_settings( { @@ -112,7 +120,7 @@ def test_normalize_llm_settings_keeps_assistant_planner_settings(): assert normalized["assistant_planner_model"] == "gpt-4.1-mini" -# #endregion test_normalize_llm_settings_keeps_assistant_planner_settings +# [/DEF:test_normalize_llm_settings_keeps_assistant_planner_settings:Function] -# #endregion test_llm_prompt_templates +# [/DEF:test_llm_prompt_templates:Module] diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index 0bac34e8..a4240646 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -1,7 +1,9 @@ -# #region test_llm_provider [C:3] [TYPE Module] [SEMANTICS tests, llm-provider, encryption, contract] -# @BRIEF Contract testing for LLMProviderService and EncryptionManager -# @RELATION VERIFIES -> [src.services.llm_provider:Module] -# #endregion test_llm_provider +# [DEF:test_llm_provider:Module] +# @RELATION: VERIFIES -> [src.services.llm_provider:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, llm-provider, encryption, contract +# @PURPOSE: Contract testing for LLMProviderService and EncryptionManager +# [/DEF:test_llm_provider:Module] import pytest import os @@ -12,18 +14,18 @@ from src.services.llm_provider import EncryptionManager, LLMProviderService from src.models.llm import LLMProvider from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType -# #region _test_encryption_key_fixture [TYPE Global] -# @BRIEF Ensure encryption-dependent provider tests run with a valid Fernet key. -# @RELATION DEPENDS_ON -> [pytest:Module] +# [DEF:_test_encryption_key_fixture:Global] +# @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key. +# @RELATION: DEPENDS_ON -> [pytest:Module] os.environ.setdefault("ENCRYPTION_KEY", Fernet.generate_key().decode()) -# #endregion _test_encryption_key_fixture +# [/DEF:_test_encryption_key_fixture:Global] # @TEST_CONTRACT: EncryptionManagerModel -> Invariants # @TEST_INVARIANT: symmetric_encryption -# #region test_encryption_cycle [TYPE Function] -# @BRIEF Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_encryption_cycle:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets. def test_encryption_cycle(): """Verify encrypted data can be decrypted back to original string.""" manager = EncryptionManager() @@ -34,12 +36,12 @@ def test_encryption_cycle(): # @TEST_EDGE: empty_string_encryption -# #endregion test_encryption_cycle +# [/DEF:test_encryption_cycle:Function] -# #region test_empty_string_encryption [TYPE Function] -# @BRIEF Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_empty_string_encryption:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle. def test_empty_string_encryption(): manager = EncryptionManager() original = "" @@ -48,12 +50,12 @@ def test_empty_string_encryption(): # @TEST_EDGE: decrypt_invalid_data -# #endregion test_empty_string_encryption +# [/DEF:test_empty_string_encryption:Function] -# #region test_decrypt_invalid_data [TYPE Function] -# @BRIEF Ensure decrypt rejects invalid ciphertext input by raising an exception. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_decrypt_invalid_data:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure decrypt rejects invalid ciphertext input by raising an exception. def test_decrypt_invalid_data(): manager = EncryptionManager() with pytest.raises(Exception): @@ -61,48 +63,50 @@ def test_decrypt_invalid_data(): # @TEST_FIXTURE: mock_db_session -# #endregion test_decrypt_invalid_data +# [/DEF:test_decrypt_invalid_data:Function] -# #region mock_db [C:1] [TYPE Fixture] -# @BRIEF MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests. -# @INVARIANT Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:mock_db:Fixture] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @COMPLEXITY: 1 +# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests. +# @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced. @pytest.fixture def mock_db(): # @RISK: query() returns unconstrained MagicMock — chain beyond query() has no spec protection. Consider create_autospec(Session) for full chain safety. return MagicMock(spec=Session) -# #endregion mock_db +# [/DEF:mock_db:Fixture] -# #region service [C:1] [TYPE Fixture] -# @BRIEF LLMProviderService fixture wired to mock_db for provider CRUD tests. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:service:Fixture] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @COMPLEXITY: 1 +# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests. @pytest.fixture def service(mock_db): return LLMProviderService(db=mock_db) -# #endregion service +# [/DEF:service:Fixture] -# #region test_get_all_providers [TYPE Function] -# @BRIEF Verify provider list retrieval issues query/all calls on the backing DB session. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_get_all_providers:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify provider list retrieval issues query/all calls on the backing DB session. def test_get_all_providers(service, mock_db): service.get_all_providers() mock_db.query.assert_called() mock_db.query().all.assert_called() -# #endregion test_get_all_providers +# [/DEF:test_get_all_providers:Function] -# #region test_create_provider [TYPE Function] -# @BRIEF Ensure provider creation persists entity and stores API key in encrypted form. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_create_provider:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure provider creation persists entity and stores API key in encrypted form. def test_create_provider(service, mock_db): config = LLMProviderConfig( provider_type=LLMProviderType.OPENAI, @@ -123,12 +127,12 @@ def test_create_provider(service, mock_db): assert EncryptionManager().decrypt(provider.api_key) == "sk-test" -# #endregion test_create_provider +# [/DEF:test_create_provider:Function] -# #region test_get_decrypted_api_key [TYPE Function] -# @BRIEF Verify service decrypts stored provider API key for an existing provider record. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_get_decrypted_api_key:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify service decrypts stored provider API key for an existing provider record. def test_get_decrypted_api_key(service, mock_db): # Setup mock provider encrypted_key = EncryptionManager().encrypt("secret-value") @@ -139,23 +143,23 @@ def test_get_decrypted_api_key(service, mock_db): assert key == "secret-value" -# #endregion test_get_decrypted_api_key +# [/DEF:test_get_decrypted_api_key:Function] -# #region test_get_decrypted_api_key_not_found [TYPE Function] -# @BRIEF Verify missing provider lookup returns None instead of attempting decryption. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_get_decrypted_api_key_not_found:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Verify missing provider lookup returns None instead of attempting decryption. def test_get_decrypted_api_key_not_found(service, mock_db): mock_db.query().filter().first.return_value = None assert service.get_decrypted_api_key("missing") is None -# #endregion test_get_decrypted_api_key_not_found +# [/DEF:test_get_decrypted_api_key_not_found:Function] -# #region test_update_provider_ignores_masked_placeholder_api_key [TYPE Function] -# @BRIEF Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets. -# @RELATION BINDS_TO -> [test_llm_provider:Module] +# [DEF:test_update_provider_ignores_masked_placeholder_api_key:Function] +# @RELATION: BINDS_TO -> [test_llm_provider:Module] +# @PURPOSE: Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets. def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db): existing_encrypted = EncryptionManager().encrypt("secret-value") mock_provider = LLMProvider( @@ -186,4 +190,4 @@ def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db): assert updated.is_active is False -# #endregion test_update_provider_ignores_masked_placeholder_api_key +# [/DEF:test_update_provider_ignores_masked_placeholder_api_key:Function] diff --git a/backend/src/services/__tests__/test_rbac_permission_catalog.py b/backend/src/services/__tests__/test_rbac_permission_catalog.py index a524717f..7f2e2c92 100644 --- a/backend/src/services/__tests__/test_rbac_permission_catalog.py +++ b/backend/src/services/__tests__/test_rbac_permission_catalog.py @@ -1,8 +1,10 @@ -# #region test_rbac_permission_catalog [C:3] [TYPE Module] [SEMANTICS tests, rbac, permissions, catalog, discovery, sync] -# @BRIEF Verifies RBAC permission catalog discovery and idempotent synchronization behavior. -# @LAYER Service Tests -# @INVARIANT Synchronization adds only missing normalized permission pairs. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:test_rbac_permission_catalog:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 3 +# @SEMANTICS: tests, rbac, permissions, catalog, discovery, sync +# @PURPOSE: Verifies RBAC permission catalog discovery and idempotent synchronization behavior. +# @LAYER: Service Tests +# @INVARIANT: Synchronization adds only missing normalized permission pairs. # [SECTION: IMPORTS] from types import SimpleNamespace @@ -12,11 +14,11 @@ import src.services.rbac_permission_catalog as catalog # [/SECTION: IMPORTS] -# #region test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests [TYPE Function] -# @BRIEF Ensures route-scanner extracts has_permission pairs from route files and skips __tests__. -# @PRE Temporary route directory contains route and test files. -# @POST Returned set includes production route permissions and excludes test-only declarations. -# @RELATION BINDS_TO -> [test_rbac_permission_catalog] +# [DEF:test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests:Function] +# @RELATION: BINDS_TO -> test_rbac_permission_catalog +# @PURPOSE: Ensures route-scanner extracts has_permission pairs from route files and skips __tests__. +# @PRE: Temporary route directory contains route and test files. +# @POST: Returned set includes production route permissions and excludes test-only declarations. def test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests(tmp_path, monkeypatch): routes_dir = tmp_path / "routes" routes_dir.mkdir(parents=True, exist_ok=True) @@ -47,14 +49,14 @@ def test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests(tm assert ("plugin:migration", "EXECUTE") in discovered assert ("tasks", "WRITE") in discovered assert ("plugin:ignored", "READ") not in discovered -# #endregion test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests +# [/DEF:test_discover_route_permissions_extracts_declared_pairs_and_ignores_tests:Function] -# #region test_discover_declared_permissions_unions_route_and_plugin_permissions [TYPE Function] -# @BRIEF Ensures full catalog includes route-level permissions plus dynamic plugin EXECUTE rights. -# @PRE Route discovery and plugin loader both return permission sources. -# @POST Result set contains union of both sources. -# @RELATION BINDS_TO -> [test_rbac_permission_catalog] +# [DEF:test_discover_declared_permissions_unions_route_and_plugin_permissions:Function] +# @RELATION: BINDS_TO -> test_rbac_permission_catalog +# @PURPOSE: Ensures full catalog includes route-level permissions plus dynamic plugin EXECUTE rights. +# @PRE: Route discovery and plugin loader both return permission sources. +# @POST: Result set contains union of both sources. def test_discover_declared_permissions_unions_route_and_plugin_permissions(monkeypatch): monkeypatch.setattr( catalog, @@ -74,14 +76,14 @@ def test_discover_declared_permissions_unions_route_and_plugin_permissions(monke assert ("plugin:migration", "READ") in discovered assert ("plugin:superset-backup", "EXECUTE") in discovered assert ("plugin:llm_dashboard_validation", "EXECUTE") in discovered -# #endregion test_discover_declared_permissions_unions_route_and_plugin_permissions +# [/DEF:test_discover_declared_permissions_unions_route_and_plugin_permissions:Function] -# #region test_sync_permission_catalog_inserts_only_missing_normalized_pairs [TYPE Function] -# @BRIEF Ensures synchronization inserts only missing pairs and normalizes action/resource tokens. -# @PRE DB already contains subset of permissions. -# @POST Only missing normalized pairs are inserted and commit is executed once. -# @RELATION BINDS_TO -> [test_rbac_permission_catalog] +# [DEF:test_sync_permission_catalog_inserts_only_missing_normalized_pairs:Function] +# @RELATION: BINDS_TO -> test_rbac_permission_catalog +# @PURPOSE: Ensures synchronization inserts only missing pairs and normalizes action/resource tokens. +# @PRE: DB already contains subset of permissions. +# @POST: Only missing normalized pairs are inserted and commit is executed once. def test_sync_permission_catalog_inserts_only_missing_normalized_pairs(): db = MagicMock() db.query.return_value.all.return_value = [ @@ -108,14 +110,14 @@ def test_sync_permission_catalog_inserts_only_missing_normalized_pairs(): assert inserted_permission.resource == "plugin:migration" assert inserted_permission.action == "READ" db.commit.assert_called_once() -# #endregion test_sync_permission_catalog_inserts_only_missing_normalized_pairs +# [/DEF:test_sync_permission_catalog_inserts_only_missing_normalized_pairs:Function] -# #region test_sync_permission_catalog_is_noop_when_all_permissions_exist [TYPE Function] -# @BRIEF Ensures synchronization is idempotent when all declared pairs already exist. -# @PRE DB contains full declared permission set. -# @POST No inserts are added and commit is not called. -# @RELATION BINDS_TO -> [test_rbac_permission_catalog] +# [DEF:test_sync_permission_catalog_is_noop_when_all_permissions_exist:Function] +# @RELATION: BINDS_TO -> test_rbac_permission_catalog +# @PURPOSE: Ensures synchronization is idempotent when all declared pairs already exist. +# @PRE: DB contains full declared permission set. +# @POST: No inserts are added and commit is not called. def test_sync_permission_catalog_is_noop_when_all_permissions_exist(): db = MagicMock() db.query.return_value.all.return_value = [ @@ -136,7 +138,7 @@ def test_sync_permission_catalog_is_noop_when_all_permissions_exist(): assert inserted_count == 0 db.add.assert_not_called() db.commit.assert_not_called() -# #endregion test_sync_permission_catalog_is_noop_when_all_permissions_exist +# [/DEF:test_sync_permission_catalog_is_noop_when_all_permissions_exist:Function] -# #endregion test_rbac_permission_catalog +# [/DEF:test_rbac_permission_catalog:Module] \ No newline at end of file diff --git a/backend/src/services/__tests__/test_resource_service.py b/backend/src/services/__tests__/test_resource_service.py index bb4dc106..da65e9a5 100644 --- a/backend/src/services/__tests__/test_resource_service.py +++ b/backend/src/services/__tests__/test_resource_service.py @@ -1,17 +1,19 @@ -# #region TestResourceService [C:3] [TYPE Module] [SEMANTICS resource-service, tests, dashboards, datasets, activity] -# @BRIEF Unit tests for ResourceService -# @LAYER Service -# @INVARIANT Resource summaries preserve task linkage and status projection behavior. -# @RELATION VERIFIES -> [src.services.resource_service.ResourceService] +# [DEF:TestResourceService:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: resource-service, tests, dashboards, datasets, activity +# @PURPOSE: Unit tests for ResourceService +# @LAYER: Service +# @RELATION: VERIFIES ->[src.services.resource_service.ResourceService] +# @INVARIANT: Resource summaries preserve task linkage and status projection behavior. import pytest from unittest.mock import MagicMock, patch, AsyncMock from datetime import datetime, timezone -# #region test_get_dashboards_with_status [TYPE Function] -# @BRIEF Validate dashboard enrichment includes git/task status projections. -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_dashboards_with_status:Function] +# @RELATION: BINDS_TO ->[TestResourceService] +# @PURPOSE: Validate dashboard enrichment includes git/task status projections. # @TEST: get_dashboards_with_status returns dashboards with git and task status # @PRE: SupersetClient returns dashboard list # @POST: Each dashboard has git_status and last_task fields @@ -71,11 +73,11 @@ async def test_get_dashboards_with_status(): assert result[0]["last_task"]["validation_status"] == "FAIL" -# #endregion test_get_dashboards_with_status +# [/DEF:test_get_dashboards_with_status:Function] -# #region test_get_datasets_with_status [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_datasets_with_status:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: get_datasets_with_status returns datasets with task status # @PRE: SupersetClient returns dataset list # @POST: Each dataset has last_task field @@ -112,11 +114,11 @@ async def test_get_datasets_with_status(): assert result[0]["last_task"]["status"] == "RUNNING" -# #endregion test_get_datasets_with_status +# [/DEF:test_get_datasets_with_status:Function] -# #region test_get_activity_summary [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_activity_summary:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: get_activity_summary returns active count and recent tasks # @PRE: tasks list provided # @POST: Returns dict with active_count and recent_tasks @@ -151,11 +153,11 @@ def test_get_activity_summary(): assert len(result["recent_tasks"]) == 3 -# #endregion test_get_activity_summary +# [/DEF:test_get_activity_summary:Function] -# #region test_get_git_status_for_dashboard_no_repo [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_git_status_for_dashboard_no_repo:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _get_git_status_for_dashboard returns None when no repo exists # @PRE: GitService returns None for repo # @POST: Returns None @@ -174,11 +176,11 @@ def test_get_git_status_for_dashboard_no_repo(): assert result["has_repo"] is False -# #endregion test_get_git_status_for_dashboard_no_repo +# [/DEF:test_get_git_status_for_dashboard_no_repo:Function] -# #region test_get_last_task_for_resource [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_last_task_for_resource:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _get_last_task_for_resource returns most recent task for resource # @PRE: tasks list with matching resource_id # @POST: Returns task summary with task_id and status @@ -208,11 +210,11 @@ def test_get_last_task_for_resource(): assert result["status"] == "RUNNING" -# #endregion test_get_last_task_for_resource +# [/DEF:test_get_last_task_for_resource:Function] -# #region test_extract_resource_name_from_task [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_extract_resource_name_from_task:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _extract_resource_name_from_task extracts name from params # @PRE: task has resource_name in params # @POST: Returns resource name or fallback @@ -239,11 +241,11 @@ def test_extract_resource_name_from_task(): assert "task-456" in result2 -# #endregion test_extract_resource_name_from_task +# [/DEF:test_extract_resource_name_from_task:Function] -# #region test_get_last_task_for_resource_empty_tasks [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_last_task_for_resource_empty_tasks:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _get_last_task_for_resource returns None for empty tasks list # @PRE: tasks is empty list # @POST: Returns None @@ -257,11 +259,11 @@ def test_get_last_task_for_resource_empty_tasks(): assert result is None -# #endregion test_get_last_task_for_resource_empty_tasks +# [/DEF:test_get_last_task_for_resource_empty_tasks:Function] -# #region test_get_last_task_for_resource_no_match [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_last_task_for_resource_no_match:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _get_last_task_for_resource returns None when no tasks match resource_id # @PRE: tasks list has no matching resource_id # @POST: Returns None @@ -281,11 +283,11 @@ def test_get_last_task_for_resource_no_match(): assert result is None -# #endregion test_get_last_task_for_resource_no_match +# [/DEF:test_get_last_task_for_resource_no_match:Function] -# #region test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_datetimes [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_datetimes:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: get_dashboards_with_status handles mixed naive/aware datetimes without comparison errors. # @PRE: Task list includes both timezone-aware and timezone-naive timestamps. # @POST: Latest task is selected deterministically and no exception is raised. @@ -325,11 +327,11 @@ async def test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_dat assert result[0]["last_task"]["task_id"] == "task-aware" -# #endregion test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_datetimes +# [/DEF:test_get_dashboards_with_status_handles_mixed_naive_and_aware_task_datetimes:Function] -# #region test_get_dashboards_with_status_prefers_latest_decisive_validation_status_over_newer_unknown [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_dashboards_with_status_prefers_latest_decisive_validation_status_over_newer_unknown:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: get_dashboards_with_status keeps latest task identity while falling back to older decisive validation status. # @PRE: Same dashboard has older WARN and newer UNKNOWN validation tasks. # @POST: Returned last_task points to newest task but preserves WARN as last meaningful validation state. @@ -375,11 +377,11 @@ async def test_get_dashboards_with_status_prefers_latest_decisive_validation_sta assert result[0]["last_task"]["validation_status"] == "WARN" -# #endregion test_get_dashboards_with_status_prefers_latest_decisive_validation_status_over_newer_unknown +# [/DEF:test_get_dashboards_with_status_prefers_latest_decisive_validation_status_over_newer_unknown:Function] -# #region test_get_dashboards_with_status_falls_back_to_latest_unknown_without_decisive_history [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_dashboards_with_status_falls_back_to_latest_unknown_without_decisive_history:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: get_dashboards_with_status still returns newest UNKNOWN when no decisive validation exists. # @PRE: Same dashboard has only UNKNOWN validation tasks. # @POST: Returned last_task keeps newest UNKNOWN task. @@ -424,11 +426,11 @@ async def test_get_dashboards_with_status_falls_back_to_latest_unknown_without_d assert result[0]["last_task"]["validation_status"] == "UNKNOWN" -# #endregion test_get_dashboards_with_status_falls_back_to_latest_unknown_without_decisive_history +# [/DEF:test_get_dashboards_with_status_falls_back_to_latest_unknown_without_decisive_history:Function] -# #region test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at [TYPE Function] -# @RELATION BINDS_TO -> [TestResourceService] +# [DEF:test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at:Function] +# @RELATION: BINDS_TO ->[TestResourceService] # @TEST: _get_last_task_for_resource handles mixed naive/aware created_at values. # @PRE: Matching tasks include naive and aware created_at timestamps. # @POST: Latest task is returned without raising datetime comparison errors. @@ -458,7 +460,7 @@ def test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at(): assert result["task_id"] == "task-new" -# #endregion test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at +# [/DEF:test_get_last_task_for_resource_handles_mixed_naive_and_aware_created_at:Function] -# #endregion TestResourceService +# [/DEF:TestResourceService:Module] diff --git a/backend/src/services/auth_service.py b/backend/src/services/auth_service.py index 505c74d1..f899bd22 100644 --- a/backend/src/services/auth_service.py +++ b/backend/src/services/auth_service.py @@ -1,16 +1,16 @@ # #region auth_service [C:5] [TYPE Module] [SEMANTICS auth, service, business-logic, login, jwt, adfs, jit-provisioning] # @BRIEF Orchestrates credential authentication and ADFS JIT user provisioning. -# @LAYER Domain -# @PRE Core auth models and security utilities available. -# @POST User identity verified and session tokens issued according to role scopes. -# @SIDE_EFFECT Writes last login timestamps and JIT-provisions external users. -# @DATA_CONTRACT [Credentials | ADFSClaims] -> [UserEntity | SessionToken] -# @INVARIANT Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [AuthRepository] # @RELATION DEPENDS_ON -> [verify_password] # @RELATION DEPENDS_ON -> [create_access_token] # @RELATION DEPENDS_ON -> [User] # @RELATION DEPENDS_ON -> [Role] +# @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles. +# @PRE: Core auth models and security utilities available. +# @POST: User identity verified and session tokens issued according to role scopes. +# @SIDE_EFFECT: Writes last login timestamps and JIT-provisions external users. +# @DATA_CONTRACT: [Credentials | ADFSClaims] -> [UserEntity | SessionToken] from typing import Dict, Any, Optional, List from datetime import datetime @@ -30,28 +30,30 @@ from ..core.logger import belief_scope # @RELATION DEPENDS_ON -> [User] # @RELATION DEPENDS_ON -> [Role] class AuthService: - # #region AuthService_init [C:1] [TYPE Function] - # @BRIEF Initializes the authentication service with repository access over an active DB session. - # @PRE db is a valid SQLAlchemy Session instance bound to the auth persistence context. - # @POST self.repo is initialized and ready for auth user/role CRUD operations. - # @SIDE_EFFECT Allocates AuthRepository and binds it to the provided Session. - # @DATA_CONTRACT Input(Session) -> Model(AuthRepository) + # [DEF:AuthService_init:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Initializes the authentication service with repository access over an active DB session. + # @PRE: db is a valid SQLAlchemy Session instance bound to the auth persistence context. + # @POST: self.repo is initialized and ready for auth user/role CRUD operations. + # @SIDE_EFFECT: Allocates AuthRepository and binds it to the provided Session. + # @DATA_CONTRACT: Input(Session) -> Model(AuthRepository) # @PARAM: db (Session) - SQLAlchemy session. def __init__(self, db: Session): self.db = db self.repo = AuthRepository(db) - # #endregion AuthService_init + # [/DEF:AuthService_init:Function] - # #region AuthService.authenticate_user [C:3] [TYPE Function] - # @BRIEF Validates credentials and account state for local username/password authentication. - # @PRE username and password are non-empty credential inputs. - # @POST Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None. - # @SIDE_EFFECT Persists last_login update for successful authentications via repository. - # @DATA_CONTRACT Input(str username, str password) -> Output(User | None) - # @RELATION DEPENDS_ON -> [AuthRepository] - # @RELATION CALLS -> [verify_password] - # @RELATION DEPENDS_ON -> [User] + # [DEF:AuthService.authenticate_user:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Validates credentials and account state for local username/password authentication. + # @PRE: username and password are non-empty credential inputs. + # @POST: Returns User only when user exists, is active, and password hash verification succeeds; otherwise returns None. + # @SIDE_EFFECT: Persists last_login update for successful authentications via repository. + # @DATA_CONTRACT: Input(str username, str password) -> Output(User | None) + # @RELATION: [DEPENDS_ON] ->[AuthRepository] + # @RELATION: [CALLS] ->[verify_password] + # @RELATION: [DEPENDS_ON] ->[User] # @PARAM: username (str) - The username. # @PARAM: password (str) - The plain password. # @RETURN: Optional[User] - The authenticated user or None. @@ -71,17 +73,18 @@ class AuthService: return user - # #endregion AuthService.authenticate_user + # [/DEF:AuthService.authenticate_user:Function] - # #region AuthService.create_session [C:3] [TYPE Function] - # @BRIEF Issues an access token payload for an already authenticated user. - # @PRE user is a valid User entity containing username and iterable roles with role.name values. - # @POST Returns session dict with non-empty access_token and token_type='bearer'. - # @SIDE_EFFECT Generates signed JWT via auth JWT provider. - # @DATA_CONTRACT Input(User) -> Output(Dict[str, str]{access_token, token_type}) - # @RELATION CALLS -> [create_access_token] - # @RELATION DEPENDS_ON -> [User] - # @RELATION DEPENDS_ON -> [Role] + # [DEF:AuthService.create_session:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Issues an access token payload for an already authenticated user. + # @PRE: user is a valid User entity containing username and iterable roles with role.name values. + # @POST: Returns session dict with non-empty access_token and token_type='bearer'. + # @SIDE_EFFECT: Generates signed JWT via auth JWT provider. + # @DATA_CONTRACT: Input(User) -> Output(Dict[str, str]{access_token, token_type}) + # @RELATION: [CALLS] ->[create_access_token] + # @RELATION: [DEPENDS_ON] ->[User] + # @RELATION: [DEPENDS_ON] ->[Role] # @PARAM: user (User) - The authenticated user. # @RETURN: Dict[str, str] - Session data. def create_session(self, user: User) -> Dict[str, str]: @@ -92,17 +95,18 @@ class AuthService: ) return {"access_token": access_token, "token_type": "bearer"} - # #endregion AuthService.create_session + # [/DEF:AuthService.create_session:Function] - # #region AuthService.provision_adfs_user [C:3] [TYPE Function] - # @BRIEF Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings. - # @PRE user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent. - # @POST Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state. - # @SIDE_EFFECT May insert new User, mutate user.roles, commit transaction, and refresh ORM state. - # @DATA_CONTRACT Input(Dict[str, Any]{upn|email, email, groups[]}) -> Output(User persisted) - # @RELATION DEPENDS_ON -> [AuthRepository] - # @RELATION DEPENDS_ON -> [User] - # @RELATION DEPENDS_ON -> [Role] + # [DEF:AuthService.provision_adfs_user:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings. + # @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent. + # @POST: Returns persisted user entity with roles synchronized to mapped AD groups and refreshed state. + # @SIDE_EFFECT: May insert new User, mutate user.roles, commit transaction, and refresh ORM state. + # @DATA_CONTRACT: Input(Dict[str, Any]{upn|email, email, groups[]}) -> Output(User persisted) + # @RELATION: [DEPENDS_ON] ->[AuthRepository] + # @RELATION: [DEPENDS_ON] ->[User] + # @RELATION: [DEPENDS_ON] ->[Role] # @PARAM: user_info (Dict[str, Any]) - Claims from ADFS token. # @RETURN: User - The provisioned user. def provision_adfs_user(self, user_info: Dict[str, Any]) -> User: @@ -134,7 +138,9 @@ class AuthService: return user - # #endregion AuthService.provision_adfs_user + # [/DEF:AuthService.provision_adfs_user:Function] # #endregion AuthService + +# #endregion auth_service diff --git a/backend/src/services/clean_release/__init__.py b/backend/src/services/clean_release/__init__.py index b16639cd..e435ba2a 100644 --- a/backend/src/services/clean_release/__init__.py +++ b/backend/src/services/clean_release/__init__.py @@ -1,6 +1,6 @@ # #region CleanReleaseContracts [C:3] [TYPE Module] # @BRIEF Publish the canonical semantic root for the clean-release backend service cluster. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [ComplianceOrchestrator] # @RELATION DEPENDS_ON -> [ManifestBuilder] # @RELATION DEPENDS_ON -> [PolicyEngine] diff --git a/backend/src/services/clean_release/__tests__/test_audit_service.py b/backend/src/services/clean_release/__tests__/test_audit_service.py index 91b6545c..55b38df4 100644 --- a/backend/src/services/clean_release/__tests__/test_audit_service.py +++ b/backend/src/services/clean_release/__tests__/test_audit_service.py @@ -1,7 +1,9 @@ -# #region TestAuditService [C:3] [TYPE Module] [SEMANTICS tests, clean-release, audit, logging] -# @BRIEF Validate audit hooks emit expected log patterns for clean release lifecycle. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [AuditService] +# [DEF:TestAuditService:Module] +# @RELATION: [DEPENDS_ON] ->[AuditService] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, audit, logging +# @PURPOSE: Validate audit hooks emit expected log patterns for clean release lifecycle. +# @LAYER: Infra from unittest.mock import patch from src.services.clean_release.audit_service import ( @@ -12,9 +14,9 @@ from src.services.clean_release.audit_service import ( @patch("src.services.clean_release.audit_service.logger") -# #region test_audit_preparation [TYPE Function] -# @BRIEF Verify audit preparation stage correctly initializes and validates candidate state. -# @RELATION BINDS_TO -> [TestAuditService] +# [DEF:test_audit_preparation:Function] +# @RELATION: BINDS_TO -> TestAuditService +# @PURPOSE: Verify audit preparation stage correctly initializes and validates candidate state. def test_audit_preparation(mock_logger): audit_preparation("cand-1", "PREPARED") mock_logger.info.assert_called_with( @@ -22,13 +24,13 @@ def test_audit_preparation(mock_logger): ) -# #endregion test_audit_preparation +# [/DEF:test_audit_preparation:Function] @patch("src.services.clean_release.audit_service.logger") -# #region test_audit_check_run [TYPE Function] -# @BRIEF Verify audit check run executes all checks and collects results. -# @RELATION BINDS_TO -> [TestAuditService] +# [DEF:test_audit_check_run:Function] +# @RELATION: BINDS_TO -> TestAuditService +# @PURPOSE: Verify audit check run executes all checks and collects results. def test_audit_check_run(mock_logger): audit_check_run("check-1", "COMPLIANT") mock_logger.info.assert_called_with( @@ -36,13 +38,13 @@ def test_audit_check_run(mock_logger): ) -# #endregion test_audit_check_run +# [/DEF:test_audit_check_run:Function] @patch("src.services.clean_release.audit_service.logger") -# #region test_audit_report [TYPE Function] -# @BRIEF Verify audit report generation aggregates check results into a structured report. -# @RELATION BINDS_TO -> [TestAuditService] +# [DEF:test_audit_report:Function] +# @RELATION: BINDS_TO -> TestAuditService +# @PURPOSE: Verify audit report generation aggregates check results into a structured report. def test_audit_report(mock_logger): audit_report("rep-1", "cand-1") mock_logger.info.assert_called_with( @@ -50,5 +52,5 @@ def test_audit_report(mock_logger): ) -# #endregion test_audit_report -# #endregion TestAuditService +# [/DEF:test_audit_report:Function] +# [/DEF:TestAuditService:Module] diff --git a/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py b/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py index 1e3a32f3..180d369b 100644 --- a/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py +++ b/backend/src/services/clean_release/__tests__/test_compliance_orchestrator.py @@ -1,8 +1,10 @@ -# #region TestComplianceOrchestrator [C:3] [TYPE Module] [SEMANTICS tests, clean-release, orchestrator, stage-state-machine] -# @BRIEF Validate compliance orchestrator stage transitions and final status derivation. -# @LAYER Domain -# @INVARIANT Failed mandatory stage forces BLOCKED terminal status. -# @RELATION DEPENDS_ON -> [ComplianceOrchestrator] +# [DEF:TestComplianceOrchestrator:Module] +# @RELATION: [DEPENDS_ON] ->[ComplianceOrchestrator] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, orchestrator, stage-state-machine +# @PURPOSE: Validate compliance orchestrator stage transitions and final status derivation. +# @LAYER: Domain +# @INVARIANT: Failed mandatory stage forces BLOCKED terminal status. from unittest.mock import patch @@ -21,9 +23,9 @@ from src.services.clean_release.report_builder import ComplianceReportBuilder from src.services.clean_release.repository import CleanReleaseRepository -# #region test_orchestrator_stage_failure_blocks_release [TYPE Function] -# @BRIEF Verify mandatory stage failure forces BLOCKED final status. -# @RELATION BINDS_TO -> [TestComplianceOrchestrator] +# [DEF:test_orchestrator_stage_failure_blocks_release:Function] +# @RELATION: BINDS_TO -> TestComplianceOrchestrator +# @PURPOSE: Verify mandatory stage failure forces BLOCKED final status. def test_orchestrator_stage_failure_blocks_release(): repository = CleanReleaseRepository() orchestrator = CleanComplianceOrchestrator(repository) @@ -64,12 +66,12 @@ def test_orchestrator_stage_failure_blocks_release(): assert run.final_status == CheckFinalStatus.BLOCKED -# #endregion test_orchestrator_stage_failure_blocks_release +# [/DEF:test_orchestrator_stage_failure_blocks_release:Function] -# #region test_orchestrator_compliant_candidate [TYPE Function] -# @BRIEF Verify happy path where all mandatory stages pass yields COMPLIANT. -# @RELATION BINDS_TO -> [TestComplianceOrchestrator] +# [DEF:test_orchestrator_compliant_candidate:Function] +# @RELATION: BINDS_TO -> TestComplianceOrchestrator +# @PURPOSE: Verify happy path where all mandatory stages pass yields COMPLIANT. def test_orchestrator_compliant_candidate(): repository = CleanReleaseRepository() orchestrator = CleanComplianceOrchestrator(repository) @@ -110,12 +112,12 @@ def test_orchestrator_compliant_candidate(): assert run.final_status == CheckFinalStatus.COMPLIANT -# #endregion test_orchestrator_compliant_candidate +# [/DEF:test_orchestrator_compliant_candidate:Function] -# #region test_orchestrator_missing_stage_result [TYPE Function] -# @BRIEF Verify incomplete mandatory stage set cannot end as COMPLIANT and results in FAILED. -# @RELATION BINDS_TO -> [TestComplianceOrchestrator] +# [DEF:test_orchestrator_missing_stage_result:Function] +# @RELATION: BINDS_TO -> TestComplianceOrchestrator +# @PURPOSE: Verify incomplete mandatory stage set cannot end as COMPLIANT and results in FAILED. def test_orchestrator_missing_stage_result(): repository = CleanReleaseRepository() orchestrator = CleanComplianceOrchestrator(repository) @@ -136,12 +138,12 @@ def test_orchestrator_missing_stage_result(): assert run.final_status == CheckFinalStatus.FAILED -# #endregion test_orchestrator_missing_stage_result +# [/DEF:test_orchestrator_missing_stage_result:Function] -# #region test_orchestrator_report_generation_error [TYPE Function] -# @BRIEF Verify downstream report errors do not mutate orchestrator final status. -# @RELATION BINDS_TO -> [TestComplianceOrchestrator] +# [DEF:test_orchestrator_report_generation_error:Function] +# @RELATION: BINDS_TO -> TestComplianceOrchestrator +# @PURPOSE: Verify downstream report errors do not mutate orchestrator final status. def test_orchestrator_report_generation_error(): repository = CleanReleaseRepository() orchestrator = CleanComplianceOrchestrator(repository) @@ -162,5 +164,5 @@ def test_orchestrator_report_generation_error(): assert run.final_status == CheckFinalStatus.FAILED -# #endregion test_orchestrator_report_generation_error -# #endregion TestComplianceOrchestrator +# [/DEF:test_orchestrator_report_generation_error:Function] +# [/DEF:TestComplianceOrchestrator:Module] diff --git a/backend/src/services/clean_release/__tests__/test_manifest_builder.py b/backend/src/services/clean_release/__tests__/test_manifest_builder.py index 2c559b68..46bb2c68 100644 --- a/backend/src/services/clean_release/__tests__/test_manifest_builder.py +++ b/backend/src/services/clean_release/__tests__/test_manifest_builder.py @@ -1,21 +1,23 @@ -# #region TestManifestBuilder [C:5] [TYPE Module] [SEMANTICS tests, clean-release, manifest, deterministic] -# @BRIEF Validate deterministic manifest generation behavior for US1. -# @LAYER Domain -# @PRE Test fixtures are properly initialized -# @POST All test assertions pass -# @SIDE_EFFECT None - test isolation -# @DATA_CONTRACT TestInput -> TestOutput -# @INVARIANT Same input artifacts produce identical deterministic hash. -# @RELATION DEPENDS_ON -> [ManifestBuilder] +# [DEF:TestManifestBuilder:Module] +# @COMPLEXITY: 5 +# @SEMANTICS: tests, clean-release, manifest, deterministic +# @PURPOSE: Validate deterministic manifest generation behavior for US1. +# @LAYER: Domain +# @RELATION: [DEPENDS_ON] ->[ManifestBuilder] +# @INVARIANT: Same input artifacts produce identical deterministic hash. +# @PRE: Test fixtures are properly initialized +# @POST: All test assertions pass +# @SIDE_EFFECT: None - test isolation +# @DATA_CONTRACT: TestInput -> TestOutput from src.services.clean_release.manifest_builder import build_distribution_manifest -# #region test_manifest_deterministic_hash_for_same_input [TYPE Function] -# @BRIEF Ensure hash is stable for same candidate/policy/artifact input. -# @PRE Same input lists are passed twice. -# @POST Hash and summary remain identical. -# @RELATION BINDS_TO -> [TestManifestBuilder] +# [DEF:test_manifest_deterministic_hash_for_same_input:Function] +# @RELATION: BINDS_TO -> TestManifestBuilder +# @PURPOSE: Ensure hash is stable for same candidate/policy/artifact input. +# @PRE: Same input lists are passed twice. +# @POST: Hash and summary remain identical. def test_manifest_deterministic_hash_for_same_input(): artifacts = [ { @@ -52,5 +54,5 @@ def test_manifest_deterministic_hash_for_same_input(): assert manifest1.summary.excluded_count == manifest2.summary.excluded_count -# #endregion test_manifest_deterministic_hash_for_same_input -# #endregion TestManifestBuilder +# [/DEF:test_manifest_deterministic_hash_for_same_input:Function] +# [/DEF:TestManifestBuilder:Module] diff --git a/backend/src/services/clean_release/__tests__/test_policy_engine.py b/backend/src/services/clean_release/__tests__/test_policy_engine.py index 11777e37..b53fff43 100644 --- a/backend/src/services/clean_release/__tests__/test_policy_engine.py +++ b/backend/src/services/clean_release/__tests__/test_policy_engine.py @@ -1,7 +1,7 @@ -# #region TestPolicyEngine [TYPE Module] -# @BRIEF Contract testing for CleanPolicyEngine -# @RELATION DEPENDS_ON -> [PolicyEngine] -# #endregion TestPolicyEngine +# [DEF:TestPolicyEngine:Module] +# @RELATION: [DEPENDS_ON] ->[PolicyEngine] +# @PURPOSE: Contract testing for CleanPolicyEngine +# [/DEF:TestPolicyEngine:Module] import pytest from datetime import datetime @@ -48,9 +48,9 @@ def enterprise_clean_setup(): # @TEST_SCENARIO: policy_valid -# #region test_policy_valid [TYPE Function] -# @BRIEF Verify policy validation passes when all required fields are present and valid. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_policy_valid:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify policy validation passes when all required fields are present and valid. def test_policy_valid(enterprise_clean_setup): policy, registry = enterprise_clean_setup engine = CleanPolicyEngine(policy, registry) @@ -60,12 +60,12 @@ def test_policy_valid(enterprise_clean_setup): # @TEST_EDGE: missing_registry_ref -# #endregion test_policy_valid +# [/DEF:test_policy_valid:Function] -# #region test_missing_registry_ref [TYPE Function] -# @BRIEF Verify policy validation fails when registry_ref is missing. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_missing_registry_ref:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify policy validation fails when registry_ref is missing. def test_missing_registry_ref(enterprise_clean_setup): policy, registry = enterprise_clean_setup policy.internal_source_registry_ref = " " @@ -76,12 +76,12 @@ def test_missing_registry_ref(enterprise_clean_setup): # @TEST_EDGE: conflicting_registry -# #endregion test_missing_registry_ref +# [/DEF:test_missing_registry_ref:Function] -# #region test_conflicting_registry [TYPE Function] -# @BRIEF Verify policy engine rejects conflicting registry references. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_conflicting_registry:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify policy engine rejects conflicting registry references. def test_conflicting_registry(enterprise_clean_setup): policy, registry = enterprise_clean_setup registry.registry_id = "WRONG-REG" @@ -95,12 +95,12 @@ def test_conflicting_registry(enterprise_clean_setup): # @TEST_INVARIANT: deterministic_classification -# #endregion test_conflicting_registry +# [/DEF:test_conflicting_registry:Function] -# #region test_classify_artifact [TYPE Function] -# @BRIEF Verify policy engine correctly classifies artifacts based on source and type. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_classify_artifact:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify policy engine correctly classifies artifacts based on source and type. def test_classify_artifact(enterprise_clean_setup): policy, registry = enterprise_clean_setup engine = CleanPolicyEngine(policy, registry) @@ -120,12 +120,12 @@ def test_classify_artifact(enterprise_clean_setup): # @TEST_EDGE: external_endpoint -# #endregion test_classify_artifact +# [/DEF:test_classify_artifact:Function] -# #region test_validate_resource_source [TYPE Function] -# @BRIEF Verify validate_resource_source correctly validates or rejects resource source identifiers. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_validate_resource_source:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify validate_resource_source correctly validates or rejects resource source identifiers. def test_validate_resource_source(enterprise_clean_setup): policy, registry = enterprise_clean_setup engine = CleanPolicyEngine(policy, registry) @@ -141,12 +141,12 @@ def test_validate_resource_source(enterprise_clean_setup): assert res_fail.violation["blocked_release"] is True -# #endregion test_validate_resource_source +# [/DEF:test_validate_resource_source:Function] -# #region test_evaluate_candidate [TYPE Function] -# @BRIEF Verify policy engine evaluates release candidates against configured policies. -# @RELATION BINDS_TO -> [TestPolicyEngine] +# [DEF:test_evaluate_candidate:Function] +# @RELATION: BINDS_TO -> TestPolicyEngine +# @PURPOSE: Verify policy engine evaluates release candidates against configured policies. def test_evaluate_candidate(enterprise_clean_setup): policy, registry = enterprise_clean_setup engine = CleanPolicyEngine(policy, registry) @@ -169,4 +169,4 @@ def test_evaluate_candidate(enterprise_clean_setup): assert violations[1]["category"] == "external-source" -# #endregion test_evaluate_candidate +# [/DEF:test_evaluate_candidate:Function] diff --git a/backend/src/services/clean_release/__tests__/test_preparation_service.py b/backend/src/services/clean_release/__tests__/test_preparation_service.py index e58c3362..8b7423ac 100644 --- a/backend/src/services/clean_release/__tests__/test_preparation_service.py +++ b/backend/src/services/clean_release/__tests__/test_preparation_service.py @@ -1,8 +1,10 @@ -# #region TestPreparationService [C:3] [TYPE Module] [SEMANTICS tests, clean-release, preparation, flow] -# @BRIEF Validate release candidate preparation flow, including policy evaluation and manifest persisting. -# @LAYER Domain -# @INVARIANT Candidate preparation always persists manifest and candidate status deterministically. -# @RELATION DEPENDS_ON -> [PreparationService] +# [DEF:TestPreparationService:Module] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, preparation, flow +# @PURPOSE: Validate release candidate preparation flow, including policy evaluation and manifest persisting. +# @LAYER: Domain +# @RELATION: [DEPENDS_ON] ->[PreparationService] +# @INVARIANT: Candidate preparation always persists manifest and candidate status deterministically. import pytest from unittest.mock import MagicMock, patch @@ -20,9 +22,9 @@ from src.models.clean_release import ( from src.services.clean_release.preparation_service import prepare_candidate -# #region _mock_policy [TYPE Function] -# @BRIEF Build a valid clean profile policy fixture for preparation tests. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:_mock_policy:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Build a valid clean profile policy fixture for preparation tests. def _mock_policy() -> CleanProfilePolicy: return CleanProfilePolicy( policy_id="pol-1", @@ -37,12 +39,12 @@ def _mock_policy() -> CleanProfilePolicy: ) -# #endregion _mock_policy +# [/DEF:_mock_policy:Function] -# #region _mock_registry [TYPE Function] -# @BRIEF Build an internal-only source registry fixture for preparation tests. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:_mock_registry:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Build an internal-only source registry fixture for preparation tests. def _mock_registry() -> ResourceSourceRegistry: return ResourceSourceRegistry( registry_id="reg-1", @@ -61,12 +63,12 @@ def _mock_registry() -> ResourceSourceRegistry: ) -# #endregion _mock_registry +# [/DEF:_mock_registry:Function] -# #region _mock_candidate [TYPE Function] -# @BRIEF Build a draft release candidate fixture with provided identifier. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:_mock_candidate:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Build a draft release candidate fixture with provided identifier. def _mock_candidate(candidate_id: str) -> ReleaseCandidate: return ReleaseCandidate( candidate_id=candidate_id, @@ -79,12 +81,12 @@ def _mock_candidate(candidate_id: str) -> ReleaseCandidate: ) -# #endregion _mock_candidate +# [/DEF:_mock_candidate:Function] -# #region test_prepare_candidate_success [TYPE Function] -# @BRIEF Verify candidate transitions to PREPARED when evaluation returns no violations. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:test_prepare_candidate_success:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Verify candidate transitions to PREPARED when evaluation returns no violations. # @TEST_CONTRACT: [valid_candidate + active_policy + internal_sources + no_violations] -> [status=PREPARED, manifest_persisted, candidate_saved] # @TEST_SCENARIO: [prepare_success] -> [prepared status and persistence side effects are produced] # @TEST_FIXTURE: [INLINE_MOCKS] -> INLINE_JSON @@ -131,12 +133,12 @@ def test_prepare_candidate_success(): repository.save_candidate.assert_called_with(candidate) -# #endregion test_prepare_candidate_success +# [/DEF:test_prepare_candidate_success:Function] -# #region test_prepare_candidate_with_violations [TYPE Function] -# @BRIEF Verify candidate transitions to BLOCKED when evaluation returns blocking violations. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:test_prepare_candidate_with_violations:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Verify candidate transitions to BLOCKED when evaluation returns blocking violations. # @TEST_CONTRACT: [valid_candidate + active_policy + evaluation_with_violations] -> [status=BLOCKED, violations_exposed] # @TEST_SCENARIO: [prepare_blocked_due_to_policy] -> [blocked status and violation list are produced] # @TEST_FIXTURE: [INLINE_MOCKS] -> INLINE_JSON @@ -182,12 +184,12 @@ def test_prepare_candidate_with_violations(): assert len(result["violations"]) == 1 -# #endregion test_prepare_candidate_with_violations +# [/DEF:test_prepare_candidate_with_violations:Function] -# #region test_prepare_candidate_not_found [TYPE Function] -# @BRIEF Verify preparation raises ValueError when candidate does not exist. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:test_prepare_candidate_not_found:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Verify preparation raises ValueError when candidate does not exist. # @TEST_CONTRACT: [missing_candidate] -> [ValueError('Candidate not found')] # @TEST_SCENARIO: [prepare_missing_candidate] -> [raises candidate not found error] # @TEST_FIXTURE: [INLINE_MOCKS] -> INLINE_JSON @@ -201,12 +203,12 @@ def test_prepare_candidate_not_found(): prepare_candidate(repository, "non-existent", [], [], "op") -# #endregion test_prepare_candidate_not_found +# [/DEF:test_prepare_candidate_not_found:Function] -# #region test_prepare_candidate_no_active_policy [TYPE Function] -# @BRIEF Verify preparation raises ValueError when no active policy is available. -# @RELATION BINDS_TO -> [TestPreparationService] +# [DEF:test_prepare_candidate_no_active_policy:Function] +# @RELATION: BINDS_TO -> TestPreparationService +# @PURPOSE: Verify preparation raises ValueError when no active policy is available. # @TEST_CONTRACT: [candidate_present + missing_active_policy] -> [ValueError('Active clean policy not found')] # @TEST_SCENARIO: [prepare_missing_policy] -> [raises active policy missing error] # @TEST_FIXTURE: [INLINE_MOCKS] -> INLINE_JSON @@ -221,7 +223,7 @@ def test_prepare_candidate_no_active_policy(): prepare_candidate(repository, "cand-1", [], [], "op") -# #endregion test_prepare_candidate_no_active_policy +# [/DEF:test_prepare_candidate_no_active_policy:Function] -# #endregion TestPreparationService +# [/DEF:TestPreparationService:Module] diff --git a/backend/src/services/clean_release/__tests__/test_report_builder.py b/backend/src/services/clean_release/__tests__/test_report_builder.py index 65075339..a0c4b10f 100644 --- a/backend/src/services/clean_release/__tests__/test_report_builder.py +++ b/backend/src/services/clean_release/__tests__/test_report_builder.py @@ -1,8 +1,10 @@ -# #region TestReportBuilder [C:3] [TYPE Module] [SEMANTICS tests, clean-release, report-builder, counters] -# @BRIEF Validate compliance report builder counter integrity and blocked-run constraints. -# @LAYER Domain -# @INVARIANT blocked run requires at least one blocking violation. -# @RELATION DEPENDS_ON -> [ReportBuilder] +# [DEF:TestReportBuilder:Module] +# @RELATION: [DEPENDS_ON] ->[ReportBuilder] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, report-builder, counters +# @PURPOSE: Validate compliance report builder counter integrity and blocked-run constraints. +# @LAYER: Domain +# @INVARIANT: blocked run requires at least one blocking violation. from datetime import datetime, timezone @@ -20,9 +22,9 @@ from src.services.clean_release.report_builder import ComplianceReportBuilder from src.services.clean_release.repository import CleanReleaseRepository -# #region _terminal_run [TYPE Function] -# @BRIEF Build terminal/non-terminal run fixtures for report builder tests. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:_terminal_run:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Build terminal/non-terminal run fixtures for report builder tests. def _terminal_run(status: CheckFinalStatus) -> ComplianceCheckRun: return ComplianceCheckRun( check_run_id="check-1", @@ -37,12 +39,12 @@ def _terminal_run(status: CheckFinalStatus) -> ComplianceCheckRun: ) -# #endregion _terminal_run +# [/DEF:_terminal_run:Function] -# #region _blocking_violation [TYPE Function] -# @BRIEF Build a blocking violation fixture for blocked report scenarios. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:_blocking_violation:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Build a blocking violation fixture for blocked report scenarios. def _blocking_violation() -> ComplianceViolation: return ComplianceViolation( violation_id="viol-1", @@ -56,12 +58,12 @@ def _blocking_violation() -> ComplianceViolation: ) -# #endregion _blocking_violation +# [/DEF:_blocking_violation:Function] -# #region test_report_builder_blocked_requires_blocking_violations [TYPE Function] -# @BRIEF Verify BLOCKED run requires at least one blocking violation. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:test_report_builder_blocked_requires_blocking_violations:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Verify BLOCKED run requires at least one blocking violation. def test_report_builder_blocked_requires_blocking_violations(): builder = ComplianceReportBuilder(CleanReleaseRepository()) run = _terminal_run(CheckFinalStatus.BLOCKED) @@ -70,12 +72,12 @@ def test_report_builder_blocked_requires_blocking_violations(): builder.build_report_payload(run, []) -# #endregion test_report_builder_blocked_requires_blocking_violations +# [/DEF:test_report_builder_blocked_requires_blocking_violations:Function] -# #region test_report_builder_blocked_with_two_violations [TYPE Function] -# @BRIEF Verify report builder generates conformant payload for a BLOCKED run with violations. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:test_report_builder_blocked_with_two_violations:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Verify report builder generates conformant payload for a BLOCKED run with violations. def test_report_builder_blocked_with_two_violations(): builder = ComplianceReportBuilder(CleanReleaseRepository()) run = _terminal_run(CheckFinalStatus.BLOCKED) @@ -93,12 +95,12 @@ def test_report_builder_blocked_with_two_violations(): assert report.blocking_violations_count == 2 -# #endregion test_report_builder_blocked_with_two_violations +# [/DEF:test_report_builder_blocked_with_two_violations:Function] -# #region test_report_builder_counter_consistency [TYPE Function] -# @BRIEF Verify violations counters remain consistent for blocking payload. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:test_report_builder_counter_consistency:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Verify violations counters remain consistent for blocking payload. def test_report_builder_counter_consistency(): builder = ComplianceReportBuilder(CleanReleaseRepository()) run = _terminal_run(CheckFinalStatus.BLOCKED) @@ -108,12 +110,12 @@ def test_report_builder_counter_consistency(): assert report.blocking_violations_count == 1 -# #endregion test_report_builder_counter_consistency +# [/DEF:test_report_builder_counter_consistency:Function] -# #region test_missing_operator_summary [TYPE Function] -# @BRIEF Validate non-terminal run prevents operator summary/report generation. -# @RELATION BINDS_TO -> [TestReportBuilder] +# [DEF:test_missing_operator_summary:Function] +# @RELATION: BINDS_TO -> TestReportBuilder +# @PURPOSE: Validate non-terminal run prevents operator summary/report generation. def test_missing_operator_summary(): builder = ComplianceReportBuilder(CleanReleaseRepository()) run = _terminal_run(CheckFinalStatus.RUNNING) @@ -124,5 +126,5 @@ def test_missing_operator_summary(): assert "Cannot build report for non-terminal run" in str(exc.value) -# #endregion test_missing_operator_summary -# #endregion TestReportBuilder +# [/DEF:test_missing_operator_summary:Function] +# [/DEF:TestReportBuilder:Module] diff --git a/backend/src/services/clean_release/__tests__/test_source_isolation.py b/backend/src/services/clean_release/__tests__/test_source_isolation.py index 380c0833..12c9aa9b 100644 --- a/backend/src/services/clean_release/__tests__/test_source_isolation.py +++ b/backend/src/services/clean_release/__tests__/test_source_isolation.py @@ -1,8 +1,10 @@ -# #region TestSourceIsolation [C:3] [TYPE Module] [SEMANTICS tests, clean-release, source-isolation, internal-only] -# @BRIEF Verify internal source registry validation behavior. -# @LAYER Domain -# @INVARIANT External endpoints always produce blocking violations. -# @RELATION DEPENDS_ON -> [SourceIsolation] +# [DEF:TestSourceIsolation:Module] +# @RELATION: [DEPENDS_ON] ->[SourceIsolation] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, source-isolation, internal-only +# @PURPOSE: Verify internal source registry validation behavior. +# @LAYER: Domain +# @INVARIANT: External endpoints always produce blocking violations. from datetime import datetime, timezone @@ -10,8 +12,8 @@ from src.models.clean_release import ResourceSourceEntry, ResourceSourceRegistry from src.services.clean_release.source_isolation import validate_internal_sources -# #region _registry [TYPE Function] -# @RELATION BINDS_TO -> [TestSourceIsolation] +# [DEF:_registry:Function] +# @RELATION: BINDS_TO -> TestSourceIsolation def _registry() -> ResourceSourceRegistry: return ResourceSourceRegistry( registry_id="registry-internal-v1", @@ -38,12 +40,12 @@ def _registry() -> ResourceSourceRegistry: ) -# #endregion _registry +# [/DEF:_registry:Function] -# #region test_validate_internal_sources_all_internal_ok [TYPE Function] -# @BRIEF Verify validate_internal_sources passes when all sources are internal and allowed. -# @RELATION BINDS_TO -> [TestSourceIsolation] +# [DEF:test_validate_internal_sources_all_internal_ok:Function] +# @RELATION: BINDS_TO -> TestSourceIsolation +# @PURPOSE: Verify validate_internal_sources passes when all sources are internal and allowed. def test_validate_internal_sources_all_internal_ok(): result = validate_internal_sources( registry=_registry(), @@ -53,12 +55,12 @@ def test_validate_internal_sources_all_internal_ok(): assert result["violations"] == [] -# #endregion test_validate_internal_sources_all_internal_ok +# [/DEF:test_validate_internal_sources_all_internal_ok:Function] -# #region test_validate_internal_sources_external_blocked [TYPE Function] -# @BRIEF Verify validate_internal_sources blocks external sources when policy requires internal-only. -# @RELATION BINDS_TO -> [TestSourceIsolation] +# [DEF:test_validate_internal_sources_external_blocked:Function] +# @RELATION: BINDS_TO -> TestSourceIsolation +# @PURPOSE: Verify validate_internal_sources blocks external sources when policy requires internal-only. def test_validate_internal_sources_external_blocked(): result = validate_internal_sources( registry=_registry(), @@ -70,5 +72,5 @@ def test_validate_internal_sources_external_blocked(): assert result["violations"][0]["blocked_release"] is True -# #endregion test_validate_internal_sources_external_blocked -# #endregion TestSourceIsolation +# [/DEF:test_validate_internal_sources_external_blocked:Function] +# [/DEF:TestSourceIsolation:Module] diff --git a/backend/src/services/clean_release/__tests__/test_stages.py b/backend/src/services/clean_release/__tests__/test_stages.py index 4cb1581e..8cc69a63 100644 --- a/backend/src/services/clean_release/__tests__/test_stages.py +++ b/backend/src/services/clean_release/__tests__/test_stages.py @@ -1,7 +1,9 @@ -# #region TestStages [C:3] [TYPE Module] [SEMANTICS tests, clean-release, compliance, stages] -# @BRIEF Validate final status derivation logic from stage results. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [ComplianceStages] +# [DEF:TestStages:Module] +# @RELATION: [DEPENDS_ON] ->[ComplianceStages] +# @COMPLEXITY: 3 +# @SEMANTICS: tests, clean-release, compliance, stages +# @PURPOSE: Validate final status derivation logic from stage results. +# @LAYER: Domain from src.models.clean_release import ( CheckFinalStatus, @@ -12,9 +14,9 @@ from src.models.clean_release import ( from src.services.clean_release.stages import derive_final_status, MANDATORY_STAGE_ORDER -# #region test_derive_final_status_compliant [TYPE Function] -# @BRIEF Verify derive_final_status returns compliant when all stages pass. -# @RELATION BINDS_TO -> [TestStages] +# [DEF:test_derive_final_status_compliant:Function] +# @RELATION: BINDS_TO -> TestStages +# @PURPOSE: Verify derive_final_status returns compliant when all stages pass. def test_derive_final_status_compliant(): results = [ CheckStageResult(stage=s, status=CheckStageStatus.PASS, details="ok") @@ -23,12 +25,12 @@ def test_derive_final_status_compliant(): assert derive_final_status(results) == CheckFinalStatus.COMPLIANT -# #endregion test_derive_final_status_compliant +# [/DEF:test_derive_final_status_compliant:Function] -# #region test_derive_final_status_blocked [TYPE Function] -# @BRIEF Verify derive_final_status returns blocked when any stage fails. -# @RELATION BINDS_TO -> [TestStages] +# [DEF:test_derive_final_status_blocked:Function] +# @RELATION: BINDS_TO -> TestStages +# @PURPOSE: Verify derive_final_status returns blocked when any stage fails. def test_derive_final_status_blocked(): results = [ CheckStageResult(stage=s, status=CheckStageStatus.PASS, details="ok") @@ -38,12 +40,12 @@ def test_derive_final_status_blocked(): assert derive_final_status(results) == CheckFinalStatus.BLOCKED -# #endregion test_derive_final_status_blocked +# [/DEF:test_derive_final_status_blocked:Function] -# #region test_derive_final_status_failed_missing [TYPE Function] -# @BRIEF Verify derive_final_status returns failed when required stages are missing. -# @RELATION BINDS_TO -> [TestStages] +# [DEF:test_derive_final_status_failed_missing:Function] +# @RELATION: BINDS_TO -> TestStages +# @PURPOSE: Verify derive_final_status returns failed when required stages are missing. def test_derive_final_status_failed_missing(): results = [ CheckStageResult( @@ -53,12 +55,12 @@ def test_derive_final_status_failed_missing(): assert derive_final_status(results) == CheckFinalStatus.FAILED -# #endregion test_derive_final_status_failed_missing +# [/DEF:test_derive_final_status_failed_missing:Function] -# #region test_derive_final_status_failed_skipped [TYPE Function] -# @BRIEF Verify derive_final_status returns failed when critical stages are skipped. -# @RELATION BINDS_TO -> [TestStages] +# [DEF:test_derive_final_status_failed_skipped:Function] +# @RELATION: BINDS_TO -> TestStages +# @PURPOSE: Verify derive_final_status returns failed when critical stages are skipped. def test_derive_final_status_failed_skipped(): results = [ CheckStageResult(stage=s, status=CheckStageStatus.PASS, details="ok") @@ -68,5 +70,5 @@ def test_derive_final_status_failed_skipped(): assert derive_final_status(results) == CheckFinalStatus.FAILED -# #endregion test_derive_final_status_failed_skipped -# #endregion TestStages +# [/DEF:test_derive_final_status_failed_skipped:Function] +# [/DEF:TestStages:Module] diff --git a/backend/src/services/clean_release/approval_service.py b/backend/src/services/clean_release/approval_service.py index c439bf41..ebaac8b5 100644 --- a/backend/src/services/clean_release/approval_service.py +++ b/backend/src/services/clean_release/approval_service.py @@ -1,11 +1,9 @@ -# [DEF:ApprovalService:Module] -# @COMPLEXITY: 5 -# @SEMANTICS: clean-release, approval, decision, lifecycle, gate -# @PURPOSE: Enforce approval/rejection gates over immutable compliance reports. +# #region ApprovalService [C:5] [TYPE Module] [SEMANTICS clean-release, approval, decision, lifecycle, gate] +# @BRIEF Enforce approval/rejection gates over immutable compliance reports. # @LAYER: Domain -# @RELATION: [DEPENDS_ON] ->[RepositoryRelations] -# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels] -# @RELATION: [DEPENDS_ON] ->[AuditService] +# @RELATION DEPENDS_ON -> [RepositoryRelations] +# @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @RELATION DEPENDS_ON -> [AuditService] # @INVARIANT: Approval is allowed only for PASSED report bound to candidate; decisions are append-only. from __future__ import annotations @@ -24,8 +22,8 @@ from .repository import CleanReleaseRepository # #region _get_or_init_decisions_store [TYPE Function] # @BRIEF Provide append-only in-memory storage for approval decisions. -# @PRE repository is initialized. -# @POST Returns mutable decision list attached to repository. +# @PRE: repository is initialized. +# @POST: Returns mutable decision list attached to repository. def _get_or_init_decisions_store( repository: CleanReleaseRepository, ) -> List[ApprovalDecision]: @@ -41,8 +39,8 @@ def _get_or_init_decisions_store( # #region _latest_decision_for_candidate [TYPE Function] # @BRIEF Resolve latest approval decision for candidate from append-only store. -# @PRE candidate_id is non-empty. -# @POST Returns latest ApprovalDecision or None. +# @PRE: candidate_id is non-empty. +# @POST: Returns latest ApprovalDecision or None. def _latest_decision_for_candidate( repository: CleanReleaseRepository, candidate_id: str ) -> ApprovalDecision | None: @@ -62,8 +60,8 @@ def _latest_decision_for_candidate( # #region _resolve_candidate_and_report [TYPE Function] # @BRIEF Validate candidate/report existence and ownership prior to decision persistence. -# @PRE candidate_id and report_id are non-empty. -# @POST Returns tuple(candidate, report); raises ApprovalGateError on contract violation. +# @PRE: candidate_id and report_id are non-empty. +# @POST: Returns tuple(candidate, report); raises ApprovalGateError on contract violation. def _resolve_candidate_and_report( repository: CleanReleaseRepository, *, @@ -89,8 +87,8 @@ def _resolve_candidate_and_report( # #region approve_candidate [TYPE Function] # @BRIEF Persist immutable APPROVED decision and advance candidate lifecycle to APPROVED. -# @PRE Candidate exists, report belongs to candidate, report final_status is PASSED, candidate not already APPROVED. -# @POST Approval decision is appended and candidate transitions to APPROVED. +# @PRE: Candidate exists, report belongs to candidate, report final_status is PASSED, candidate not already APPROVED. +# @POST: Approval decision is appended and candidate transitions to APPROVED. def approve_candidate( *, repository: CleanReleaseRepository, @@ -165,8 +163,8 @@ def approve_candidate( # #region reject_candidate [TYPE Function] # @BRIEF Persist immutable REJECTED decision without promoting candidate lifecycle. -# @PRE Candidate exists and report belongs to candidate. -# @POST Rejected decision is appended; candidate lifecycle is unchanged. +# @PRE: Candidate exists and report belongs to candidate. +# @POST: Rejected decision is appended; candidate lifecycle is unchanged. def reject_candidate( *, repository: CleanReleaseRepository, @@ -210,4 +208,4 @@ def reject_candidate( # #endregion reject_candidate -# [/DEF:backend.src.services.clean_release.approval_service:Module] +# #endregion backend.src.services.clean_release.approval_service diff --git a/backend/src/services/clean_release/artifact_catalog_loader.py b/backend/src/services/clean_release/artifact_catalog_loader.py index 3540dae6..22775ac4 100644 --- a/backend/src/services/clean_release/artifact_catalog_loader.py +++ b/backend/src/services/clean_release/artifact_catalog_loader.py @@ -1,9 +1,7 @@ -# [DEF:ArtifactCatalogLoader:Module] -# @COMPLEXITY: 3 -# @SEMANTICS: clean-release, artifacts, bootstrap, json, tui -# @PURPOSE: Load bootstrap artifact catalogs for clean release real-mode flows. +# #region ArtifactCatalogLoader [C:3] [TYPE Module] [SEMANTICS clean-release, artifacts, bootstrap, json, tui] +# @BRIEF Load bootstrap artifact catalogs for clean release real-mode flows. # @LAYER: Domain -# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels] +# @RELATION DEPENDS_ON -> [CleanReleaseModels] # @INVARIANT: Artifact catalog must produce deterministic CandidateArtifact entries with required identity and checksum fields. from __future__ import annotations @@ -17,8 +15,8 @@ from ...models.clean_release import CandidateArtifact # #region load_bootstrap_artifacts [TYPE Function] # @BRIEF Parse artifact catalog JSON into CandidateArtifact models for TUI/bootstrap flows. -# @PRE path points to readable JSON file; payload is list[artifact] or {"artifacts": list[artifact]}. -# @POST Returns non-mutated CandidateArtifact models with required fields populated. +# @PRE: path points to readable JSON file; payload is list[artifact] or {"artifacts": list[artifact]}. +# @POST: Returns non-mutated CandidateArtifact models with required fields populated. def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArtifact]: if not path or not path.strip(): return [] @@ -104,4 +102,4 @@ def load_bootstrap_artifacts(path: str, candidate_id: str) -> List[CandidateArti # #endregion load_bootstrap_artifacts -# [/DEF:backend.src.services.clean_release.artifact_catalog_loader:Module] +# #endregion backend.src.services.clean_release.artifact_catalog_loader diff --git a/backend/src/services/clean_release/audit_service.py b/backend/src/services/clean_release/audit_service.py index 3e5f2e61..a61089db 100644 --- a/backend/src/services/clean_release/audit_service.py +++ b/backend/src/services/clean_release/audit_service.py @@ -1,9 +1,7 @@ -# [DEF:AuditService:Module] -# @COMPLEXITY: 3 -# @SEMANTICS: clean-release, audit, lifecycle, logging -# @PURPOSE: Provide lightweight audit hooks for clean release preparation/check/report lifecycle. +# #region AuditService [C:3] [TYPE Module] [SEMANTICS clean-release, audit, lifecycle, logging] +# @BRIEF Provide lightweight audit hooks for clean release preparation/check/report lifecycle. # @LAYER: Infra -# @RELATION: [DEPENDS_ON] ->[LoggerModule] +# @RELATION DEPENDS_ON -> [LoggerModule] # @INVARIANT: Audit hooks are append-only log actions. from __future__ import annotations @@ -116,4 +114,4 @@ def audit_report( ) -# [/DEF:backend.src.services.clean_release.audit_service:Module] +# #endregion backend.src.services.clean_release.audit_service diff --git a/backend/src/services/clean_release/candidate_service.py b/backend/src/services/clean_release/candidate_service.py index 1b83d8dd..fcb6b033 100644 --- a/backend/src/services/clean_release/candidate_service.py +++ b/backend/src/services/clean_release/candidate_service.py @@ -1,11 +1,11 @@ # #region candidate_service [C:5] [TYPE Module] [SEMANTICS clean-release, candidate, artifacts, lifecycle, validation] # @BRIEF Register release candidates with validated artifacts and advance lifecycle through legal transitions. -# @LAYER Domain -# @PRE candidate_id must be unique; artifacts input must be non-empty and valid. -# @POST candidate and artifacts are persisted; candidate transitions DRAFT -> PREPARED only. -# @INVARIANT Candidate lifecycle transitions are delegated to domain guard logic. -# @RELATION DEPENDS_ON -> [backend.src.services.clean_release.repository] -# @RELATION DEPENDS_ON -> [backend.src.models.clean_release] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> backend.src.services.clean_release.repository +# @RELATION DEPENDS_ON -> backend.src.models.clean_release +# @PRE: candidate_id must be unique; artifacts input must be non-empty and valid. +# @POST: candidate and artifacts are persisted; candidate transitions DRAFT -> PREPARED only. +# @INVARIANT: Candidate lifecycle transitions are delegated to domain guard logic. from __future__ import annotations @@ -19,8 +19,8 @@ from .repository import CleanReleaseRepository # #region _validate_artifacts [TYPE Function] # @BRIEF Validate raw artifact payload list for required fields and shape. -# @PRE artifacts payload is provided by caller. -# @POST Returns normalized artifact list or raises ValueError. +# @PRE: artifacts payload is provided by caller. +# @POST: Returns normalized artifact list or raises ValueError. def _validate_artifacts(artifacts: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]: normalized = list(artifacts) if not normalized: @@ -47,8 +47,8 @@ def _validate_artifacts(artifacts: Iterable[Dict[str, Any]]) -> List[Dict[str, A # #region register_candidate [TYPE Function] # @BRIEF Register a candidate and persist its artifacts with legal lifecycle transition. -# @PRE candidate_id must be unique and artifacts must pass validation. -# @POST Candidate exists in repository with PREPARED status and artifacts persisted. +# @PRE: candidate_id must be unique and artifacts must pass validation. +# @POST: Candidate exists in repository with PREPARED status and artifacts persisted. def register_candidate( repository: CleanReleaseRepository, candidate_id: str, @@ -102,4 +102,4 @@ def register_candidate( return candidate # #endregion register_candidate -# #endregion candidate_service +# #endregion backend.src.services.clean_release.candidate_service \ No newline at end of file diff --git a/backend/src/services/clean_release/compliance_execution_service.py b/backend/src/services/clean_release/compliance_execution_service.py index 02dab513..fe72eaae 100644 --- a/backend/src/services/clean_release/compliance_execution_service.py +++ b/backend/src/services/clean_release/compliance_execution_service.py @@ -1,15 +1,15 @@ # #region ComplianceExecutionService [C:5] [TYPE Module] [SEMANTICS clean-release, compliance, execution, stages, immutable-evidence] # @BRIEF Create and execute compliance runs with trusted snapshots, deterministic stages, violations and immutable report persistence. -# @LAYER Domain -# @PRE Repository adapters, trusted policy resolution, and deterministic stage implementations are available for the run request. -# @POST Candidate-scoped compliance runs persist stage evidence, terminal status, and immutable report artifacts when execution succeeds. -# @SIDE_EFFECT Persists runs, stage results, violations, and reports through repository adapters and audit helpers. -# @DATA_CONTRACT Input[candidate_id, requested_by, manifest_id?, policy snapshots, stage results] -> Output[ComplianceExecutionResult] -# @INVARIANT A run binds to exactly one candidate/manifest/policy/registry snapshot set. -# @RELATION DEPENDS_ON -> [RepositoryRelations] -# @RELATION DEPENDS_ON -> [PolicyResolutionService] -# @RELATION DEPENDS_ON -> [ComplianceStages] -# @RELATION DEPENDS_ON -> [ReportBuilder] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> RepositoryRelations +# @RELATION DEPENDS_ON -> PolicyResolutionService +# @RELATION DEPENDS_ON -> ComplianceStages +# @RELATION DEPENDS_ON -> ReportBuilder +# @PRE: Repository adapters, trusted policy resolution, and deterministic stage implementations are available for the run request. +# @POST: Candidate-scoped compliance runs persist stage evidence, terminal status, and immutable report artifacts when execution succeeds. +# @SIDE_EFFECT: Persists runs, stage results, violations, and reports through repository adapters and audit helpers. +# @DATA_CONTRACT: Input[candidate_id, requested_by, manifest_id?, policy snapshots, stage results] -> Output[ComplianceExecutionResult] +# @INVARIANT: A run binds to exactly one candidate/manifest/policy/registry snapshot set. from __future__ import annotations @@ -54,10 +54,10 @@ class ComplianceExecutionResult: # #region ComplianceExecutionService [TYPE Class] # @BRIEF Execute clean-release compliance lifecycle over trusted snapshots and immutable evidence. -# @PRE Database session active, candidate registered -# @POST Returns ComplianceReport with pass/fail status and violation details -# @SIDE_EFFECT Updates compliance status in database, logs violations -# @DATA_CONTRACT ComplianceCheckResult, ComplianceReport, Violation +# @PRE: Database session active, candidate registered +# @POST: Returns ComplianceReport with pass/fail status and violation details +# @SIDE_EFFECT: Updates compliance status in database, logs violations +# @DATA_CONTRACT: ComplianceCheckResult, ComplianceReport, Violation class ComplianceExecutionService: TASK_PLUGIN_ID = "clean-release-compliance" @@ -73,10 +73,10 @@ class ComplianceExecutionService: self.stages = list(stages) if stages is not None else build_default_stages() self.report_builder = ComplianceReportBuilder(repository) - # #region _resolve_manifest [TYPE Function] - # @BRIEF Resolve explicit manifest or fallback to latest candidate manifest. - # @PRE candidate exists. - # @POST Returns manifest snapshot or raises ComplianceRunError. + # [DEF:_resolve_manifest:Function] + # @PURPOSE: Resolve explicit manifest or fallback to latest candidate manifest. + # @PRE: candidate exists. + # @POST: Returns manifest snapshot or raises ComplianceRunError. def _resolve_manifest( self, candidate_id: str, manifest_id: Optional[str] ) -> DistributionManifest: @@ -99,29 +99,29 @@ class ComplianceExecutionService: manifests, key=lambda item: item.manifest_version, reverse=True )[0] - # #endregion _resolve_manifest + # [/DEF:_resolve_manifest:Function] - # #region _persist_stage_run [TYPE Function] - # @BRIEF Persist stage run if repository supports stage records. - # @POST Stage run is persisted when adapter is available, otherwise no-op. + # [DEF:_persist_stage_run:Function] + # @PURPOSE: Persist stage run if repository supports stage records. + # @POST: Stage run is persisted when adapter is available, otherwise no-op. def _persist_stage_run(self, stage_run: ComplianceStageRun) -> None: self.repository.save_stage_run(stage_run) - # #endregion _persist_stage_run + # [/DEF:_persist_stage_run:Function] - # #region _persist_violations [TYPE Function] - # @BRIEF Persist stage violations via repository adapters. - # @POST Violations are appended to repository evidence store. + # [DEF:_persist_violations:Function] + # @PURPOSE: Persist stage violations via repository adapters. + # @POST: Violations are appended to repository evidence store. def _persist_violations(self, violations: List[ComplianceViolation]) -> None: for violation in violations: self.repository.save_violation(violation) - # #endregion _persist_violations + # [/DEF:_persist_violations:Function] - # #region execute_run [TYPE Function] - # @BRIEF Execute compliance run stages and finalize immutable report on terminal success. - # @PRE candidate exists and trusted policy/registry snapshots are resolvable. - # @POST Run and evidence are persisted; report exists for SUCCEEDED runs. + # [DEF:execute_run:Function] + # @PURPOSE: Execute compliance run stages and finalize immutable report on terminal success. + # @PRE: candidate exists and trusted policy/registry snapshots are resolvable. + # @POST: Run and evidence are persisted; report exists for SUCCEEDED runs. def execute_run( self, *, @@ -223,7 +223,7 @@ class ComplianceExecutionService: violations=violations, ) - # #endregion execute_run + # [/DEF:execute_run:Function] # #endregion ComplianceExecutionService diff --git a/backend/src/services/clean_release/compliance_orchestrator.py b/backend/src/services/clean_release/compliance_orchestrator.py index 414a3910..39de1ccc 100644 --- a/backend/src/services/clean_release/compliance_orchestrator.py +++ b/backend/src/services/clean_release/compliance_orchestrator.py @@ -1,10 +1,10 @@ # #region ComplianceOrchestrator [C:5] [TYPE Module] [SEMANTICS clean-release, orchestrator, compliance-gate, stages] # @BRIEF Execute mandatory clean compliance stages and produce final COMPLIANT/BLOCKED/FAILED outcome. -# @LAYER Domain -# @INVARIANT COMPLIANT is impossible when any mandatory stage fails. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [ComplianceStages] # @RELATION DEPENDS_ON -> [RepositoryRelations] # @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @INVARIANT: COMPLIANT is impossible when any mandatory stage fails. # @TEST_CONTRACT: ComplianceCheckRun -> ComplianceCheckRun # @TEST_FIXTURE: compliant_candidate -> file:backend/tests/fixtures/clean_release/fixtures_clean_release.json # @TEST_EDGE: stage_failure_blocks_release -> Mandatory stage returns FAIL and final status becomes BLOCKED @@ -44,24 +44,24 @@ from ...core.logger import belief_scope, logger # #region CleanComplianceOrchestrator [TYPE Class] # @BRIEF Coordinate clean-release compliance verification stages. class CleanComplianceOrchestrator: - # #region __init__ [TYPE Function] - # @BRIEF Bind repository dependency used for orchestrator persistence and lookups. - # @PRE repository is a valid CleanReleaseRepository instance with required methods. - # @POST self.repository is assigned and used by all orchestration steps. - # @SIDE_EFFECT Stores repository reference on orchestrator instance. - # @DATA_CONTRACT Input -> CleanReleaseRepository, Output -> None + # [DEF:__init__:Function] + # @PURPOSE: Bind repository dependency used for orchestrator persistence and lookups. + # @PRE: repository is a valid CleanReleaseRepository instance with required methods. + # @POST: self.repository is assigned and used by all orchestration steps. + # @SIDE_EFFECT: Stores repository reference on orchestrator instance. + # @DATA_CONTRACT: Input -> CleanReleaseRepository, Output -> None def __init__(self, repository: CleanReleaseRepository): with belief_scope("CleanComplianceOrchestrator.__init__"): self.repository = repository - # #endregion __init__ + # [/DEF:__init__:Function] - # #region start_check_run [TYPE Function] - # @BRIEF Initiate a new compliance run session. - # @PRE candidate_id and policy_id are provided; legacy callers may omit persisted manifest/policy records. - # @POST Returns initialized ComplianceRun in RUNNING state persisted in repository. - # @SIDE_EFFECT Reads manifest/policy when present and writes new ComplianceRun via repository.save_check_run. - # @DATA_CONTRACT Input -> (candidate_id:str, policy_id:str, requested_by:str, manifest_id:str|None), Output -> ComplianceRun + # [DEF:start_check_run:Function] + # @PURPOSE: Initiate a new compliance run session. + # @PRE: candidate_id and policy_id are provided; legacy callers may omit persisted manifest/policy records. + # @POST: Returns initialized ComplianceRun in RUNNING state persisted in repository. + # @SIDE_EFFECT: Reads manifest/policy when present and writes new ComplianceRun via repository.save_check_run. + # @DATA_CONTRACT: Input -> (candidate_id:str, policy_id:str, requested_by:str, manifest_id:str|None), Output -> ComplianceRun def start_check_run( self, candidate_id: str, @@ -149,14 +149,14 @@ class CleanComplianceOrchestrator: ) return self.repository.save_check_run(check_run) - # #endregion start_check_run + # [/DEF:start_check_run:Function] - # #region execute_stages [TYPE Function] - # @BRIEF Execute or accept compliance stage outcomes and set intermediate/final check-run status fields. - # @PRE check_run exists and references candidate/policy/registry/manifest identifiers resolvable by repository. - # @POST Returns persisted ComplianceRun with status FAILED on missing dependencies, otherwise SUCCEEDED with final_status set. - # @SIDE_EFFECT Reads candidate/policy/registry/manifest and persists updated check_run. - # @DATA_CONTRACT Input -> (check_run:ComplianceRun, forced_results:Optional[List[ComplianceStageRun]]), Output -> ComplianceRun + # [DEF:execute_stages:Function] + # @PURPOSE: Execute or accept compliance stage outcomes and set intermediate/final check-run status fields. + # @PRE: check_run exists and references candidate/policy/registry/manifest identifiers resolvable by repository. + # @POST: Returns persisted ComplianceRun with status FAILED on missing dependencies, otherwise SUCCEEDED with final_status set. + # @SIDE_EFFECT: Reads candidate/policy/registry/manifest and persists updated check_run. + # @DATA_CONTRACT: Input -> (check_run:ComplianceRun, forced_results:Optional[List[ComplianceStageRun]]), Output -> ComplianceRun def execute_stages( self, check_run: ComplianceRun, @@ -211,14 +211,14 @@ class CleanComplianceOrchestrator: return self.repository.save_check_run(check_run) - # #endregion execute_stages + # [/DEF:execute_stages:Function] - # #region finalize_run [TYPE Function] - # @BRIEF Finalize run status based on cumulative stage results. - # @PRE check_run was started and may already contain a derived final_status from stage execution. - # @POST Returns persisted ComplianceRun in SUCCEEDED status with final_status guaranteed non-empty. - # @SIDE_EFFECT Mutates check_run terminal fields and persists via repository.save_check_run. - # @DATA_CONTRACT Input -> ComplianceRun, Output -> ComplianceRun + # [DEF:finalize_run:Function] + # @PURPOSE: Finalize run status based on cumulative stage results. + # @PRE: check_run was started and may already contain a derived final_status from stage execution. + # @POST: Returns persisted ComplianceRun in SUCCEEDED status with final_status guaranteed non-empty. + # @SIDE_EFFECT: Mutates check_run terminal fields and persists via repository.save_check_run. + # @DATA_CONTRACT: Input -> ComplianceRun, Output -> ComplianceRun def finalize_run(self, check_run: ComplianceRun) -> ComplianceRun: with belief_scope("finalize_run"): if check_run.status == RunStatus.FAILED: @@ -238,7 +238,7 @@ class CleanComplianceOrchestrator: check_run.finished_at = datetime.now(timezone.utc) return self.repository.save_check_run(check_run) - # #endregion finalize_run + # [/DEF:finalize_run:Function] # #endregion CleanComplianceOrchestrator @@ -246,10 +246,10 @@ class CleanComplianceOrchestrator: # #region run_check_legacy [TYPE Function] # @BRIEF Legacy wrapper for compatibility with previous orchestrator call style. -# @PRE repository and identifiers are valid and resolvable by orchestrator dependencies. -# @POST Returns finalized ComplianceRun produced by orchestrator start->execute->finalize sequence. -# @SIDE_EFFECT Reads/writes compliance entities through repository during orchestrator calls. -# @DATA_CONTRACT Input -> (repository:CleanReleaseRepository, candidate_id:str, policy_id:str, requested_by:str, manifest_id:str), Output -> ComplianceRun +# @PRE: repository and identifiers are valid and resolvable by orchestrator dependencies. +# @POST: Returns finalized ComplianceRun produced by orchestrator start->execute->finalize sequence. +# @SIDE_EFFECT: Reads/writes compliance entities through repository during orchestrator calls. +# @DATA_CONTRACT: Input -> (repository:CleanReleaseRepository, candidate_id:str, policy_id:str, requested_by:str, manifest_id:str), Output -> ComplianceRun def run_check_legacy( repository: CleanReleaseRepository, candidate_id: str, diff --git a/backend/src/services/clean_release/demo_data_service.py b/backend/src/services/clean_release/demo_data_service.py index dcf90201..cd209706 100644 --- a/backend/src/services/clean_release/demo_data_service.py +++ b/backend/src/services/clean_release/demo_data_service.py @@ -1,8 +1,8 @@ # #region DemoDataService [C:3] [TYPE Module] [SEMANTICS clean-release, demo-mode, namespace, isolation, repository] # @BRIEF Provide deterministic namespace helpers and isolated in-memory repository creation for demo and real modes. -# @LAYER Domain -# @INVARIANT Demo and real namespaces must never collide for generated physical identifiers. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] +# @INVARIANT: Demo and real namespaces must never collide for generated physical identifiers. from __future__ import annotations @@ -11,8 +11,8 @@ from .repository import CleanReleaseRepository # #region resolve_namespace [TYPE Function] # @BRIEF Resolve canonical clean-release namespace for requested mode. -# @PRE mode is a non-empty string identifying runtime mode. -# @POST Returns deterministic namespace key for demo/real separation. +# @PRE: mode is a non-empty string identifying runtime mode. +# @POST: Returns deterministic namespace key for demo/real separation. def resolve_namespace(mode: str) -> str: normalized = (mode or "").strip().lower() if normalized == "demo": @@ -25,8 +25,8 @@ def resolve_namespace(mode: str) -> str: # #region build_namespaced_id [TYPE Function] # @BRIEF Build storage-safe physical identifier under mode namespace. -# @PRE namespace and logical_id are non-empty strings. -# @POST Returns deterministic "{namespace}::{logical_id}" identifier. +# @PRE: namespace and logical_id are non-empty strings. +# @POST: Returns deterministic "{namespace}::{logical_id}" identifier. def build_namespaced_id(namespace: str, logical_id: str) -> str: if not namespace or not namespace.strip(): raise ValueError("namespace must be non-empty") @@ -40,8 +40,8 @@ def build_namespaced_id(namespace: str, logical_id: str) -> str: # #region create_isolated_repository [TYPE Function] # @BRIEF Create isolated in-memory repository instance for selected mode namespace. -# @PRE mode is a valid runtime mode marker. -# @POST Returns repository instance tagged with namespace metadata. +# @PRE: mode is a valid runtime mode marker. +# @POST: Returns repository instance tagged with namespace metadata. def create_isolated_repository(mode: str) -> CleanReleaseRepository: namespace = resolve_namespace(mode) repository = CleanReleaseRepository() diff --git a/backend/src/services/clean_release/dto.py b/backend/src/services/clean_release/dto.py index 7cd095d1..2501746f 100644 --- a/backend/src/services/clean_release/dto.py +++ b/backend/src/services/clean_release/dto.py @@ -1,7 +1,7 @@ # #region clean_release_dto [C:3] [TYPE Module] # @BRIEF Data Transfer Objects for clean release compliance subsystem. -# @LAYER Application -# @RELATION DEPENDS_ON -> [pydantic] +# @LAYER: Application +# @RELATION DEPENDS_ON -> pydantic from datetime import datetime from typing import List, Optional, Dict, Any @@ -82,4 +82,4 @@ class CandidateOverviewDTO(BaseModel): latest_publication_id: Optional[str] = None latest_publication_status: Optional[str] = None -# #endregion clean_release_dto +# #endregion clean_release_dto \ No newline at end of file diff --git a/backend/src/services/clean_release/enums.py b/backend/src/services/clean_release/enums.py index bc1607d4..22313487 100644 --- a/backend/src/services/clean_release/enums.py +++ b/backend/src/services/clean_release/enums.py @@ -1,7 +1,7 @@ # #region clean_release_enums [C:3] [TYPE Module] # @BRIEF Canonical enums for clean release lifecycle and compliance. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [enum] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> enum from enum import Enum @@ -69,4 +69,4 @@ class ViolationCategory(str, Enum): MANIFEST_CONSISTENCY = "MANIFEST_CONSISTENCY" EXTERNAL_ENDPOINT = "EXTERNAL_ENDPOINT" -# #endregion clean_release_enums +# #endregion clean_release_enums \ No newline at end of file diff --git a/backend/src/services/clean_release/exceptions.py b/backend/src/services/clean_release/exceptions.py index 30d7a2f5..47a7952b 100644 --- a/backend/src/services/clean_release/exceptions.py +++ b/backend/src/services/clean_release/exceptions.py @@ -1,7 +1,7 @@ # #region clean_release_exceptions [C:3] [TYPE Module] # @BRIEF Domain exceptions for clean release compliance subsystem. -# @LAYER Domain -# @RELATION DEPENDS_ON -> [Exception] +# @LAYER: Domain +# @RELATION DEPENDS_ON -> Exception class CleanReleaseError(Exception): """Base exception for clean release subsystem.""" @@ -35,4 +35,4 @@ class PublicationGateError(CleanReleaseError): """Raised when publication requirements are not met.""" pass -# #endregion clean_release_exceptions +# #endregion clean_release_exceptions \ No newline at end of file diff --git a/backend/src/services/clean_release/facade.py b/backend/src/services/clean_release/facade.py index 202aa9dc..1c9ee501 100644 --- a/backend/src/services/clean_release/facade.py +++ b/backend/src/services/clean_release/facade.py @@ -1,7 +1,7 @@ # #region clean_release_facade [C:3] [TYPE Module] # @BRIEF Unified entry point for clean release operations. -# @LAYER Application -# @RELATION DEPENDS_ON -> [ComplianceOrchestrator] +# @LAYER: Application +# @RELATION DEPENDS_ON -> ComplianceOrchestrator from typing import List, Optional from src.services.clean_release.repositories import ( @@ -119,4 +119,4 @@ class CleanReleaseFacade: candidates = self.candidate_repo.list_all() return [self.get_candidate_overview(c.id) for c in candidates] -# #endregion clean_release_facade +# #endregion clean_release_facade \ No newline at end of file diff --git a/backend/src/services/clean_release/manifest_builder.py b/backend/src/services/clean_release/manifest_builder.py index 918bb2f7..9d457f2a 100644 --- a/backend/src/services/clean_release/manifest_builder.py +++ b/backend/src/services/clean_release/manifest_builder.py @@ -1,8 +1,8 @@ # #region ManifestBuilder [C:3] [TYPE Module] [SEMANTICS clean-release, manifest, deterministic-hash, summary] # @BRIEF Build deterministic distribution manifest from classified artifact input. -# @LAYER Domain -# @INVARIANT Equal semantic artifact sets produce identical deterministic hash values. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @INVARIANT: Equal semantic artifact sets produce identical deterministic hash values. from __future__ import annotations @@ -54,8 +54,8 @@ def _stable_hash_payload( # #region build_distribution_manifest [TYPE Function] # @BRIEF Build DistributionManifest with deterministic hash and validated counters. -# @PRE artifacts list contains normalized classification values. -# @POST Returns DistributionManifest with summary counts matching items cardinality. +# @PRE: artifacts list contains normalized classification values. +# @POST: Returns DistributionManifest with summary counts matching items cardinality. def build_distribution_manifest( manifest_id: str, candidate_id: str, @@ -111,8 +111,8 @@ def build_distribution_manifest( # #region build_manifest [TYPE Function] # @BRIEF Legacy compatibility wrapper for old manifest builder import paths. -# @PRE Same as build_distribution_manifest. -# @POST Returns DistributionManifest produced by canonical builder. +# @PRE: Same as build_distribution_manifest. +# @POST: Returns DistributionManifest produced by canonical builder. def build_manifest( manifest_id: str, candidate_id: str, diff --git a/backend/src/services/clean_release/manifest_service.py b/backend/src/services/clean_release/manifest_service.py index 42228ec0..13945e32 100644 --- a/backend/src/services/clean_release/manifest_service.py +++ b/backend/src/services/clean_release/manifest_service.py @@ -1,14 +1,14 @@ # #region ManifestService [C:5] [TYPE Module] [SEMANTICS clean-release, manifest, versioning, immutability, lifecycle] # @BRIEF Build immutable distribution manifests with deterministic digest and version increment. -# @LAYER Domain -# @PRE Candidate exists and is PREPARED or MANIFEST_BUILT; artifacts are present. -# @POST New immutable manifest is persisted with incremented version and deterministic digest. -# @SIDE_EFFECT May modify manifest state during processing -# @DATA_CONTRACT Manifest -> ManifestRecord; Candidate -> ManifestRecord -# @INVARIANT Existing manifests are never mutated. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [RepositoryRelations] # @RELATION DEPENDS_ON -> [ManifestBuilder] # @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @PRE: Candidate exists and is PREPARED or MANIFEST_BUILT; artifacts are present. +# @POST: New immutable manifest is persisted with incremented version and deterministic digest. +# @INVARIANT: Existing manifests are never mutated. +# @SIDE_EFFECT: May modify manifest state during processing +# @DATA_CONTRACT: Manifest -> ManifestRecord; Candidate -> ManifestRecord from __future__ import annotations @@ -22,8 +22,8 @@ from .repository import CleanReleaseRepository # #region build_manifest_snapshot [TYPE Function] # @BRIEF Create a new immutable manifest version for a candidate. -# @PRE Candidate is prepared, artifacts are available, candidate_id is valid. -# @POST Returns persisted DistributionManifest with monotonically incremented version. +# @PRE: Candidate is prepared, artifacts are available, candidate_id is valid. +# @POST: Returns persisted DistributionManifest with monotonically incremented version. def build_manifest_snapshot( repository: CleanReleaseRepository, candidate_id: str, diff --git a/backend/src/services/clean_release/mappers.py b/backend/src/services/clean_release/mappers.py index e1f72685..7310c28e 100644 --- a/backend/src/services/clean_release/mappers.py +++ b/backend/src/services/clean_release/mappers.py @@ -1,7 +1,7 @@ # #region clean_release_mappers [C:3] [TYPE Module] # @BRIEF Map between domain entities (SQLAlchemy models) and DTOs. -# @LAYER Application -# @RELATION DEPENDS_ON -> [clean_release_dto] +# @LAYER: Application +# @RELATION DEPENDS_ON -> clean_release_dto from typing import List from src.models.clean_release import ( @@ -64,4 +64,4 @@ def map_report_to_dto(report: ComplianceReport) -> ReportDTO: generated_at=report.generated_at ) -# #endregion clean_release_mappers +# #endregion clean_release_mappers \ No newline at end of file diff --git a/backend/src/services/clean_release/policy_engine.py b/backend/src/services/clean_release/policy_engine.py index ff84166c..51135403 100644 --- a/backend/src/services/clean_release/policy_engine.py +++ b/backend/src/services/clean_release/policy_engine.py @@ -1,13 +1,13 @@ # #region PolicyEngine [C:5] [TYPE Module] [SEMANTICS clean-release, policy, classification, source-isolation] # @BRIEF Evaluate artifact/source policies for enterprise clean profile with deterministic outcomes. -# @LAYER Domain -# @PRE PolicyRepository is accessible -# @POST PolicyDecision returned with approval status -# @SIDE_EFFECT Read-only policy evaluation; no state changes -# @DATA_CONTRACT Candidate -> PolicyDecision -# @INVARIANT Enterprise-clean policy always treats non-registry sources as violations. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] # @RELATION DEPENDS_ON -> [LoggerModule] +# @INVARIANT: Enterprise-clean policy always treats non-registry sources as violations. +# @DATA_CONTRACT: Candidate -> PolicyDecision +# @PRE: PolicyRepository is accessible +# @POST: PolicyDecision returned with approval status +# @SIDE_EFFECT: Read-only policy evaluation; no state changes from __future__ import annotations @@ -36,8 +36,8 @@ class SourceValidationResult: # #region CleanPolicyEngine [TYPE Class] -# @PRE Active policy exists and is internally consistent. -# @POST Deterministic classification and source validation are available. +# @PRE: Active policy exists and is internally consistent. +# @POST: Deterministic classification and source validation are available. # @TEST_CONTRACT: CandidateEvaluationInput -> PolicyValidationResult|SourceValidationResult # @TEST_SCENARIO: policy_valid -> Enterprise clean policy with matching registry returns ok=True # @TEST_FIXTURE: policy_enterprise_clean -> file:backend/tests/fixtures/clean_release/fixtures_clean_release.json diff --git a/backend/src/services/clean_release/policy_resolution_service.py b/backend/src/services/clean_release/policy_resolution_service.py index 753f4602..c667d990 100644 --- a/backend/src/services/clean_release/policy_resolution_service.py +++ b/backend/src/services/clean_release/policy_resolution_service.py @@ -1,14 +1,14 @@ # #region PolicyResolutionService [C:5] [TYPE Module] [SEMANTICS clean-release, policy, registry, trusted-resolution, immutable-snapshots] # @BRIEF Resolve trusted policy and registry snapshots from ConfigManager without runtime overrides. -# @LAYER Domain -# @PRE PolicyRepository and Manifest are available -# @POST ResolutionResult with matched policies -# @SIDE_EFFECT Read-only policy evaluation; logs resolution decisions -# @DATA_CONTRACT PolicyRequest -> ResolutionResult -# @INVARIANT Trusted snapshot resolution is based only on ConfigManager active identifiers. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [RepositoryRelations] # @RELATION DEPENDS_ON -> [clean_release_exceptions] +# @INVARIANT: Trusted snapshot resolution is based only on ConfigManager active identifiers. +# @DATA_CONTRACT: PolicyRequest -> ResolutionResult +# @PRE: PolicyRepository and Manifest are available +# @POST: ResolutionResult with matched policies +# @SIDE_EFFECT: Read-only policy evaluation; logs resolution decisions from __future__ import annotations @@ -21,9 +21,9 @@ from .repository import CleanReleaseRepository # #region resolve_trusted_policy_snapshots [TYPE Function] # @BRIEF Resolve immutable trusted policy and registry snapshots using active config IDs only. -# @PRE ConfigManager provides active_policy_id and active_registry_id; repository contains referenced snapshots. -# @POST Returns immutable policy and registry snapshots; runtime override attempts are rejected. -# @SIDE_EFFECT None. +# @PRE: ConfigManager provides active_policy_id and active_registry_id; repository contains referenced snapshots. +# @POST: Returns immutable policy and registry snapshots; runtime override attempts are rejected. +# @SIDE_EFFECT: None. def resolve_trusted_policy_snapshots( *, config_manager, diff --git a/backend/src/services/clean_release/preparation_service.py b/backend/src/services/clean_release/preparation_service.py index 3e54da36..be31d18c 100644 --- a/backend/src/services/clean_release/preparation_service.py +++ b/backend/src/services/clean_release/preparation_service.py @@ -1,10 +1,10 @@ # #region PreparationService [C:3] [TYPE Module] [SEMANTICS clean-release, preparation, manifest, policy-evaluation] # @BRIEF Prepare release candidate by policy evaluation and deterministic manifest creation. -# @LAYER Domain -# @INVARIANT Candidate preparation always persists manifest and candidate status deterministically. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [PolicyEngine] # @RELATION DEPENDS_ON -> [ManifestBuilder] # @RELATION DEPENDS_ON -> [RepositoryRelations] +# @INVARIANT: Candidate preparation always persists manifest and candidate status deterministically. from __future__ import annotations @@ -89,8 +89,8 @@ def prepare_candidate( # #region prepare_candidate_legacy [TYPE Function] # @BRIEF Legacy compatibility wrapper kept for migration period. -# @PRE Same as prepare_candidate. -# @POST Delegates to canonical prepare_candidate and preserves response shape. +# @PRE: Same as prepare_candidate. +# @POST: Delegates to canonical prepare_candidate and preserves response shape. def prepare_candidate_legacy( repository: CleanReleaseRepository, candidate_id: str, diff --git a/backend/src/services/clean_release/publication_service.py b/backend/src/services/clean_release/publication_service.py index f151a7c2..43f8dcad 100644 --- a/backend/src/services/clean_release/publication_service.py +++ b/backend/src/services/clean_release/publication_service.py @@ -1,12 +1,10 @@ -# [DEF:PublicationService:Module] -# @COMPLEXITY: 5 -# @SEMANTICS: clean-release, publication, revoke, gate, lifecycle -# @PURPOSE: Enforce publication and revocation gates with append-only publication records. +# #region PublicationService [C:5] [TYPE Module] [SEMANTICS clean-release, publication, revoke, gate, lifecycle] +# @BRIEF Enforce publication and revocation gates with append-only publication records. # @LAYER: Domain -# @RELATION: [DEPENDS_ON] ->[RepositoryRelations] -# @RELATION: [DEPENDS_ON] ->[ApprovalService] -# @RELATION: [DEPENDS_ON] ->[CleanReleaseModels] -# @RELATION: [DEPENDS_ON] ->[AuditService] +# @RELATION DEPENDS_ON -> [RepositoryRelations] +# @RELATION DEPENDS_ON -> [ApprovalService] +# @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @RELATION DEPENDS_ON -> [AuditService] # @INVARIANT: Publication records are append-only snapshots; revoke mutates only publication status for targeted record. from __future__ import annotations @@ -25,8 +23,8 @@ from .repository import CleanReleaseRepository # #region _get_or_init_publications_store [TYPE Function] # @BRIEF Provide in-memory append-only publication storage. -# @PRE repository is initialized. -# @POST Returns publication list attached to repository. +# @PRE: repository is initialized. +# @POST: Returns publication list attached to repository. def _get_or_init_publications_store( repository: CleanReleaseRepository, ) -> List[PublicationRecord]: @@ -42,8 +40,8 @@ def _get_or_init_publications_store( # #region _latest_publication_for_candidate [TYPE Function] # @BRIEF Resolve latest publication record for candidate. -# @PRE candidate_id is non-empty. -# @POST Returns latest record or None. +# @PRE: candidate_id is non-empty. +# @POST: Returns latest record or None. def _latest_publication_for_candidate( repository: CleanReleaseRepository, candidate_id: str, @@ -67,8 +65,8 @@ def _latest_publication_for_candidate( # #region _latest_approval_for_candidate [TYPE Function] # @BRIEF Resolve latest approval decision from repository decision store. -# @PRE candidate_id is non-empty. -# @POST Returns latest decision object or None. +# @PRE: candidate_id is non-empty. +# @POST: Returns latest decision object or None. def _latest_approval_for_candidate( repository: CleanReleaseRepository, candidate_id: str ): @@ -88,8 +86,8 @@ def _latest_approval_for_candidate( # #region publish_candidate [TYPE Function] # @BRIEF Create immutable publication record for approved candidate. -# @PRE Candidate exists, report belongs to candidate, latest approval is APPROVED. -# @POST New ACTIVE publication record is appended. +# @PRE: Candidate exists, report belongs to candidate, latest approval is APPROVED. +# @POST: New ACTIVE publication record is appended. def publish_candidate( *, repository: CleanReleaseRepository, @@ -168,8 +166,8 @@ def publish_candidate( # #region revoke_publication [TYPE Function] # @BRIEF Revoke existing publication record without deleting history. -# @PRE publication_id exists in repository publication store. -# @POST Target publication status becomes REVOKED and updated record is returned. +# @PRE: publication_id exists in repository publication store. +# @POST: Target publication status becomes REVOKED and updated record is returned. def revoke_publication( *, repository: CleanReleaseRepository, @@ -212,4 +210,4 @@ def revoke_publication( # #endregion revoke_publication -# [/DEF:backend.src.services.clean_release.publication_service:Module] +# #endregion backend.src.services.clean_release.publication_service diff --git a/backend/src/services/clean_release/report_builder.py b/backend/src/services/clean_release/report_builder.py index 7e09bd30..311e19c5 100644 --- a/backend/src/services/clean_release/report_builder.py +++ b/backend/src/services/clean_release/report_builder.py @@ -1,9 +1,9 @@ # #region ReportBuilder [C:5] [TYPE Module] [SEMANTICS clean-release, report, audit, counters, violations] # @BRIEF Build and persist compliance reports with consistent counter invariants. -# @LAYER Domain -# @INVARIANT blocking_violations_count never exceeds violations_count. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] # @RELATION DEPENDS_ON -> [RepositoryRelations] +# @INVARIANT: blocking_violations_count never exceeds violations_count. # @TEST_CONTRACT: ComplianceCheckRun,List[ComplianceViolation] -> ComplianceReport # @TEST_FIXTURE: blocked_with_two_violations -> file:backend/tests/fixtures/clean_release/fixtures_clean_release.json # @TEST_EDGE: empty_violations_for_blocked -> BLOCKED run with zero blocking violations raises ValueError diff --git a/backend/src/services/clean_release/repositories/__init__.py b/backend/src/services/clean_release/repositories/__init__.py index c79f11d8..b77e9166 100644 --- a/backend/src/services/clean_release/repositories/__init__.py +++ b/backend/src/services/clean_release/repositories/__init__.py @@ -1,6 +1,6 @@ # #region clean_release_repositories [C:3] [TYPE Module] # @BRIEF Export all clean release repositories. -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @RELATION DEPENDS_ON -> sqlalchemy from .candidate_repository import CandidateRepository from .artifact_repository import ArtifactRepository @@ -25,4 +25,4 @@ __all__ = [ "CleanReleaseAuditLog" ] -# #endregion clean_release_repositories +# #endregion clean_release_repositories \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/approval_repository.py b/backend/src/services/clean_release/repositories/approval_repository.py index d44f016a..c43034f2 100644 --- a/backend/src/services/clean_release/repositories/approval_repository.py +++ b/backend/src/services/clean_release/repositories/approval_repository.py @@ -1,7 +1,7 @@ # #region approval_repository [C:3] [TYPE Module] # @BRIEF Persist and query approval decisions. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -50,4 +50,4 @@ class ApprovalRepository: with belief_scope("ApprovalRepository.list_by_candidate"): return self.db.query(ApprovalDecision).filter(ApprovalDecision.candidate_id == candidate_id).all() -# #endregion approval_repository +# #endregion approval_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/artifact_repository.py b/backend/src/services/clean_release/repositories/artifact_repository.py index 1e371855..fb8049fc 100644 --- a/backend/src/services/clean_release/repositories/artifact_repository.py +++ b/backend/src/services/clean_release/repositories/artifact_repository.py @@ -1,7 +1,7 @@ # #region artifact_repository [C:3] [TYPE Module] # @BRIEF Persist and query candidate artifacts. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -51,4 +51,4 @@ class ArtifactRepository: with belief_scope("ArtifactRepository.list_by_candidate"): return self.db.query(CandidateArtifact).filter(CandidateArtifact.candidate_id == candidate_id).all() -# #endregion artifact_repository +# #endregion artifact_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/audit_repository.py b/backend/src/services/clean_release/repositories/audit_repository.py index 8113a25b..928d353d 100644 --- a/backend/src/services/clean_release/repositories/audit_repository.py +++ b/backend/src/services/clean_release/repositories/audit_repository.py @@ -1,7 +1,7 @@ # #region audit_repository [C:3] [TYPE Module] # @BRIEF Persist and query audit logs for clean release operations. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -43,4 +43,4 @@ class AuditRepository: with belief_scope("AuditRepository.list_by_candidate"): return self.db.query(CleanReleaseAuditLog).filter(CleanReleaseAuditLog.candidate_id == candidate_id).all() -# #endregion audit_repository +# #endregion audit_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/candidate_repository.py b/backend/src/services/clean_release/repositories/candidate_repository.py index 1689f772..26bbf182 100644 --- a/backend/src/services/clean_release/repositories/candidate_repository.py +++ b/backend/src/services/clean_release/repositories/candidate_repository.py @@ -1,7 +1,7 @@ # #region candidate_repository [C:3] [TYPE Module] # @BRIEF Persist and query release candidates. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -44,4 +44,4 @@ class CandidateRepository: with belief_scope("CandidateRepository.list_all"): return self.db.query(ReleaseCandidate).all() -# #endregion candidate_repository +# #endregion candidate_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/compliance_repository.py b/backend/src/services/clean_release/repositories/compliance_repository.py index e9ad9444..e9e6c27c 100644 --- a/backend/src/services/clean_release/repositories/compliance_repository.py +++ b/backend/src/services/clean_release/repositories/compliance_repository.py @@ -1,7 +1,7 @@ # #region compliance_repository [C:3] [TYPE Module] # @BRIEF Persist and query compliance runs, stage runs, and violations. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -84,4 +84,4 @@ class ComplianceRepository: with belief_scope("ComplianceRepository.list_violations_by_run"): return self.db.query(ComplianceViolation).filter(ComplianceViolation.run_id == run_id).all() -# #endregion compliance_repository +# #endregion compliance_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/manifest_repository.py b/backend/src/services/clean_release/repositories/manifest_repository.py index 2b8145ae..5304bb18 100644 --- a/backend/src/services/clean_release/repositories/manifest_repository.py +++ b/backend/src/services/clean_release/repositories/manifest_repository.py @@ -1,9 +1,9 @@ # #region ManifestRepositoryModule [C:3] [TYPE Module] # @BRIEF Persist and query distribution manifests. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [DistributionManifest] -# @RELATION DEPENDS_ON -> [sqlalchemy] -# @RELATION DEPENDS_ON -> [belief_scope] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> DistributionManifest +# @RELATION DEPENDS_ON -> sqlalchemy +# @RELATION DEPENDS_ON -> belief_scope from typing import Optional, List from sqlalchemy.orm import Session @@ -13,50 +13,54 @@ from src.core.logger import belief_scope # #region ManifestRepository [C:3] [TYPE Class] # @BRIEF Encapsulates database CRUD operations for DistributionManifest entities. -# @RELATION DEPENDS_ON -> [DistributionManifest] -# @RELATION DEPENDS_ON -> [sqlalchemy.Session] +# @RELATION DEPENDS_ON -> DistributionManifest +# @RELATION DEPENDS_ON -> sqlalchemy.Session class ManifestRepository: """Repository for distribution manifest persistence.""" - # #region ManifestRepository.__init__ [C:1] [TYPE Function] - # @BRIEF Initialize repository with an active SQLAlchemy session. - # @PRE db is a valid SQLAlchemy Session instance. - # @POST Repository is ready for database operations. + # [DEF:ManifestRepository.__init__:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Initialize repository with an active SQLAlchemy session. + # @PRE: db is a valid SQLAlchemy Session instance. + # @POST: Repository is ready for database operations. def __init__(self, db: Session): self.db = db - # #endregion ManifestRepository.__init__ + # [/DEF:ManifestRepository.__init__:Function] - # #region ManifestRepository.save [C:3] [TYPE Function] - # @BRIEF Persist a DistributionManifest to the database. - # @PRE manifest is a valid DistributionManifest instance with required fields populated. - # @POST Manifest is committed to database and refreshed with generated ID. - # @SIDE_EFFECT Database commit via session.commit(). - # @RELATION DEPENDS_ON -> [DistributionManifest] + # [DEF:ManifestRepository.save:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persist a DistributionManifest to the database. + # @PRE: manifest is a valid DistributionManifest instance with required fields populated. + # @POST: Manifest is committed to database and refreshed with generated ID. + # @SIDE_EFFECT: Database commit via session.commit(). + # @RELATION: DEPENDS_ON -> DistributionManifest def save(self, manifest: DistributionManifest) -> DistributionManifest: with belief_scope("ManifestRepository.save"): self.db.add(manifest) self.db.commit() self.db.refresh(manifest) return manifest - # #endregion ManifestRepository.save + # [/DEF:ManifestRepository.save:Function] - # #region ManifestRepository.get_by_id [C:2] [TYPE Function] - # @BRIEF Retrieve a single DistributionManifest by its primary key. - # @PRE manifest_id is a valid string identifier. - # @POST Returns DistributionManifest if found, None otherwise. - # @RELATION DEPENDS_ON -> [DistributionManifest] + # [DEF:ManifestRepository.get_by_id:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Retrieve a single DistributionManifest by its primary key. + # @PRE: manifest_id is a valid string identifier. + # @POST: Returns DistributionManifest if found, None otherwise. + # @RELATION: DEPENDS_ON -> DistributionManifest def get_by_id(self, manifest_id: str) -> Optional[DistributionManifest]: with belief_scope("ManifestRepository.get_by_id"): return self.db.query(DistributionManifest).filter( DistributionManifest.id == manifest_id ).first() - # #endregion ManifestRepository.get_by_id + # [/DEF:ManifestRepository.get_by_id:Function] - # #region ManifestRepository.get_latest_for_candidate [C:3] [TYPE Function] - # @BRIEF Retrieve the most recent manifest version for a given candidate. - # @PRE candidate_id is a valid string identifier. - # @POST Returns the highest manifest_version manifest for the candidate, or None. - # @RELATION DEPENDS_ON -> [DistributionManifest] + # [DEF:ManifestRepository.get_latest_for_candidate:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Retrieve the most recent manifest version for a given candidate. + # @PRE: candidate_id is a valid string identifier. + # @POST: Returns the highest manifest_version manifest for the candidate, or None. + # @RELATION: DEPENDS_ON -> DistributionManifest def get_latest_for_candidate(self, candidate_id: str) -> Optional[DistributionManifest]: with belief_scope("ManifestRepository.get_latest_for_candidate"): return ( @@ -65,13 +69,14 @@ class ManifestRepository: .order_by(DistributionManifest.manifest_version.desc()) .first() ) - # #endregion ManifestRepository.get_latest_for_candidate + # [/DEF:ManifestRepository.get_latest_for_candidate:Function] - # #region ManifestRepository.list_by_candidate [C:2] [TYPE Function] - # @BRIEF List all manifests for a specific candidate, ordered by version. - # @PRE candidate_id is a valid string identifier. - # @POST Returns a list of DistributionManifest instances (may be empty). - # @RELATION DEPENDS_ON -> [DistributionManifest] + # [DEF:ManifestRepository.list_by_candidate:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List all manifests for a specific candidate, ordered by version. + # @PRE: candidate_id is a valid string identifier. + # @POST: Returns a list of DistributionManifest instances (may be empty). + # @RELATION: DEPENDS_ON -> DistributionManifest def list_by_candidate(self, candidate_id: str) -> List[DistributionManifest]: with belief_scope("ManifestRepository.list_by_candidate"): return ( @@ -79,6 +84,8 @@ class ManifestRepository: .filter(DistributionManifest.candidate_id == candidate_id) .all() ) - # #endregion ManifestRepository.list_by_candidate + # [/DEF:ManifestRepository.list_by_candidate:Function] # #endregion ManifestRepository + +# #endregion ManifestRepositoryModule \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/policy_repository.py b/backend/src/services/clean_release/repositories/policy_repository.py index 2faadc71..2696bb8a 100644 --- a/backend/src/services/clean_release/repositories/policy_repository.py +++ b/backend/src/services/clean_release/repositories/policy_repository.py @@ -1,7 +1,7 @@ # #region policy_repository [C:3] [TYPE Module] # @BRIEF Persist and query policy and registry snapshots. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -49,4 +49,4 @@ class PolicyRepository: with belief_scope("PolicyRepository.get_registry_snapshot"): return self.db.query(SourceRegistrySnapshot).filter(SourceRegistrySnapshot.id == snapshot_id).first() -# #endregion policy_repository +# #endregion policy_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/publication_repository.py b/backend/src/services/clean_release/repositories/publication_repository.py index da98a4d9..af7a3330 100644 --- a/backend/src/services/clean_release/repositories/publication_repository.py +++ b/backend/src/services/clean_release/repositories/publication_repository.py @@ -1,7 +1,7 @@ # #region publication_repository [C:3] [TYPE Module] # @BRIEF Persist and query publication records. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -50,4 +50,4 @@ class PublicationRepository: with belief_scope("PublicationRepository.list_by_candidate"): return self.db.query(PublicationRecord).filter(PublicationRecord.candidate_id == candidate_id).all() -# #endregion publication_repository +# #endregion publication_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repositories/report_repository.py b/backend/src/services/clean_release/repositories/report_repository.py index b41f3ed9..9ce6e433 100644 --- a/backend/src/services/clean_release/repositories/report_repository.py +++ b/backend/src/services/clean_release/repositories/report_repository.py @@ -1,7 +1,7 @@ # #region report_repository [C:3] [TYPE Module] # @BRIEF Persist and query compliance reports. -# @LAYER Infra -# @RELATION DEPENDS_ON -> [sqlalchemy] +# @LAYER: Infra +# @RELATION DEPENDS_ON -> sqlalchemy from typing import Optional, List from sqlalchemy.orm import Session @@ -47,4 +47,4 @@ class ReportRepository: with belief_scope("ReportRepository.list_by_candidate"): return self.db.query(ComplianceReport).filter(ComplianceReport.candidate_id == candidate_id).all() -# #endregion report_repository +# #endregion report_repository \ No newline at end of file diff --git a/backend/src/services/clean_release/repository.py b/backend/src/services/clean_release/repository.py index b256bb5f..cb2bd98e 100644 --- a/backend/src/services/clean_release/repository.py +++ b/backend/src/services/clean_release/repository.py @@ -1,8 +1,8 @@ # #region RepositoryRelations [C:3] [TYPE Module] [SEMANTICS clean-release, repository, persistence, in-memory] # @BRIEF Provide repository adapter for clean release entities with deterministic access methods. -# @LAYER Infra -# @INVARIANT Repository operations are side-effect free outside explicit save/update calls. +# @LAYER: Infra # @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @INVARIANT: Repository operations are side-effect free outside explicit save/update calls. from __future__ import annotations diff --git a/backend/src/services/clean_release/source_isolation.py b/backend/src/services/clean_release/source_isolation.py index 5787ddcb..c655ab47 100644 --- a/backend/src/services/clean_release/source_isolation.py +++ b/backend/src/services/clean_release/source_isolation.py @@ -1,8 +1,8 @@ # #region SourceIsolation [C:3] [TYPE Module] [SEMANTICS clean-release, source-isolation, internal-only, validation] # @BRIEF Validate that all resource endpoints belong to the approved internal source registry. -# @LAYER Domain -# @INVARIANT Any endpoint outside enabled registry entries is treated as external-source violation. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] +# @INVARIANT: Any endpoint outside enabled registry entries is treated as external-source violation. from __future__ import annotations diff --git a/backend/src/services/clean_release/stages/__init__.py b/backend/src/services/clean_release/stages/__init__.py index 0e111954..8c82a6d5 100644 --- a/backend/src/services/clean_release/stages/__init__.py +++ b/backend/src/services/clean_release/stages/__init__.py @@ -1,9 +1,9 @@ # #region ComplianceStages [C:3] [TYPE Module] [SEMANTICS clean-release, compliance, stages, state-machine] # @BRIEF Define compliance stage order and helper functions for deterministic run-state evaluation. -# @LAYER Domain -# @INVARIANT Stage order remains deterministic for all compliance runs. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] # @RELATION DEPENDS_ON -> [ComplianceStageBase] +# @INVARIANT: Stage order remains deterministic for all compliance runs. from __future__ import annotations @@ -32,8 +32,8 @@ MANDATORY_STAGE_ORDER: List[ComplianceStageName] = [ # #region build_default_stages [TYPE Function] # @BRIEF Build default deterministic stage pipeline implementation order. -# @PRE None. -# @POST Returns stage instances in mandatory execution order. +# @PRE: None. +# @POST: Returns stage instances in mandatory execution order. def build_default_stages() -> List[ComplianceStage]: return [ DataPurityStage(), @@ -48,8 +48,8 @@ def build_default_stages() -> List[ComplianceStage]: # #region stage_result_map [TYPE Function] # @BRIEF Convert stage result list to dictionary by stage name. -# @PRE stage_results may be empty or contain unique stage names. -# @POST Returns stage->status dictionary for downstream evaluation. +# @PRE: stage_results may be empty or contain unique stage names. +# @POST: Returns stage->status dictionary for downstream evaluation. def stage_result_map( stage_results: Iterable[ComplianceStageRun | CheckStageResult], ) -> Dict[ComplianceStageName, CheckStageStatus]: @@ -87,8 +87,8 @@ def stage_result_map( # #region missing_mandatory_stages [TYPE Function] # @BRIEF Identify mandatory stages that are absent from run results. -# @PRE stage_status_map contains zero or more known stage statuses. -# @POST Returns ordered list of missing mandatory stages. +# @PRE: stage_status_map contains zero or more known stage statuses. +# @POST: Returns ordered list of missing mandatory stages. def missing_mandatory_stages( stage_status_map: Dict[ComplianceStageName, CheckStageStatus], ) -> List[ComplianceStageName]: @@ -100,8 +100,8 @@ def missing_mandatory_stages( # #region derive_final_status [TYPE Function] # @BRIEF Derive final run status from stage results with deterministic blocking behavior. -# @PRE Stage statuses correspond to compliance checks. -# @POST Returns one of PASSED/BLOCKED/ERROR according to mandatory stage outcomes. +# @PRE: Stage statuses correspond to compliance checks. +# @POST: Returns one of PASSED/BLOCKED/ERROR according to mandatory stage outcomes. def derive_final_status( stage_results: Iterable[ComplianceStageRun | CheckStageResult], ) -> CheckFinalStatus: diff --git a/backend/src/services/clean_release/stages/base.py b/backend/src/services/clean_release/stages/base.py index 2cc97ad3..610f12f8 100644 --- a/backend/src/services/clean_release/stages/base.py +++ b/backend/src/services/clean_release/stages/base.py @@ -1,9 +1,9 @@ # #region ComplianceStageBase [C:3] [TYPE Module] [SEMANTICS clean-release, compliance, stages, contracts, base] # @BRIEF Define shared contracts and helpers for pluggable clean-release compliance stages. -# @LAYER Domain -# @INVARIANT Stage execution is deterministic for equal input context. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [CleanReleaseModels] # @RELATION DEPENDS_ON -> [LoggerModule] +# @INVARIANT: Stage execution is deterministic for equal input context. from __future__ import annotations @@ -65,8 +65,8 @@ class ComplianceStage(Protocol): # #region build_stage_run_record [TYPE Function] # @BRIEF Build persisted stage run record from stage result. -# @PRE run_id and stage_name are non-empty. -# @POST Returns ComplianceStageRun with deterministic identifiers and timestamps. +# @PRE: run_id and stage_name are non-empty. +# @POST: Returns ComplianceStageRun with deterministic identifiers and timestamps. def build_stage_run_record( *, run_id: str, @@ -96,8 +96,8 @@ def build_stage_run_record( # #region build_violation [TYPE Function] # @BRIEF Construct a compliance violation with normalized defaults. -# @PRE run_id, stage_name, code and message are non-empty. -# @POST Returns immutable-style violation payload ready for persistence. +# @PRE: run_id, stage_name, code and message are non-empty. +# @POST: Returns immutable-style violation payload ready for persistence. def build_violation( *, run_id: str, diff --git a/backend/src/services/clean_release/stages/data_purity.py b/backend/src/services/clean_release/stages/data_purity.py index 4939de7f..4e951833 100644 --- a/backend/src/services/clean_release/stages/data_purity.py +++ b/backend/src/services/clean_release/stages/data_purity.py @@ -1,9 +1,9 @@ # #region data_purity [C:3] [TYPE Module] [SEMANTICS clean-release, compliance-stage, data-purity] # @BRIEF Evaluate manifest purity counters and emit blocking violations for prohibited artifacts. -# @LAYER Domain -# @INVARIANT prohibited_detected_count > 0 always yields BLOCKED stage decision. +# @LAYER: Domain # @RELATION IMPLEMENTS -> [ComplianceStage] # @RELATION DEPENDS_ON -> [ComplianceStageBase] +# @INVARIANT: prohibited_detected_count > 0 always yields BLOCKED stage decision. from __future__ import annotations @@ -14,8 +14,8 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region DataPurityStage [TYPE Class] # @BRIEF Validate manifest summary for prohibited artifacts. -# @PRE context.manifest.content_json contains summary block or defaults to safe counters. -# @POST Returns PASSED when no prohibited artifacts are detected, otherwise BLOCKED with violations. +# @PRE: context.manifest.content_json contains summary block or defaults to safe counters. +# @POST: Returns PASSED when no prohibited artifacts are detected, otherwise BLOCKED with violations. class DataPurityStage: stage_name = ComplianceStageName.DATA_PURITY diff --git a/backend/src/services/clean_release/stages/internal_sources_only.py b/backend/src/services/clean_release/stages/internal_sources_only.py index 2bcf8ed6..74a3e81d 100644 --- a/backend/src/services/clean_release/stages/internal_sources_only.py +++ b/backend/src/services/clean_release/stages/internal_sources_only.py @@ -1,9 +1,9 @@ # #region internal_sources_only [C:3] [TYPE Module] [SEMANTICS clean-release, compliance-stage, source-isolation, registry] # @BRIEF Verify manifest-declared sources belong to trusted internal registry allowlist. -# @LAYER Domain -# @INVARIANT Any source host outside allowed_hosts yields BLOCKED decision with at least one violation. +# @LAYER: Domain # @RELATION IMPLEMENTS -> [ComplianceStage] # @RELATION DEPENDS_ON -> [ComplianceStageBase] +# @INVARIANT: Any source host outside allowed_hosts yields BLOCKED decision with at least one violation. from __future__ import annotations @@ -14,8 +14,8 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region InternalSourcesOnlyStage [TYPE Class] # @BRIEF Enforce internal-source-only policy from trusted registry snapshot. -# @PRE context.registry.allowed_hosts is available. -# @POST Returns PASSED when all hosts are allowed; otherwise BLOCKED and violations captured. +# @PRE: context.registry.allowed_hosts is available. +# @POST: Returns PASSED when all hosts are allowed; otherwise BLOCKED and violations captured. class InternalSourcesOnlyStage: stage_name = ComplianceStageName.INTERNAL_SOURCES_ONLY diff --git a/backend/src/services/clean_release/stages/manifest_consistency.py b/backend/src/services/clean_release/stages/manifest_consistency.py index 7b865731..b030b3b6 100644 --- a/backend/src/services/clean_release/stages/manifest_consistency.py +++ b/backend/src/services/clean_release/stages/manifest_consistency.py @@ -1,9 +1,9 @@ # #region manifest_consistency [C:3] [TYPE Module] [SEMANTICS clean-release, compliance-stage, manifest, consistency, digest] # @BRIEF Ensure run is bound to the exact manifest snapshot and digest used at run creation time. -# @LAYER Domain -# @INVARIANT Digest mismatch between run and manifest yields ERROR with blocking violation evidence. +# @LAYER: Domain # @RELATION IMPLEMENTS -> [ComplianceStage] # @RELATION DEPENDS_ON -> [ComplianceStageBase] +# @INVARIANT: Digest mismatch between run and manifest yields ERROR with blocking violation evidence. from __future__ import annotations @@ -14,8 +14,8 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region ManifestConsistencyStage [TYPE Class] # @BRIEF Validate run/manifest linkage consistency. -# @PRE context.run and context.manifest are loaded from repository for same run. -# @POST Returns PASSED when digests match, otherwise ERROR with one violation. +# @PRE: context.run and context.manifest are loaded from repository for same run. +# @POST: Returns PASSED when digests match, otherwise ERROR with one violation. class ManifestConsistencyStage: stage_name = ComplianceStageName.MANIFEST_CONSISTENCY diff --git a/backend/src/services/clean_release/stages/no_external_endpoints.py b/backend/src/services/clean_release/stages/no_external_endpoints.py index 5e308741..8116de3d 100644 --- a/backend/src/services/clean_release/stages/no_external_endpoints.py +++ b/backend/src/services/clean_release/stages/no_external_endpoints.py @@ -1,9 +1,9 @@ # #region no_external_endpoints [C:3] [TYPE Module] [SEMANTICS clean-release, compliance-stage, endpoints, network] # @BRIEF Block manifest payloads that expose external endpoints outside trusted schemes and hosts. -# @LAYER Domain -# @INVARIANT Endpoint outside allowed scheme/host always yields BLOCKED stage decision. +# @LAYER: Domain # @RELATION IMPLEMENTS -> [ComplianceStage] # @RELATION DEPENDS_ON -> [ComplianceStageBase] +# @INVARIANT: Endpoint outside allowed scheme/host always yields BLOCKED stage decision. from __future__ import annotations @@ -16,8 +16,8 @@ from .base import ComplianceStageContext, StageExecutionResult, build_violation # #region NoExternalEndpointsStage [TYPE Class] # @BRIEF Validate endpoint references from manifest against trusted registry. -# @PRE context.registry includes allowed hosts and schemes. -# @POST Returns PASSED when all endpoints are trusted, otherwise BLOCKED with endpoint violations. +# @PRE: context.registry includes allowed hosts and schemes. +# @POST: Returns PASSED when all endpoints are trusted, otherwise BLOCKED with endpoint violations. class NoExternalEndpointsStage: stage_name = ComplianceStageName.NO_EXTERNAL_ENDPOINTS diff --git a/backend/src/services/dataset_review/__init__.py b/backend/src/services/dataset_review/__init__.py index 65215a09..ace8ec0e 100644 --- a/backend/src/services/dataset_review/__init__.py +++ b/backend/src/services/dataset_review/__init__.py @@ -1,7 +1,7 @@ # #region dataset_review [TYPE Module] [SEMANTICS dataset, review, orchestration] +# # @BRIEF Provides services for dataset-centered orchestration flow. -# @LAYER Services # @RELATION EXPORTS -> [DatasetReviewOrchestrator:Class] +# @LAYER: Services # -# -# #endregion dataset_review +# #endregion dataset_review \ No newline at end of file diff --git a/backend/src/services/dataset_review/clarification_engine.py b/backend/src/services/dataset_review/clarification_engine.py index 842848d4..f819216d 100644 --- a/backend/src/services/dataset_review/clarification_engine.py +++ b/backend/src/services/dataset_review/clarification_engine.py @@ -1,19 +1,19 @@ # #region ClarificationEngine [C:4] [TYPE Module] [SEMANTICS dataset_review, clarification, question_payload, answer_persistence, readiness, findings] # @BRIEF Manage one-question-at-a-time clarification state, deterministic answer persistence, and readiness/finding updates. -# @LAYER Domain -# @PRE Target session contains a persisted clarification aggregate in the current ownership scope. -# @POST Active clarification payload exposes one highest-priority unresolved question, and each recorded answer is persisted before pointer/readiness mutation. -# @SIDE_EFFECT Persists clarification answers, question/session states, and related readiness/finding changes. -# @DATA_CONTRACT Input[DatasetReviewSession|ClarificationAnswerCommand] -> Output[ClarificationStateResult] -# @INVARIANT Only one active clarification question may exist at a time; skipped and expert-review items remain unresolved and visible. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION DEPENDS_ON -> [ClarificationSession] # @RELATION DEPENDS_ON -> [ClarificationQuestion] # @RELATION DEPENDS_ON -> [ClarificationAnswer] # @RELATION DEPENDS_ON -> [ValidationFinding] # @RELATION DISPATCHES -> [ClarificationHelpers:Module] -# @RATIONALE Original 635-line file exceeded INV_7 (400-line module limit). Extracted pure helpers into _helpers sub-module. -# @REJECTED Keeping all clarification logic in one file because it exceeded the fractal limit. +# @PRE: Target session contains a persisted clarification aggregate in the current ownership scope. +# @POST: Active clarification payload exposes one highest-priority unresolved question, and each recorded answer is persisted before pointer/readiness mutation. +# @SIDE_EFFECT: Persists clarification answers, question/session states, and related readiness/finding changes. +# @DATA_CONTRACT: Input[DatasetReviewSession|ClarificationAnswerCommand] -> Output[ClarificationStateResult] +# @INVARIANT: Only one active clarification question may exist at a time; skipped and expert-review items remain unresolved and visible. +# @RATIONALE: Original 635-line file exceeded INV_7 (400-line module limit). Extracted pure helpers into _helpers sub-module. +# @REJECTED: Keeping all clarification logic in one file because it exceeded the fractal limit. from __future__ import annotations @@ -21,8 +21,7 @@ from dataclasses import dataclass, field from datetime import datetime from typing import List, Optional -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.models.auth import User from src.models.dataset_review import ( AnswerKind, @@ -51,8 +50,6 @@ from src.services.dataset_review.clarification_pkg._helpers import ( derive_recommended_action, ) -log = MarkerLogger("ClarificationEngine") - # #region ClarificationQuestionPayload [C:2] [TYPE Class] # @BRIEF Typed active-question payload returned to the API layer. @@ -101,31 +98,33 @@ class ClarificationAnswerCommand: # #region ClarificationEngine [C:4] [TYPE Class] # @BRIEF Provide deterministic one-question-at-a-time clarification selection and answer persistence. -# @PRE Repository is bound to the current request transaction scope. -# @POST Returned clarification state is persistence-backed and aligned with session readiness/recommended action. -# @SIDE_EFFECT Mutates clarification answers, session flags, and related clarification findings. # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION CALLS -> [ClarificationHelpers:Module] +# @PRE: Repository is bound to the current request transaction scope. +# @POST: Returned clarification state is persistence-backed and aligned with session readiness/recommended action. +# @SIDE_EFFECT: Mutates clarification answers, session flags, and related clarification findings. class ClarificationEngine: - # #region ClarificationEngine_init [C:2] [TYPE Function] - # @BRIEF Bind repository dependency for clarification persistence operations. + # [DEF:ClarificationEngine_init:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind repository dependency for clarification persistence operations. def __init__(self, repository: DatasetReviewSessionRepository) -> None: self.repository = repository - # #endregion ClarificationEngine_init + # [/DEF:ClarificationEngine_init:Function] - # #region build_question_payload [C:4] [TYPE Function] - # @BRIEF Return the one active highest-priority clarification question payload. - # @PRE Session contains unresolved clarification state or a resumable clarification session. - # @POST Returns exactly one active/open question payload or None when no unresolved question remains. - # @SIDE_EFFECT Normalizes the active-question pointer and clarification status in persistence. + # [DEF:build_question_payload:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Return the one active highest-priority clarification question payload. + # @PRE: Session contains unresolved clarification state or a resumable clarification session. + # @POST: Returns exactly one active/open question payload or None when no unresolved question remains. + # @SIDE_EFFECT: Normalizes the active-question pointer and clarification status in persistence. def build_question_payload( self, session: DatasetReviewSession, ) -> Optional[ClarificationQuestionPayload]: with belief_scope("ClarificationEngine.build_question_payload"): clarification_session = self._get_latest_clarification_session(session) if clarification_session is None: - log.reason("No clarification session found", payload={"session_id": session.session_id}) + logger.reason("No clarification session found", extra={"session_id": session.session_id}) return None active_questions = [ @@ -141,7 +140,7 @@ class ClarificationEngine: if session.current_phase == SessionPhase.CLARIFICATION: session.current_phase = SessionPhase.REVIEW self.repository.db.commit() - log.reflect("No unresolved clarification question remains", payload={"session_id": session.session_id}) + logger.reflect("No unresolved clarification question remains", extra={"session_id": session.session_id}) return None selected_question = active_questions[0] @@ -151,7 +150,7 @@ class ClarificationEngine: session.recommended_action = RecommendedAction.ANSWER_NEXT_QUESTION session.current_phase = SessionPhase.CLARIFICATION - log.reason("Selected active clarification question", payload={"session_id": session.session_id, "question_id": selected_question.question_id, "priority": selected_question.priority}) + logger.reason("Selected active clarification question", extra={"session_id": session.session_id, "question_id": selected_question.question_id, "priority": selected_question.priority}) self.repository.db.commit() payload = ClarificationQuestionPayload( @@ -168,40 +167,41 @@ class ClarificationEngine: for o in sorted(selected_question.options, key=lambda item: (item.display_order, item.label, item.option_id)) ], ) - log.reflect("Clarification payload built", payload={"session_id": session.session_id, "question_id": payload.question_id, "option_count": len(payload.options)}) + logger.reflect("Clarification payload built", extra={"session_id": session.session_id, "question_id": payload.question_id, "option_count": len(payload.options)}) return payload - # #endregion build_question_payload + # [/DEF:build_question_payload:Function] - # #region record_answer [C:4] [TYPE Function] - # @BRIEF Persist one clarification answer before any pointer/readiness mutation. - # @PRE Target question belongs to the session's active clarification session and is still open. - # @POST Answer row is persisted before current-question pointer advances. - # @SIDE_EFFECT Inserts answer row, mutates question/session states, updates clarification findings, and commits. + # [DEF:record_answer:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Persist one clarification answer before any pointer/readiness mutation. + # @PRE: Target question belongs to the session's active clarification session and is still open. + # @POST: Answer row is persisted before current-question pointer advances. + # @SIDE_EFFECT: Inserts answer row, mutates question/session states, updates clarification findings, and commits. def record_answer(self, command: ClarificationAnswerCommand) -> ClarificationStateResult: with belief_scope("ClarificationEngine.record_answer"): session = command.session clarification_session = self._get_latest_clarification_session(session) if clarification_session is None: - log.explore("Cannot record clarification answer because no clarification session exists", payload={"session_id": session.session_id}, error="No clarification session found") + logger.explore("Cannot record clarification answer because no clarification session exists", extra={"session_id": session.session_id}) raise ValueError("Clarification session not found") question = self._find_question(clarification_session, command.question_id) if question is None: - log.explore("Cannot record clarification answer for foreign or missing question", payload={"session_id": session.session_id, "question_id": command.question_id}, error="Question not found in clarification session") + logger.explore("Cannot record clarification answer for foreign or missing question", extra={"session_id": session.session_id, "question_id": command.question_id}) raise ValueError("Clarification question not found") if question.answer is not None: - log.explore("Rejected duplicate clarification answer submission", payload={"session_id": session.session_id, "question_id": command.question_id}, error="Question already answered") + logger.explore("Rejected duplicate clarification answer submission", extra={"session_id": session.session_id, "question_id": command.question_id}) raise ValueError("Clarification question already answered") if clarification_session.current_question_id and clarification_session.current_question_id != question.question_id: - log.explore("Rejected answer for non-active clarification question", payload={"session_id": session.session_id, "question_id": question.question_id, "current_question_id": clarification_session.current_question_id}, error="Only active question can be answered") + logger.explore("Rejected answer for non-active clarification question", extra={"session_id": session.session_id, "question_id": question.question_id, "current_question_id": clarification_session.current_question_id}) raise ValueError("Only the active clarification question can be answered") normalized_answer_value = normalize_answer_value(command.answer_kind, command.answer_value, question) - log.reason("Persisting clarification answer before state advancement", payload={"session_id": session.session_id, "question_id": question.question_id, "answer_kind": command.answer_kind.value}) + logger.reason("Persisting clarification answer before state advancement", extra={"session_id": session.session_id, "question_id": question.question_id, "answer_kind": command.answer_kind.value}) persisted_answer = ClarificationAnswer( question_id=question.question_id, answer_kind=command.answer_kind, @@ -248,7 +248,7 @@ class ClarificationEngine: self.repository.db.commit() self.repository.db.refresh(session) - log.reflect("Clarification answer recorded and session advanced", payload={"session_id": session.session_id, "question_id": question.question_id, "next_question_id": clarification_session.current_question_id, "readiness_state": session.readiness_state.value, "remaining_count": clarification_session.remaining_count}) + logger.reflect("Clarification answer recorded and session advanced", extra={"session_id": session.session_id, "question_id": question.question_id, "next_question_id": clarification_session.current_question_id, "readiness_state": session.readiness_state.value, "remaining_count": clarification_session.remaining_count}) return ClarificationStateResult( clarification_session=clarification_session, @@ -257,36 +257,41 @@ class ClarificationEngine: changed_findings=[changed_finding] if changed_finding else [], ) - # #endregion record_answer + # [/DEF:record_answer:Function] - # #region summarize_progress [C:1] [TYPE Function] - # @BRIEF Produce a compact progress summary for pause/resume and completion UX. + # [DEF:summarize_progress:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Produce a compact progress summary for pause/resume and completion UX. def summarize_progress(self, clarification_session: ClarificationSession) -> str: resolved = count_resolved_questions(clarification_session) remaining = count_remaining_questions(clarification_session) return f"{resolved} resolved, {remaining} unresolved" - # #endregion summarize_progress + # [/DEF:summarize_progress:Function] - # #region _get_latest_clarification_session [C:2] [TYPE Function] - # @BRIEF Select the latest clarification session for the current dataset review aggregate. + # [DEF:_get_latest_clarification_session:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Select the latest clarification session for the current dataset review aggregate. def _get_latest_clarification_session(self, session: DatasetReviewSession) -> Optional[ClarificationSession]: if not session.clarification_sessions: return None ordered = sorted(session.clarification_sessions, key=lambda item: (item.started_at, item.clarification_session_id), reverse=True) return ordered[0] - # #endregion _get_latest_clarification_session + # [/DEF:_get_latest_clarification_session:Function] - # #region _find_question [C:1] [TYPE Function] - # @BRIEF Resolve a clarification question from the active clarification aggregate. + # [DEF:_find_question:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Resolve a clarification question from the active clarification aggregate. def _find_question(self, clarification_session: ClarificationSession, question_id: str) -> Optional[ClarificationQuestion]: for q in clarification_session.questions: if q.question_id == question_id: return q return None - # #endregion _find_question + # [/DEF:_find_question:Function] # #endregion ClarificationEngine + +# #endregion ClarificationEngine diff --git a/backend/src/services/dataset_review/clarification_pkg/_helpers.py b/backend/src/services/dataset_review/clarification_pkg/_helpers.py index 466baa33..584458c6 100644 --- a/backend/src/services/dataset_review/clarification_pkg/_helpers.py +++ b/backend/src/services/dataset_review/clarification_pkg/_helpers.py @@ -1,6 +1,6 @@ # #region ClarificationHelpers [C:3] [TYPE Module] # @BRIEF Pure helper functions for clarification engine — question selection, counting, normalization, finding upsert, and readiness derivation. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] from __future__ import annotations diff --git a/backend/src/services/dataset_review/event_logger.py b/backend/src/services/dataset_review/event_logger.py index c16ae9fd..a5412892 100644 --- a/backend/src/services/dataset_review/event_logger.py +++ b/backend/src/services/dataset_review/event_logger.py @@ -1,12 +1,12 @@ # #region SessionEventLoggerModule [C:4] [TYPE Module] [SEMANTICS dataset_review, audit, session_events, persistence, observability] # @BRIEF Persist explicit session mutation events for dataset-review audit trails without weakening ownership or approval invariants. -# @LAYER Domain -# @PRE Caller provides an owned session scope and an authenticated actor identifier for each persisted mutation event. -# @POST Every logged event is committed as an explicit, queryable audit record with deterministic event metadata. -# @SIDE_EFFECT Inserts persisted session event rows and emits runtime belief-state logs for audit-sensitive mutations. -# @DATA_CONTRACT Input[SessionEventPayload] -> Output[SessionEvent] +# @LAYER: Domain # @RELATION DEPENDS_ON -> [SessionEvent] # @RELATION DEPENDS_ON -> [DatasetReviewSession] +# @PRE: Caller provides an owned session scope and an authenticated actor identifier for each persisted mutation event. +# @POST: Every logged event is committed as an explicit, queryable audit record with deterministic event metadata. +# @SIDE_EFFECT: Inserts persisted session event rows and emits runtime belief-state logs for audit-sensitive mutations. +# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent] from __future__ import annotations @@ -16,13 +16,10 @@ from typing import Any, Dict, Optional from sqlalchemy.orm import Session -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.models.dataset_review import DatasetReviewSession, SessionEvent # #endregion SessionEventLoggerImports -log = MarkerLogger("SessionEventLogger") - # #region SessionEventPayload [C:2] [TYPE Class] # @BRIEF Typed input contract for one persisted dataset-review session audit event. @@ -40,26 +37,28 @@ class SessionEventPayload: # #region SessionEventLogger [C:4] [TYPE Class] # @BRIEF Persist explicit dataset-review session audit events with meaningful runtime reasoning logs. -# @PRE The database session is live and payload identifiers are non-empty. -# @POST Returns the committed session event row with a stable identifier and stored detail payload. -# @SIDE_EFFECT Writes one audit row to persistence and emits logger.reason/logger.reflect traces. -# @DATA_CONTRACT Input[SessionEventPayload] -> Output[SessionEvent] # @RELATION DEPENDS_ON -> [SessionEvent] # @RELATION DEPENDS_ON -> [SessionEventPayload] +# @PRE: The database session is live and payload identifiers are non-empty. +# @POST: Returns the committed session event row with a stable identifier and stored detail payload. +# @SIDE_EFFECT: Writes one audit row to persistence and emits logger.reason/logger.reflect traces. +# @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent] class SessionEventLogger: - # #region SessionEventLogger_init [C:2] [TYPE Function] - # @BRIEF Bind a live SQLAlchemy session to the session-event logger. + # [DEF:SessionEventLogger_init:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind a live SQLAlchemy session to the session-event logger. def __init__(self, db: Session) -> None: self.db = db - # #endregion SessionEventLogger_init + # [/DEF:SessionEventLogger_init:Function] - # #region log_event [C:4] [TYPE Function] - # @BRIEF Persist one explicit session event row for an owned dataset-review mutation. - # @PRE session_id, actor_user_id, event_type, and event_summary are non-empty. - # @POST Returns the committed SessionEvent record with normalized detail payload. - # @SIDE_EFFECT Inserts and commits one session_events row. - # @DATA_CONTRACT Input[SessionEventPayload] -> Output[SessionEvent] - # @RELATION DEPENDS_ON -> [SessionEvent] + # [DEF:log_event:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Persist one explicit session event row for an owned dataset-review mutation. + # @RELATION: [DEPENDS_ON] ->[SessionEvent] + # @PRE: session_id, actor_user_id, event_type, and event_summary are non-empty. + # @POST: Returns the committed SessionEvent record with normalized detail payload. + # @SIDE_EFFECT: Inserts and commits one session_events row. + # @DATA_CONTRACT: Input[SessionEventPayload] -> Output[SessionEvent] def log_event(self, payload: SessionEventPayload) -> SessionEvent: with belief_scope("SessionEventLogger.log_event"): session_id = str(payload.session_id or "").strip() @@ -68,22 +67,31 @@ class SessionEventLogger: event_summary = str(payload.event_summary or "").strip() if not session_id: - log.explore("Session event logging rejected because session_id is empty", error="session_id is empty") + logger.explore("Session event logging rejected because session_id is empty") raise ValueError("session_id must be non-empty") if not actor_user_id: - log.explore("Session event logging rejected because actor_user_id is empty", error="actor_user_id is empty", payload={"session_id": session_id}) + logger.explore( + "Session event logging rejected because actor_user_id is empty", + extra={"session_id": session_id}, + ) raise ValueError("actor_user_id must be non-empty") if not event_type: - log.explore("Session event logging rejected because event_type is empty", error="event_type is empty", payload={"session_id": session_id, "actor_user_id": actor_user_id}) + logger.explore( + "Session event logging rejected because event_type is empty", + extra={"session_id": session_id, "actor_user_id": actor_user_id}, + ) raise ValueError("event_type must be non-empty") if not event_summary: - log.explore("Session event logging rejected because event_summary is empty", error="event_summary is empty", payload={"session_id": session_id, "event_type": event_type}) + logger.explore( + "Session event logging rejected because event_summary is empty", + extra={"session_id": session_id, "event_type": event_type}, + ) raise ValueError("event_summary must be non-empty") normalized_details = dict(payload.event_details or {}) - log.reason( + logger.reason( "Persisting explicit dataset-review session audit event", - payload={ + extra={ "session_id": session_id, "actor_user_id": actor_user_id, "event_type": event_type, @@ -105,20 +113,21 @@ class SessionEventLogger: self.db.commit() self.db.refresh(event) - log.reflect( + logger.reflect( "Dataset-review session audit event persisted", - payload={ + extra={ "session_id": session_id, "session_event_id": event.session_event_id, "event_type": event.event_type, }, ) return event - # #endregion log_event + # [/DEF:log_event:Function] - # #region log_for_session [C:2] [TYPE Function] - # @BRIEF Convenience wrapper for logging an event directly from a session aggregate root. - # @RELATION CALLS -> [SessionEventLogger.log_event] + # [DEF:log_for_session:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Convenience wrapper for logging an event directly from a session aggregate root. + # @RELATION: [CALLS] ->[SessionEventLogger.log_event] def log_for_session( self, session: DatasetReviewSession, @@ -139,5 +148,7 @@ class SessionEventLogger: event_details=dict(event_details or {}), ) ) - # #endregion log_for_session + # [/DEF:log_for_session:Function] # #endregion SessionEventLogger + +# #endregion SessionEventLoggerModule \ No newline at end of file diff --git a/backend/src/services/dataset_review/orchestrator.py b/backend/src/services/dataset_review/orchestrator.py index 4a7c3c49..6dde2c08 100644 --- a/backend/src/services/dataset_review/orchestrator.py +++ b/backend/src/services/dataset_review/orchestrator.py @@ -1,11 +1,6 @@ # #region DatasetReviewOrchestrator [C:5] [TYPE Module] [SEMANTICS dataset_review, orchestration, session_lifecycle, intake, recovery] # @BRIEF Coordinate dataset review session startup and lifecycle-safe intake recovery for one authenticated user. -# @LAYER Domain -# @PRE session mutations must execute inside a persisted session boundary scoped to one authenticated user. -# @POST state transitions are persisted atomically and emit observable progress for long-running steps. -# @SIDE_EFFECT creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts. -# @DATA_CONTRACT Input[SessionCommand] -> Output[DatasetReviewSession | CompiledPreview | DatasetRunContext] -# @INVARIANT Launch is blocked unless a current session has no open blocking findings, all launch-sensitive mappings are approved, and a non-stale Superset-generated compiled preview matches the current input fingerprint. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION DEPENDS_ON -> [SemanticSourceResolver] # @RELATION DEPENDS_ON -> [SupersetContextExtractor] @@ -13,8 +8,13 @@ # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DISPATCHES -> [OrchestratorHelpers:Module] # @RELATION DISPATCHES -> [OrchestratorCommands:Module] -# @RATIONALE Original 1198-line monolith violated INV_7 (400-line module limit). Decomposed into commands and helpers sub-modules while preserving the orchestrator class as the single entry point. -# @REJECTED Keeping all orchestration logic in one file because it exceeded the fractal limit by 3x. +# @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user. +# @POST: state transitions are persisted atomically and emit observable progress for long-running steps. +# @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts. +# @DATA_CONTRACT: Input[SessionCommand] -> Output[DatasetReviewSession | CompiledPreview | DatasetRunContext] +# @INVARIANT: Launch is blocked unless a current session has no open blocking findings, all launch-sensitive mappings are approved, and a non-stale Superset-generated compiled preview matches the current input fingerprint. +# @RATIONALE: Original 1198-line monolith violated INV_7 (400-line module limit). Decomposed into commands and helpers sub-modules while preserving the orchestrator class as the single entry point. +# @REJECTED: Keeping all orchestration logic in one file because it exceeded the fractal limit by 3x. from __future__ import annotations @@ -23,8 +23,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional, cast from src.core.config_manager import ConfigManager -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.core.task_manager import TaskManager from src.core.utils.superset_compilation_adapter import ( PreviewCompilationPayload, @@ -88,27 +87,28 @@ from src.services.dataset_review.orchestrator_pkg._helpers import ( extract_effective_filter_value, ) -log = MarkerLogger("Orchestrator") +logger = cast(Any, logger) # #region DatasetReviewOrchestrator [C:5] [TYPE Class] # @BRIEF Coordinate safe session startup while preserving cross-user isolation and explicit partial recovery. -# @PRE constructor dependencies are valid and tied to the current request/task scope. -# @POST orchestrator instance can execute session-scoped mutations for one authenticated user. -# @SIDE_EFFECT downstream operations may persist session/profile/finding state and enqueue background tasks. -# @DATA_CONTRACT Input[StartSessionCommand] -> Output[StartSessionResult] -# @INVARIANT session ownership is preserved on every mutation and recovery remains explicit when partial. # @RELATION DEPENDS_ON -> [DatasetReviewSessionRepository] # @RELATION DEPENDS_ON -> [SupersetContextExtractor] # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [ConfigManager] # @RELATION DEPENDS_ON -> [SemanticSourceResolver] # @RELATION CALLS -> [OrchestratorHelpers:Module] +# @PRE: constructor dependencies are valid and tied to the current request/task scope. +# @POST: orchestrator instance can execute session-scoped mutations for one authenticated user. +# @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks. +# @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult] +# @INVARIANT: session ownership is preserved on every mutation and recovery remains explicit when partial. class DatasetReviewOrchestrator: - # #region DatasetReviewOrchestrator_init [C:3] [TYPE Function] - # @BRIEF Bind repository, config, and task dependencies required by the orchestration boundary. - # @PRE repository/config_manager are valid collaborators for the current request scope. - # @POST Instance holds collaborator references used by start/preview/launch orchestration methods. + # [DEF:DatasetReviewOrchestrator_init:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary. + # @PRE: repository/config_manager are valid collaborators for the current request scope. + # @POST: Instance holds collaborator references used by start/preview/launch orchestration methods. def __init__( self, repository: DatasetReviewSessionRepository, @@ -121,17 +121,18 @@ class DatasetReviewOrchestrator: self.task_manager = task_manager self.semantic_resolver = semantic_resolver or SemanticSourceResolver() - # #endregion DatasetReviewOrchestrator_init + # [/DEF:DatasetReviewOrchestrator_init:Function] - # #region start_session [C:5] [TYPE Function] - # @BRIEF Initialize a new session from a Superset link or dataset selection and trigger context recovery. - # @PRE source input is non-empty and environment is accessible. - # @POST session exists in persisted storage with intake/recovery state and task linkage when async work is required. - # @SIDE_EFFECT persists session and may enqueue recovery task. - # @DATA_CONTRACT Input[StartSessionCommand] -> Output[StartSessionResult] - # @INVARIANT no cross-user session leakage occurs; session and follow-up task remain owned by the authenticated user. - # @RELATION CALLS -> [SupersetContextExtractor.parse_superset_link] - # @RELATION CALLS -> [TaskManager.create_task] + # [DEF:start_session:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery. + # @RELATION: CALLS -> [SupersetContextExtractor.parse_superset_link] + # @RELATION: CALLS -> [TaskManager.create_task] + # @PRE: source input is non-empty and environment is accessible. + # @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required. + # @SIDE_EFFECT: persists session and may enqueue recovery task. + # @DATA_CONTRACT: Input[StartSessionCommand] -> Output[StartSessionResult] + # @INVARIANT: no cross-user session leakage occurs; session and follow-up task remain owned by the authenticated user. def start_session(self, command: StartSessionCommand) -> StartSessionResult: with belief_scope("DatasetReviewOrchestrator.start_session"): normalized_source_kind = str(command.source_kind or "").strip() @@ -139,19 +140,19 @@ class DatasetReviewOrchestrator: normalized_environment_id = str(command.environment_id or "").strip() if not normalized_source_input: - log.explore("Blocked dataset review session start due to empty source input", error="source_input was empty") + logger.explore("Blocked dataset review session start due to empty source input") raise ValueError("source_input must be non-empty") if normalized_source_kind not in {"superset_link", "dataset_selection"}: - log.explore("Blocked dataset review session start due to unsupported source kind", payload={"source_kind": normalized_source_kind}, error="Unsupported source_kind") + logger.explore("Blocked dataset review session start due to unsupported source kind", extra={"source_kind": normalized_source_kind}) raise ValueError("source_kind must be 'superset_link' or 'dataset_selection'") environment = self.config_manager.get_environment(normalized_environment_id) if environment is None: - log.explore("Blocked dataset review session start because environment was not found", payload={"environment_id": normalized_environment_id}, error="Environment not found in config") + logger.explore("Blocked dataset review session start because environment was not found", extra={"environment_id": normalized_environment_id}) raise ValueError("Environment not found") - log.reason("Starting dataset review session", payload={"user_id": command.user.id, "environment_id": normalized_environment_id, "source_kind": normalized_source_kind}) + logger.reason("Starting dataset review session", extra={"user_id": command.user.id, "environment_id": normalized_environment_id, "source_kind": normalized_source_kind}) parsed_context: Optional[SupersetParsedContext] = None findings: List[ValidationFinding] = [] @@ -213,7 +214,7 @@ class DatasetReviewOrchestrator: parsed_context=parsed_context, dataset_ref=dataset_ref, ) - self.repository.event_log.log_event( + self.repository.event_logger.log_event( SessionEventPayload( session_id=persisted_session.session_id, actor_user_id=command.user.id, @@ -255,7 +256,7 @@ class DatasetReviewOrchestrator: self.repository.bump_session_version(persisted_session) self.repository.db.commit() self.repository.db.refresh(persisted_session) - self.repository.event_log.log_event( + self.repository.event_logger.log_event( SessionEventPayload( session_id=persisted_session.session_id, actor_user_id=command.user.id, @@ -266,29 +267,30 @@ class DatasetReviewOrchestrator: event_details={"task_id": active_task_id}, ) ) - log.reason("Linked recovery task to started dataset review session", payload={"session_id": persisted_session.session_id, "task_id": active_task_id}) + logger.reason("Linked recovery task to started dataset review session", extra={"session_id": persisted_session.session_id, "task_id": active_task_id}) - log.reflect("Dataset review session start completed", payload={"session_id": persisted_session.session_id, "dataset_ref": persisted_session.dataset_ref, "readiness_state": persisted_session.readiness_state.value, "active_task_id": persisted_session.active_task_id, "finding_count": len(findings)}) + logger.reflect("Dataset review session start completed", extra={"session_id": persisted_session.session_id, "dataset_ref": persisted_session.dataset_ref, "readiness_state": persisted_session.readiness_state.value, "active_task_id": persisted_session.active_task_id, "finding_count": len(findings)}) return StartSessionResult( session=persisted_session, parsed_context=parsed_context, findings=findings, ) - # #endregion start_session + # [/DEF:start_session:Function] - # #region prepare_launch_preview [C:4] [TYPE Function] - # @BRIEF Assemble effective execution inputs and trigger Superset-side preview compilation. - # @PRE all required variables have candidate values or explicitly accepted defaults. - # @POST returns preview artifact in pending, ready, failed, or stale state. - # @SIDE_EFFECT persists preview attempt and upstream compilation diagnostics. - # @DATA_CONTRACT Input[PreparePreviewCommand] -> Output[PreparePreviewResult] - # @RELATION CALLS -> [SupersetCompilationAdapter.compile_preview] + # [DEF:prepare_launch_preview:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Assemble effective execution inputs and trigger Superset-side preview compilation. + # @RELATION: CALLS -> [SupersetCompilationAdapter.compile_preview] + # @PRE: all required variables have candidate values or explicitly accepted defaults. + # @POST: returns preview artifact in pending, ready, failed, or stale state. + # @SIDE_EFFECT: persists preview attempt and upstream compilation diagnostics. + # @DATA_CONTRACT: Input[PreparePreviewCommand] -> Output[PreparePreviewResult] def prepare_launch_preview(self, command: PreparePreviewCommand) -> PreparePreviewResult: with belief_scope("DatasetReviewOrchestrator.prepare_launch_preview"): session = self.repository.load_session_detail(command.session_id, command.user.id) if session is None or session.user_id != command.user.id: - log.explore("Preview preparation rejected because owned session was not found", payload={"session_id": command.session_id, "user_id": command.user.id}, error="Session not found or access denied") + logger.explore("Preview preparation rejected because owned session was not found", extra={"session_id": command.session_id, "user_id": command.user.id}) raise ValueError("Session not found") if command.expected_version is not None: @@ -304,7 +306,7 @@ class DatasetReviewOrchestrator: execution_snapshot = build_execution_snapshot(session) preview_blockers = execution_snapshot["preview_blockers"] if preview_blockers: - log.explore("Preview preparation blocked by incomplete execution context", payload={"session_id": session.session_id, "blocked_reasons": preview_blockers}, error="Preview blockers present") + logger.explore("Preview preparation blocked by incomplete execution context", extra={"session_id": session.session_id, "blocked_reasons": preview_blockers}) raise ValueError("Preview blocked: " + "; ".join(preview_blockers)) adapter = SupersetCompilationAdapter(environment) @@ -339,7 +341,7 @@ class DatasetReviewOrchestrator: session.recommended_action = RecommendedAction.GENERATE_SQL_PREVIEW self.repository.db.commit() self.repository.db.refresh(session) - self.repository.event_log.log_event( + self.repository.event_logger.log_event( SessionEventPayload( session_id=session.session_id, actor_user_id=command.user.id, @@ -351,24 +353,25 @@ class DatasetReviewOrchestrator: ) ) - log.reflect("Superset preview preparation completed", payload={"session_id": session.session_id, "preview_id": persisted_preview.preview_id, "preview_status": persisted_preview.preview_status.value}) + logger.reflect("Superset preview preparation completed", extra={"session_id": session.session_id, "preview_id": persisted_preview.preview_id, "preview_status": persisted_preview.preview_status.value}) return PreparePreviewResult(session=session, preview=persisted_preview, blocked_reasons=[]) - # #endregion prepare_launch_preview + # [/DEF:prepare_launch_preview:Function] - # #region launch_dataset [C:5] [TYPE Function] - # @BRIEF Start the approved dataset execution through SQL Lab and persist run context for audit/replay. - # @PRE session is run-ready and compiled preview is current. - # @POST returns persisted run context with SQL Lab session reference and launch outcome. - # @SIDE_EFFECT creates SQL Lab execution session and audit snapshot. - # @DATA_CONTRACT Input[LaunchDatasetCommand] -> Output[LaunchDatasetResult] - # @INVARIANT launch remains blocked unless blocking findings are closed, approvals are satisfied, and the latest preview fingerprint matches current execution inputs. - # @RELATION CALLS -> [SupersetCompilationAdapter.create_sql_lab_session] + # [DEF:launch_dataset:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Start the approved dataset execution through SQL Lab and persist run context for audit/replay. + # @RELATION: CALLS -> [SupersetCompilationAdapter.create_sql_lab_session] + # @PRE: session is run-ready and compiled preview is current. + # @POST: returns persisted run context with SQL Lab session reference and launch outcome. + # @SIDE_EFFECT: creates SQL Lab execution session and audit snapshot. + # @DATA_CONTRACT: Input[LaunchDatasetCommand] -> Output[LaunchDatasetResult] + # @INVARIANT: launch remains blocked unless blocking findings are closed, approvals are satisfied, and the latest preview fingerprint matches current execution inputs. def launch_dataset(self, command: LaunchDatasetCommand) -> LaunchDatasetResult: with belief_scope("DatasetReviewOrchestrator.launch_dataset"): session = self.repository.load_session_detail(command.session_id, command.user.id) if session is None or session.user_id != command.user.id: - log.explore("Launch rejected because owned session was not found", payload={"session_id": command.session_id, "user_id": command.user.id}, error="Session not found or access denied") + logger.explore("Launch rejected because owned session was not found", extra={"session_id": command.session_id, "user_id": command.user.id}) raise ValueError("Session not found") if command.expected_version is not None: @@ -385,7 +388,7 @@ class DatasetReviewOrchestrator: current_preview = get_latest_preview(session) launch_blockers_list = build_launch_blockers(session=session, execution_snapshot=execution_snapshot, preview=current_preview) if launch_blockers_list: - log.explore("Launch gate blocked dataset execution", payload={"session_id": session.session_id, "blocked_reasons": launch_blockers_list}, error="Launch blockers present") + logger.explore("Launch gate blocked dataset execution", extra={"session_id": session.session_id, "blocked_reasons": launch_blockers_list}) raise ValueError("Launch blocked: " + "; ".join(launch_blockers_list)) adapter = SupersetCompilationAdapter(environment) @@ -402,7 +405,7 @@ class DatasetReviewOrchestrator: launch_status = LaunchStatus.STARTED launch_error = None except Exception as exc: - log.explore("SQL Lab launch failed after passing gates", payload={"session_id": session.session_id}, error=str(exc)) + logger.explore("SQL Lab launch failed after passing gates", extra={"session_id": session.session_id, "error": str(exc)}) sql_lab_session_ref = "unavailable" launch_status = LaunchStatus.FAILED launch_error = str(exc) @@ -438,7 +441,7 @@ class DatasetReviewOrchestrator: session.recommended_action = RecommendedAction.EXPORT_OUTPUTS self.repository.db.commit() self.repository.db.refresh(session) - self.repository.event_log.log_event( + self.repository.event_logger.log_event( SessionEventPayload( session_id=session.session_id, actor_user_id=command.user.id, @@ -450,16 +453,17 @@ class DatasetReviewOrchestrator: ) ) - log.reflect("Dataset launch orchestration completed with audited run context", payload={"session_id": session.session_id, "run_context_id": persisted_run_context.run_context_id, "launch_status": persisted_run_context.launch_status.value}) + logger.reflect("Dataset launch orchestration completed with audited run context", extra={"session_id": session.session_id, "run_context_id": persisted_run_context.run_context_id, "launch_status": persisted_run_context.launch_status.value}) return LaunchDatasetResult(session=session, run_context=persisted_run_context, blocked_reasons=[]) - # #endregion launch_dataset + # [/DEF:launch_dataset:Function] - # #region _build_recovery_bootstrap [C:4] [TYPE Function] - # @BRIEF Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation. - # @PRE session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope. - # @POST Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly. - # @SIDE_EFFECT Performs Superset reads through the extractor and may append warning findings for incomplete recovery. + # [DEF:_build_recovery_bootstrap:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Recover and materialize initial imported filters, template variables, and draft execution mappings after session creation. + # @PRE: session belongs to the just-created review aggregate and parsed_context was produced for the same environment scope. + # @POST: Returns bootstrap imported filters, template variables, execution mappings, and updated findings without persisting them directly. + # @SIDE_EFFECT: Performs Superset reads through the extractor and may append warning findings for incomplete recovery. def _build_recovery_bootstrap( self, environment, @@ -525,7 +529,7 @@ class DatasetReviewOrchestrator: caused_by_ref="dataset_template_variable_discovery_failed", ) ) - log.explore("Template variable discovery failed during session bootstrap", payload={"session_id": session_record.session_id, "dataset_id": session_record.dataset_id}, error=str(exc)) + logger.explore("Template variable discovery failed during session bootstrap", extra={"session_id": session_record.session_id, "dataset_id": session_record.dataset_id, "error": str(exc)}) filter_lookup = {str(f.filter_name or "").strip().lower(): f for f in imported_filters if str(f.filter_name or "").strip()} for tv in template_variables: @@ -552,13 +556,14 @@ class DatasetReviewOrchestrator: return imported_filters, template_variables, execution_mappings, findings - # #endregion _build_recovery_bootstrap + # [/DEF:_build_recovery_bootstrap:Function] - # #region _enqueue_recovery_task [C:3] [TYPE Function] - # @BRIEF Link session start to observable async recovery when task infrastructure is available. - # @PRE session is already persisted. - # @POST returns task identifier when a task could be enqueued, otherwise None. - # @SIDE_EFFECT may create one background task for progressive recovery. + # [DEF:_enqueue_recovery_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Link session start to observable async recovery when task infrastructure is available. + # @PRE: session is already persisted. + # @POST: returns task identifier when a task could be enqueued, otherwise None. + # @SIDE_EFFECT: may create one background task for progressive recovery. def _enqueue_recovery_task( self, command: StartSessionCommand, @@ -567,7 +572,7 @@ class DatasetReviewOrchestrator: ) -> Optional[str]: session_record = cast(Any, session) if self.task_manager is None: - log.reason("Dataset review session started without task manager; continuing synchronously", payload={"session_id": session_record.session_id}) + logger.reason("Dataset review session started without task manager; continuing synchronously", extra={"session_id": session_record.session_id}) return None task_params: Dict[str, Any] = { @@ -584,19 +589,21 @@ class DatasetReviewOrchestrator: create_task = getattr(self.task_manager, "create_task", None) if create_task is None: - log.explore("Task manager has no create_task method; skipping recovery enqueue", error="create_task method not found") + logger.explore("Task manager has no create_task method; skipping recovery enqueue") return None try: task_object = create_task(plugin_id="dataset-review-recovery", params=task_params) except TypeError: - log.explore("Recovery task enqueue skipped because task manager create_task contract is incompatible", payload={"session_id": session_record.session_id}, error="TypeError from create_task") + logger.explore("Recovery task enqueue skipped because task manager create_task contract is incompatible", extra={"session_id": session_record.session_id}) return None task_id = getattr(task_object, "id", None) return str(task_id) if task_id else None - # #endregion _enqueue_recovery_task + # [/DEF:_enqueue_recovery_task:Function] # #endregion DatasetReviewOrchestrator + +# #endregion DatasetReviewOrchestrator diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py index 8b0e6516..55781608 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_commands.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_commands.py @@ -1,6 +1,6 @@ # #region OrchestratorCommands [C:2] [TYPE Module] # @BRIEF Typed command and result dataclasses for dataset review orchestration boundary. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] # @RELATION DEPENDS_ON -> [SupersetContextExtractor] diff --git a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py index f53ae775..2080aac3 100644 --- a/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py +++ b/backend/src/services/dataset_review/orchestrator_pkg/_helpers.py @@ -1,10 +1,10 @@ # #region OrchestratorHelpers [C:4] [TYPE Module] # @BRIEF Pure helper methods extracted from DatasetReviewOrchestrator for INV_7 compliance — snapshot, blockers, fingerprint, recovery bootstrap. -# @LAYER Domain -# @PRE Caller provides a loaded session aggregate with hydrated child collections. -# @POST Helper results are deterministic and do not mutate persistence directly. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] # @RELATION DEPENDS_ON -> [SupersetContextExtractor] +# @PRE: Caller provides a loaded session aggregate with hydrated child collections. +# @POST: Helper results are deterministic and do not mutate persistence directly. from __future__ import annotations @@ -13,7 +13,7 @@ import json from datetime import datetime from typing import Any, Dict, List, Optional, cast -from src.core.cot_logger import MarkerLogger +from src.core.logger import belief_scope, logger from src.models.dataset_review import ( ApprovalState, CompiledPreview, @@ -37,7 +37,8 @@ from src.models.dataset_review import ( BusinessSummarySource, ) -log = MarkerLogger("OrchestratorHelpers") +logger = cast(Any, logger) + # #region parse_dataset_selection [C:2] [TYPE Function] # @BRIEF Normalize dataset-selection payload into canonical session references. @@ -103,8 +104,8 @@ def build_initial_profile( # #region build_partial_recovery_findings [C:3] [TYPE Function] # @BRIEF Project partial Superset intake recovery into explicit findings without blocking session usability. -# @PRE parsed_context.partial_recovery is true. -# @POST Returns warning-level findings that preserve usable but incomplete state. +# @PRE: parsed_context.partial_recovery is true. +# @POST: Returns warning-level findings that preserve usable but incomplete state. def build_partial_recovery_findings(parsed_context: Any) -> List[ValidationFinding]: findings: List[ValidationFinding] = [] for unresolved_ref in getattr(parsed_context, "unresolved_references", []): @@ -146,8 +147,8 @@ def extract_effective_filter_value( # #region build_execution_snapshot [C:4] [TYPE Function] # @BRIEF Build effective filters, template params, approvals, and fingerprint for preview and launch gating. -# @PRE Session aggregate includes imported filters, template variables, and current execution mappings. -# @POST Returns deterministic execution snapshot for current session state without mutating persistence. +# @PRE: Session aggregate includes imported filters, template variables, and current execution mappings. +# @POST: Returns deterministic execution snapshot for current session state without mutating persistence. def build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]: session_record = cast(Any, session) filter_lookup = { @@ -275,8 +276,8 @@ def build_execution_snapshot(session: DatasetReviewSession) -> Dict[str, Any]: # #region build_launch_blockers [C:3] [TYPE Function] # @BRIEF Enforce launch gates from findings, approvals, and current preview truth. -# @PRE execution_snapshot was computed from current session state. -# @POST Returns explicit blocker codes for every unmet launch invariant. +# @PRE: execution_snapshot was computed from current session state. +# @POST: Returns explicit blocker codes for every unmet launch invariant. def build_launch_blockers( session: DatasetReviewSession, execution_snapshot: Dict[str, Any], diff --git a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py index eb12eef4..de239355 100644 --- a/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py +++ b/backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py @@ -26,16 +26,18 @@ from src.services.dataset_review.repositories.session_repository import ( DatasetReviewSessionVersionConflictError, ) -# #region SessionRepositoryTests [C:2] [TYPE Module] -# @BRIEF Unit tests for DatasetReviewSessionRepository. -# @RELATION BELONGS_TO -> [SrcRoot] +# [DEF:SessionRepositoryTests:Module] +# @RELATION: BELONGS_TO -> SrcRoot +# @COMPLEXITY: 2 +# @PURPOSE: Unit tests for DatasetReviewSessionRepository. @pytest.fixture def db_session(): - # #region db_session [C:2] [TYPE Function] - # @BRIEF Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows. - # @RELATION BINDS_TO -> [SessionRepositoryTests] + # [DEF:db_session:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows. + # @RELATION: BINDS_TO -> [SessionRepositoryTests] engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(engine) Session = sessionmaker(bind=engine) @@ -55,11 +57,11 @@ def db_session(): session.close() - # #endregion db_session +# [/DEF:db_session:Function] -# #region test_create_session [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_create_session:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_create_session(db_session): # @PURPOSE: Verify session creation and persistence. repo = DatasetReviewSessionRepository(db_session) @@ -82,12 +84,12 @@ def test_create_session(db_session): assert loaded.version == 0 -# #endregion test_create_session +# [/DEF:test_create_session:Function] -# #region test_require_session_version_conflict [TYPE Function] -# @BRIEF Verify optimistic-lock conflict is raised when caller version is stale. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_require_session_version_conflict:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify optimistic-lock conflict is raised when caller version is stale. def test_require_session_version_conflict(db_session): repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( @@ -107,12 +109,12 @@ def test_require_session_version_conflict(db_session): assert exc_info.value.actual_version == 2 -# #endregion test_require_session_version_conflict +# [/DEF:test_require_session_version_conflict:Function] -# #region test_bump_session_version_updates_last_activity [TYPE Function] -# @BRIEF Verify repository version bump increments monotonically and refreshes last activity. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_bump_session_version_updates_last_activity:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify repository version bump increments monotonically and refreshes last activity. def test_bump_session_version_updates_last_activity(db_session): repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( @@ -132,12 +134,12 @@ def test_bump_session_version_updates_last_activity(db_session): assert session.last_activity_at >= before_activity -# #endregion test_bump_session_version_updates_last_activity +# [/DEF:test_bump_session_version_updates_last_activity:Function] -# #region test_save_recovery_state_preserves_raw_value_masked_flag [TYPE Function] -# @BRIEF Verify imported-filter masking metadata persists with recovery bootstrap state. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_recovery_state_preserves_raw_value_masked_flag:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify imported-filter masking metadata persists with recovery bootstrap state. def test_save_recovery_state_preserves_raw_value_masked_flag(db_session): repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( @@ -173,11 +175,11 @@ def test_save_recovery_state_preserves_raw_value_masked_flag(db_session): assert updated.version == 1 -# #endregion test_save_recovery_state_preserves_raw_value_masked_flag +# [/DEF:test_save_recovery_state_preserves_raw_value_masked_flag:Function] -# #region test_load_session_detail_ownership [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_load_session_detail_ownership:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_load_session_detail_ownership(db_session): # @PURPOSE: Verify ownership enforcement in detail loading. repo = DatasetReviewSessionRepository(db_session) @@ -199,11 +201,11 @@ def test_load_session_detail_ownership(db_session): assert loaded_wrong is None -# #endregion test_load_session_detail_ownership +# [/DEF:test_load_session_detail_ownership:Function] -# #region test_load_session_detail_collaborator [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_load_session_detail_collaborator:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_load_session_detail_collaborator(db_session): # @PURPOSE: Verify collaborator access in detail loading. repo = DatasetReviewSessionRepository(db_session) @@ -236,11 +238,11 @@ def test_load_session_detail_collaborator(db_session): assert loaded.session_id == session.session_id -# #endregion test_load_session_detail_collaborator +# [/DEF:test_load_session_detail_collaborator:Function] -# #region test_save_preview_marks_stale [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_preview_marks_stale:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_preview_marks_stale(db_session): # @PURPOSE: Verify that saving a new preview marks old ones as stale. repo = DatasetReviewSessionRepository(db_session) @@ -269,12 +271,12 @@ def test_save_preview_marks_stale(db_session): assert session.last_preview_id == p2.preview_id -# #endregion test_save_preview_marks_stale +# [/DEF:test_save_preview_marks_stale:Function] -# #region test_save_preview_increments_session_version_once_per_call [TYPE Function] -# @BRIEF Verify preview persistence itself contributes exactly one optimistic-lock version increment so higher orchestration layers do not need to bump again for the same preview mutation. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_preview_increments_session_version_once_per_call:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify preview persistence itself contributes exactly one optimistic-lock version increment so higher orchestration layers do not need to bump again for the same preview mutation. def test_save_preview_increments_session_version_once_per_call(db_session): repo = DatasetReviewSessionRepository(db_session) session = DatasetReviewSession( @@ -311,11 +313,11 @@ def test_save_preview_increments_session_version_once_per_call(db_session): assert session.version == 2 -# #endregion test_save_preview_increments_session_version_once_per_call +# [/DEF:test_save_preview_increments_session_version_once_per_call:Function] -# #region test_save_profile_and_findings [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_profile_and_findings:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_profile_and_findings(db_session): # @PURPOSE: Verify persistence of profile and findings. repo = DatasetReviewSessionRepository(db_session) @@ -372,12 +374,12 @@ def test_save_profile_and_findings(db_session): assert final_session.version == 2 -# #endregion test_save_profile_and_findings +# [/DEF:test_save_profile_and_findings:Function] -# #region test_save_profile_and_findings_rejects_stale_concurrent_write [TYPE Function] -# @BRIEF Verify repository save path translates concurrent stale session writes into deterministic optimistic-lock conflicts. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_profile_and_findings_rejects_stale_concurrent_write:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify repository save path translates concurrent stale session writes into deterministic optimistic-lock conflicts. def test_save_profile_and_findings_rejects_stale_concurrent_write(tmp_path: Path): db_path = tmp_path / "dataset_review_session_repository.sqlite" engine = create_engine(f"sqlite:///{db_path}") @@ -472,11 +474,11 @@ def test_save_profile_and_findings_rejects_stale_concurrent_write(tmp_path: Path writer_b.close() -# #endregion test_save_profile_and_findings_rejects_stale_concurrent_write +# [/DEF:test_save_profile_and_findings_rejects_stale_concurrent_write:Function] -# #region test_save_run_context [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_save_run_context:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_save_run_context(db_session): # @PURPOSE: Verify saving of run context. repo = DatasetReviewSessionRepository(db_session) @@ -507,12 +509,12 @@ def test_save_run_context(db_session): assert session.last_run_context_id == rc.run_context_id -# #endregion test_save_run_context +# [/DEF:test_save_run_context:Function] -# #region test_ensure_dataset_review_session_columns_adds_missing_legacy_columns [TYPE Function] -# @BRIEF Verify additive dataset review migration creates missing legacy columns for session and imported-filter tables without dropping rows. -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_ensure_dataset_review_session_columns_adds_missing_legacy_columns:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests +# @PURPOSE: Verify additive dataset review migration creates missing legacy columns for session and imported-filter tables without dropping rows. def test_ensure_dataset_review_session_columns_adds_missing_legacy_columns(): engine = create_engine("sqlite:///:memory:") with engine.begin() as connection: @@ -674,11 +676,11 @@ def test_ensure_dataset_review_session_columns_adds_missing_legacy_columns(): assert raw_value_masked in (False, 0) -# #endregion test_ensure_dataset_review_session_columns_adds_missing_legacy_columns +# [/DEF:test_ensure_dataset_review_session_columns_adds_missing_legacy_columns:Function] -# #region test_list_sessions_for_user [TYPE Function] -# @RELATION BINDS_TO -> [SessionRepositoryTests] +# [DEF:test_list_sessions_for_user:Function] +# @RELATION: BINDS_TO -> SessionRepositoryTests def test_list_sessions_for_user(db_session): # @PURPOSE: Verify listing of sessions by user. repo = DatasetReviewSessionRepository(db_session) @@ -712,5 +714,5 @@ def test_list_sessions_for_user(db_session): assert all(s.user_id == "user1" for s in sessions) -# #endregion test_list_sessions_for_user -# #endregion SessionRepositoryTests +# [/DEF:test_list_sessions_for_user:Function] +# [/DEF:SessionRepositoryTests:Module] diff --git a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py index 991ee223..0ad1bc94 100644 --- a/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py +++ b/backend/src/services/dataset_review/repositories/repository_pkg/_mutations.py @@ -1,10 +1,10 @@ # #region SessionRepositoryMutations [C:4] [TYPE Module] # @BRIEF Persistence mutation operations for dataset review session aggregates — profile/findings, recovery state, preview, run context. -# @LAYER Domain -# @PRE All mutations execute within authenticated request or task scope. -# @POST Session aggregate writes preserve ownership and version semantics. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewModels] # @RELATION DEPENDS_ON -> [SessionEventLogger] +# @PRE: All mutations execute within authenticated request or task scope. +# @POST: Session aggregate writes preserve ownership and version semantics. from __future__ import annotations @@ -13,8 +13,7 @@ from typing import Any, List, Optional, cast from sqlalchemy.orm import Session -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.models.dataset_review import ( ClarificationQuestion, ClarificationSession, @@ -32,14 +31,14 @@ from src.models.dataset_review import ( ) from src.services.dataset_review.event_logger import SessionEventLogger -log = MarkerLogger("SessionMutations") +logger = cast(Any, logger) # #region save_profile_and_findings [C:4] [TYPE Function] # @BRIEF Persist profile state and replace validation findings for an owned session in one transaction. -# @PRE session_id belongs to user_id and the supplied profile/findings belong to the same aggregate scope. -# @POST stored profile matches the current session and findings are replaced by the supplied collection. -# @SIDE_EFFECT updates profile rows, deletes stale findings, inserts current findings, and commits the transaction. +# @PRE: session_id belongs to user_id and the supplied profile/findings belong to the same aggregate scope. +# @POST: stored profile matches the current session and findings are replaced by the supplied collection. +# @SIDE_EFFECT: updates profile rows, deletes stale findings, inserts current findings, and commits the transaction. def save_profile_and_findings( db: Session, event_logger: SessionEventLogger, @@ -56,7 +55,7 @@ def save_profile_and_findings( session = get_owned_session(session_id, user_id) if expected_version is not None: require_session_version(session, expected_version) - log.reason("Persisting dataset profile and replacing validation findings", payload={"session_id": session_id, "user_id": user_id, "has_profile": bool(profile), "findings_count": len(findings)}) + logger.reason("Persisting dataset profile and replacing validation findings", extra={"session_id": session_id, "user_id": user_id, "has_profile": bool(profile), "findings_count": len(findings)}) if profile: existing_profile = db.query(DatasetProfile).filter_by(session_id=session_id).first() @@ -70,7 +69,7 @@ def save_profile_and_findings( db.add(finding) commit_session_mutation(session, expected_version=expected_version) - log.reflect("Dataset profile and validation findings committed", payload={"session_id": session.session_id, "user_id": user_id, "findings_count": len(findings)}) + logger.reflect("Dataset profile and validation findings committed", extra={"session_id": session.session_id, "user_id": user_id, "findings_count": len(findings)}) from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository return session @@ -81,9 +80,9 @@ def save_profile_and_findings( # #region save_recovery_state [C:4] [TYPE Function] # @BRIEF Persist imported filters, template variables, and initial execution mappings for one owned session. -# @PRE session_id belongs to user_id. -# @POST Recovery state persisted to database. -# @SIDE_EFFECT Writes to database. +# @PRE: session_id belongs to user_id. +# @POST: Recovery state persisted to database. +# @SIDE_EFFECT: Writes to database. def save_recovery_state( db: Session, get_owned_session, @@ -101,7 +100,7 @@ def save_recovery_state( session = get_owned_session(session_id, user_id) if expected_version is not None: require_session_version(session, expected_version) - log.reason("Persisting dataset review recovery bootstrap state", payload={"session_id": session_id, "user_id": user_id, "imported_filters_count": len(imported_filters), "template_variables_count": len(template_variables), "execution_mappings_count": len(execution_mappings)}) + logger.reason("Persisting dataset review recovery bootstrap state", extra={"session_id": session_id, "user_id": user_id, "imported_filters_count": len(imported_filters), "template_variables_count": len(template_variables), "execution_mappings_count": len(execution_mappings)}) db.query(ExecutionMapping).filter(ExecutionMapping.session_id == session_id).delete() db.query(TemplateVariable).filter(TemplateVariable.session_id == session_id).delete() @@ -119,7 +118,7 @@ def save_recovery_state( db.add(em) commit_session_mutation(session, expected_version=expected_version) - log.reflect("Dataset review recovery bootstrap state committed", payload={"session_id": session.session_id, "user_id": user_id}) + logger.reflect("Dataset review recovery bootstrap state committed", extra={"session_id": session.session_id, "user_id": user_id}) return load_session_detail_fn(session_id, user_id) @@ -128,9 +127,9 @@ def save_recovery_state( # #region save_preview [C:3] [TYPE Function] # @BRIEF Persist a preview snapshot and mark prior session previews stale. -# @PRE session_id belongs to user_id and preview is prepared for the same session aggregate. -# @POST preview is persisted and the session points to the latest preview identifier. -# @SIDE_EFFECT updates prior preview statuses, inserts a preview row, mutates the parent session, and commits. +# @PRE: session_id belongs to user_id and preview is prepared for the same session aggregate. +# @POST: preview is persisted and the session points to the latest preview identifier. +# @SIDE_EFFECT: updates prior preview statuses, inserts a preview row, mutates the parent session, and commits. def save_preview( db: Session, get_owned_session, @@ -146,7 +145,7 @@ def save_preview( session_record = cast(Any, session) if expected_version is not None: require_session_version(session, expected_version) - log.reason("Persisting compiled preview and staling previous preview snapshots", payload={"session_id": session_id, "user_id": user_id}) + logger.reason("Persisting compiled preview and staling previous preview snapshots", extra={"session_id": session_id, "user_id": user_id}) db.query(CompiledPreview).filter(CompiledPreview.session_id == session_id).update({"preview_status": "stale"}) db.add(preview) @@ -154,7 +153,7 @@ def save_preview( session_record.last_preview_id = preview.preview_id commit_session_mutation(session, refresh_targets=[preview], expected_version=expected_version) - log.reflect("Compiled preview committed as latest session preview", payload={"session_id": session.session_id, "preview_id": preview.preview_id}) + logger.reflect("Compiled preview committed as latest session preview", extra={"session_id": session.session_id, "preview_id": preview.preview_id}) return preview @@ -163,9 +162,9 @@ def save_preview( # #region save_run_context [C:3] [TYPE Function] # @BRIEF Persist an immutable launch audit snapshot for an owned session. -# @PRE session_id belongs to user_id and run_context targets the same aggregate. -# @POST run context is persisted and linked as the latest launch snapshot for the session. -# @SIDE_EFFECT inserts a run-context row, mutates the parent session pointer, and commits. +# @PRE: session_id belongs to user_id and run_context targets the same aggregate. +# @POST: run context is persisted and linked as the latest launch snapshot for the session. +# @SIDE_EFFECT: inserts a run-context row, mutates the parent session pointer, and commits. def save_run_context( db: Session, get_owned_session, @@ -181,14 +180,14 @@ def save_run_context( session_record = cast(Any, session) if expected_version is not None: require_session_version(session, expected_version) - log.reason("Persisting dataset run context audit snapshot", payload={"session_id": session_id, "user_id": user_id}) + logger.reason("Persisting dataset run context audit snapshot", extra={"session_id": session_id, "user_id": user_id}) db.add(run_context) db.flush() session_record.last_run_context_id = run_context.run_context_id commit_session_mutation(session, refresh_targets=[run_context], expected_version=expected_version) - log.reflect("Dataset run context committed as latest launch snapshot", payload={"session_id": session.session_id, "run_context_id": run_context.run_context_id}) + logger.reflect("Dataset run context committed as latest launch snapshot", extra={"session_id": session.session_id, "run_context_id": run_context.run_context_id}) return run_context diff --git a/backend/src/services/dataset_review/repositories/session_repository.py b/backend/src/services/dataset_review/repositories/session_repository.py index b61af13c..c99a6030 100644 --- a/backend/src/services/dataset_review/repositories/session_repository.py +++ b/backend/src/services/dataset_review/repositories/session_repository.py @@ -1,26 +1,24 @@ # #region DatasetReviewSessionRepository [C:5] [TYPE Module] # @BRIEF Persist and retrieve dataset review session aggregates, including readiness, findings, semantic decisions, clarification state, previews, and run contexts. -# @LAYER Domain -# @PRE repository operations execute within authenticated request or task scope. -# @POST session aggregate reads are structurally consistent and writes preserve ownership and version semantics. -# @SIDE_EFFECT reads and writes SQLAlchemy-backed session aggregates. -# @DATA_CONTRACT Input[SessionMutation] -> Output[PersistedSessionAggregate] -# @INVARIANT answers, mapping approvals, preview artifacts, and launch snapshots are never attributed to the wrong user or session. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [DatasetProfile] # @RELATION DEPENDS_ON -> [ValidationFinding] # @RELATION DEPENDS_ON -> [CompiledPreview] # @RELATION DISPATCHES -> [SessionRepositoryMutations:Module] -# @RATIONALE Original 627-line file exceeded INV_7 (400-line module limit). Extracted mutation operations into _mutations sub-module. -# @REJECTED Keeping all repository operations in one file because it exceeded the fractal limit. +# @PRE: repository operations execute within authenticated request or task scope. +# @POST: session aggregate reads are structurally consistent and writes preserve ownership and version semantics. +# @SIDE_EFFECT: reads and writes SQLAlchemy-backed session aggregates. +# @DATA_CONTRACT: Input[SessionMutation] -> Output[PersistedSessionAggregate] +# @INVARIANT: answers, mapping approvals, preview artifacts, and launch snapshots are never attributed to the wrong user or session. +# @RATIONALE: Original 627-line file exceeded INV_7 (400-line module limit). Extracted mutation operations into _mutations sub-module. +# @REJECTED: Keeping all repository operations in one file because it exceeded the fractal limit. from datetime import datetime -from typing import Any, Optional, List +from typing import Any, Optional, List, cast from sqlalchemy import or_ from sqlalchemy.orm import Session, joinedload from sqlalchemy.orm.exc import StaleDataError -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope from src.models.dataset_review import ( ClarificationQuestion, ClarificationSession, @@ -36,9 +34,10 @@ from src.models.dataset_review import ( SessionEvent, TemplateVariable, ) +from src.core.logger import belief_scope, logger from src.services.dataset_review.event_logger import SessionEventLogger -log = MarkerLogger("SessionRepository") +logger = cast(Any, logger) # #region DatasetReviewSessionVersionConflictError [C:2] [TYPE Class] @@ -58,93 +57,99 @@ class DatasetReviewSessionVersionConflictError(ValueError): # #region DatasetReviewSessionRepository [C:4] [TYPE Class] # @BRIEF Enforce ownership-scoped persistence and retrieval for dataset review session aggregates. -# @PRE constructor receives a live SQLAlchemy session and callers provide authenticated user scope. -# @POST repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning. -# @SIDE_EFFECT mutates and queries the persistence layer through the injected database session. # @RELATION DEPENDS_ON -> [DatasetReviewSession] # @RELATION DEPENDS_ON -> [SessionEventLogger] +# @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope. +# @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning. +# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session. class DatasetReviewSessionRepository: - # #region init_repo [C:2] [TYPE Function] - # @BRIEF Bind one live SQLAlchemy session to the repository instance. - # @PRE db_session is not None - # @POST Repository instance initialized with valid session + # [DEF:init_repo:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Bind one live SQLAlchemy session to the repository instance. + # @PRE: db_session is not None + # @POST: Repository instance initialized with valid session def __init__(self, db: Session): self.db = db self.event_logger = SessionEventLogger(db) - # #endregion init_repo + # [/DEF:init_repo:Function] - # #region get_owned_session [C:3] [TYPE Function] - # @BRIEF Resolve one owner-scoped dataset review session for mutation paths. - # @PRE session_id and user_id are non-empty identifiers from the authenticated ownership scope. - # @POST returns the owned session or raises a deterministic access error. + # [DEF:get_owned_session:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Resolve one owner-scoped dataset review session for mutation paths. + # @PRE: session_id and user_id are non-empty identifiers from the authenticated ownership scope. + # @POST: returns the owned session or raises a deterministic access error. def _get_owned_session(self, session_id: str, user_id: str) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.get_owned_session"): - log.reason("Resolving owner-scoped dataset review session", payload={"session_id": session_id, "user_id": user_id}) + logger.reason("Resolving owner-scoped dataset review session", extra={"session_id": session_id, "user_id": user_id}) session = ( self.db.query(DatasetReviewSession) .filter(DatasetReviewSession.session_id == session_id, DatasetReviewSession.user_id == user_id) .first() ) if not session: - log.explore("Owner-scoped dataset review session lookup failed", payload={"session_id": session_id, "user_id": user_id}, error="Session not found or access denied") + logger.explore("Owner-scoped dataset review session lookup failed", extra={"session_id": session_id, "user_id": user_id}) raise ValueError("Session not found or access denied") - log.reflect("Owner-scoped dataset review session resolved", payload={"session_id": session.session_id}) + logger.reflect("Owner-scoped dataset review session resolved", extra={"session_id": session.session_id}) return session - # #endregion get_owned_session + # [/DEF:get_owned_session:Function] - # #region create_sess [C:3] [TYPE Function] - # @BRIEF Persist an initial dataset review session shell. - # @POST session is committed, refreshed, and returned with persisted identifiers. + # [DEF:create_sess:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persist an initial dataset review session shell. + # @POST: session is committed, refreshed, and returned with persisted identifiers. def create_session(self, session: DatasetReviewSession) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.create_session"): - log.reason("Persisting dataset review session shell", payload={"user_id": session.user_id, "environment_id": session.environment_id}) + logger.reason("Persisting dataset review session shell", extra={"user_id": session.user_id, "environment_id": session.environment_id}) self.db.add(session) self.db.commit() self.db.refresh(session) - log.reflect("Dataset review session shell persisted", payload={"session_id": session.session_id}) + logger.reflect("Dataset review session shell persisted", extra={"session_id": session.session_id}) return session - # #endregion create_sess + # [/DEF:create_sess:Function] - # #region require_session_version [C:3] [TYPE Function] - # @BRIEF Enforce optimistic-lock version matching before a session mutation is persisted. - # @POST returns the same session when versions match; otherwise raises deterministic conflict error. + # [DEF:require_session_version:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Enforce optimistic-lock version matching before a session mutation is persisted. + # @POST: returns the same session when versions match; otherwise raises deterministic conflict error. def require_session_version(self, session: DatasetReviewSession, expected_version: int) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.require_session_version"): actual_version = int(getattr(session, "version", 0) or 0) - log.reason("Checking optimistic-lock version", payload={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) + logger.reason("Checking optimistic-lock version", extra={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) if actual_version != expected_version: - log.explore("Rejected mutation due to stale session version", payload={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}, error="Optimistic lock version mismatch") + logger.explore("Rejected mutation due to stale session version", extra={"session_id": session.session_id, "expected_version": expected_version, "actual_version": actual_version}) raise DatasetReviewSessionVersionConflictError(str(session.session_id), expected_version, actual_version) - log.reflect("Optimistic-lock version accepted", payload={"session_id": session.session_id, "version": actual_version}) + logger.reflect("Optimistic-lock version accepted", extra={"session_id": session.session_id, "version": actual_version}) return session - # #endregion require_session_version + # [/DEF:require_session_version:Function] - # #region bump_session_version [C:2] [TYPE Function] - # @BRIEF Increment optimistic-lock version after a successful session mutation is assembled. - # @POST session version increments monotonically. + # [DEF:bump_session_version:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Increment optimistic-lock version after a successful session mutation is assembled. + # @POST: session version increments monotonically. def bump_session_version(self, session: DatasetReviewSession) -> int: with belief_scope("DatasetReviewSessionRepository.bump_session_version"): next_version = int(getattr(session, "version", 0) or 0) + 1 setattr(session, "version", next_version) session.last_activity_at = datetime.utcnow() - log.reflect("Prepared incremented session version", payload={"session_id": session.session_id, "version": next_version}) + logger.reflect("Prepared incremented session version", extra={"session_id": session.session_id, "version": next_version}) return next_version - # #endregion bump_session_version + # [/DEF:bump_session_version:Function] - # #region commit_session_mutation [C:4] [TYPE Function] - # @BRIEF Commit one prepared session mutation and translate stale writes into deterministic conflicts. - # @POST session mutation is committed with one version increment or a deterministic conflict error is raised. + # [DEF:commit_session_mutation:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Commit one prepared session mutation and translate stale writes into deterministic conflicts. + # @POST: session mutation is committed with one version increment or a deterministic conflict error is raised. def commit_session_mutation( self, session: DatasetReviewSession, *, refresh_targets: Optional[List[Any]] = None, expected_version: Optional[int] = None, ) -> DatasetReviewSession: with belief_scope("DatasetReviewSessionRepository.commit_session_mutation"): observed_version = int(expected_version if expected_version is not None else getattr(session, "version", 0) or 0) - log.reason("Committing session mutation with optimistic lock", payload={"session_id": session.session_id, "observed_version": observed_version}) + logger.reason("Committing session mutation with optimistic lock", extra={"session_id": session.session_id, "observed_version": observed_version}) self.bump_session_version(session) try: self.db.commit() @@ -152,22 +157,23 @@ class DatasetReviewSessionRepository: self.db.rollback() actual_version_row = self.db.query(DatasetReviewSession.version).filter(DatasetReviewSession.session_id == session.session_id).first() actual_version = int(actual_version_row[0] or 0) if actual_version_row else 0 - log.explore("Session commit rejected by optimistic lock", payload={"session_id": session.session_id, "expected_version": observed_version, "actual_version": actual_version}, error="StaleDataError on session commit") + logger.explore("Session commit rejected by optimistic lock", extra={"session_id": session.session_id, "expected_version": observed_version, "actual_version": actual_version}) raise DatasetReviewSessionVersionConflictError(session.session_id, observed_version, actual_version) from exc self.db.refresh(session) for target in refresh_targets or []: self.db.refresh(target) - log.reflect("Session mutation committed", payload={"session_id": session.session_id, "version": getattr(session, "version", None)}) + logger.reflect("Session mutation committed", extra={"session_id": session.session_id, "version": getattr(session, "version", None)}) return session - # #endregion commit_session_mutation + # [/DEF:commit_session_mutation:Function] - # #region load_detail [C:3] [TYPE Function] - # @BRIEF Return the full session aggregate for API and frontend resume flows. - # @POST Returns SessionDetail with all fields populated or None. + # [DEF:load_detail:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Return the full session aggregate for API and frontend resume flows. + # @POST: Returns SessionDetail with all fields populated or None. def load_session_detail(self, session_id: str, user_id: str) -> Optional[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.load_session_detail"): - log.reason("Loading dataset review session detail", payload={"session_id": session_id, "user_id": user_id}) + logger.reason("Loading dataset review session detail", extra={"session_id": session_id, "user_id": user_id}) session = ( self.db.query(DatasetReviewSession) .outerjoin(SessionCollaborator, DatasetReviewSession.session_id == SessionCollaborator.session_id) @@ -190,14 +196,15 @@ class DatasetReviewSessionRepository: .filter(or_(DatasetReviewSession.user_id == user_id, SessionCollaborator.user_id == user_id)) .first() ) - log.reflect("Session detail lookup completed", payload={"session_id": session_id, "found": bool(session)}) + logger.reflect("Session detail lookup completed", extra={"session_id": session_id, "found": bool(session)}) return session - # #endregion load_detail + # [/DEF:load_detail:Function] - # #region save_profile_and_findings [C:4] [TYPE Function] - # @BRIEF Persist profile state and replace validation findings for an owned session. - # @POST stored profile matches the current session and findings are replaced. + # [DEF:save_profile_and_findings:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Persist profile state and replace validation findings for an owned session. + # @POST: stored profile matches the current session and findings are replaced. def save_profile_and_findings( self, session_id: str, user_id: str, profile: DatasetProfile, findings: List[ValidationFinding], expected_version: Optional[int] = None, ) -> DatasetReviewSession: @@ -207,10 +214,11 @@ class DatasetReviewSessionRepository: self.commit_session_mutation, session_id, user_id, profile, findings, expected_version, ) - # #endregion save_profile_and_findings + # [/DEF:save_profile_and_findings:Function] - # #region save_recovery_state [C:3] [TYPE Function] - # @BRIEF Persist imported filters, template variables, and initial execution mappings. + # [DEF:save_recovery_state:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persist imported filters, template variables, and initial execution mappings. def save_recovery_state( self, session_id: str, user_id: str, imported_filters: List[ImportedFilter], template_variables: List[TemplateVariable], execution_mappings: List[ExecutionMapping], @@ -223,10 +231,11 @@ class DatasetReviewSessionRepository: session_id, user_id, imported_filters, template_variables, execution_mappings, expected_version, ) - # #endregion save_recovery_state + # [/DEF:save_recovery_state:Function] - # #region save_preview [C:3] [TYPE Function] - # @BRIEF Persist a preview snapshot and mark prior session previews stale. + # [DEF:save_preview:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persist a preview snapshot and mark prior session previews stale. def save_preview( self, session_id: str, user_id: str, preview: CompiledPreview, expected_version: Optional[int] = None, ) -> CompiledPreview: @@ -236,10 +245,11 @@ class DatasetReviewSessionRepository: self.commit_session_mutation, session_id, user_id, preview, expected_version, ) - # #endregion save_preview + # [/DEF:save_preview:Function] - # #region save_run_context [C:3] [TYPE Function] - # @BRIEF Persist an immutable launch audit snapshot for an owned session. + # [DEF:save_run_context:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Persist an immutable launch audit snapshot for an owned session. def save_run_context( self, session_id: str, user_id: str, run_context: DatasetRunContext, expected_version: Optional[int] = None, ) -> DatasetRunContext: @@ -249,23 +259,26 @@ class DatasetReviewSessionRepository: self.commit_session_mutation, session_id, user_id, run_context, expected_version, ) - # #endregion save_run_context + # [/DEF:save_run_context:Function] - # #region list_user_sess [C:2] [TYPE Function] - # @BRIEF List review sessions owned by a specific user ordered by most recent update. + # [DEF:list_user_sess:Function] + # @COMPLEXITY: 2 + # @PURPOSE: List review sessions owned by a specific user ordered by most recent update. def list_sessions_for_user(self, user_id: str) -> List[DatasetReviewSession]: with belief_scope("DatasetReviewSessionRepository.list_sessions_for_user"): - log.reason("Listing dataset review sessions for owner scope", payload={"user_id": user_id}) + logger.reason("Listing dataset review sessions for owner scope", extra={"user_id": user_id}) sessions = ( self.db.query(DatasetReviewSession) .filter(DatasetReviewSession.user_id == user_id) .order_by(DatasetReviewSession.updated_at.desc()) .all() ) - log.reflect("Session list assembled", payload={"user_id": user_id, "session_count": len(sessions)}) + logger.reflect("Session list assembled", extra={"user_id": user_id, "session_count": len(sessions)}) return sessions - # #endregion list_user_sess + # [/DEF:list_user_sess:Function] # #endregion DatasetReviewSessionRepository + +# #endregion DatasetReviewSessionRepository diff --git a/backend/src/services/dataset_review/semantic_resolver.py b/backend/src/services/dataset_review/semantic_resolver.py index 6c144588..f2e68b14 100644 --- a/backend/src/services/dataset_review/semantic_resolver.py +++ b/backend/src/services/dataset_review/semantic_resolver.py @@ -1,14 +1,14 @@ # #region SemanticSourceResolver [C:4] [TYPE Module] [SEMANTICS dataset_review, semantic_resolution, dictionary, trusted_sources, ranking] # @BRIEF Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback. -# @LAYER Domain -# @PRE selected source and target field set must be known. -# @POST candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable. -# @SIDE_EFFECT may create conflict findings and semantic candidate records. -# @INVARIANT Manual overrides are never silently replaced by imported, inferred, or AI-generated values. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [SemanticSource] # @RELATION DEPENDS_ON -> [SemanticFieldEntry] # @RELATION DEPENDS_ON -> [SemanticCandidate] +# @PRE: selected source and target field set must be known. +# @POST: candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable. +# @SIDE_EFFECT: may create conflict findings and semantic candidate records. +# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values. from __future__ import annotations @@ -17,8 +17,7 @@ from dataclasses import dataclass, field from difflib import SequenceMatcher from typing import Any, Dict, Iterable, List, Mapping, Optional -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope +from src.core.logger import belief_scope, logger from src.models.dataset_review import ( CandidateMatchType, CandidateStatus, @@ -27,8 +26,6 @@ from src.models.dataset_review import ( ) # #endregion imports -log = MarkerLogger("SemanticSourceResolver") - # #region DictionaryResolutionResult [C:2] [TYPE Class] # @BRIEF Carries field-level dictionary resolution output with explicit review and partial-recovery state. @@ -43,26 +40,28 @@ class DictionaryResolutionResult: # #region SemanticSourceResolver [C:4] [TYPE Class] # @BRIEF Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering. -# @PRE source payload and target field collection are provided by the caller. -# @POST result contains confidence-ranked candidates and does not overwrite manual locks implicitly. -# @SIDE_EFFECT emits semantic trace logs for ranking and fallback decisions. # @RELATION DEPENDS_ON -> [SemanticFieldEntry] # @RELATION DEPENDS_ON -> [SemanticCandidate] +# @PRE: source payload and target field collection are provided by the caller. +# @POST: result contains confidence-ranked candidates and does not overwrite manual locks implicitly. +# @SIDE_EFFECT: emits semantic trace logs for ranking and fallback decisions. class SemanticSourceResolver: - # #region resolve_from_file [C:2] [TYPE Function] - # @BRIEF Normalize uploaded semantic file records into field-level candidates. + # [DEF:resolve_from_file:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize uploaded semantic file records into field-level candidates. def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult: return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "uploaded_file")) - # #endregion resolve_from_file + # [/DEF:resolve_from_file:Function] - # #region resolve_from_dictionary [C:4] [TYPE Function] - # @BRIEF Resolve candidates from connected tabular dictionary sources. - # @PRE dictionary source exists and fields contain stable field_name values. - # @POST returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit. - # @SIDE_EFFECT emits belief-state logs describing trusted-match and partial-recovery outcomes. - # @DATA_CONTRACT Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult] - # @RELATION DEPENDS_ON -> [SemanticFieldEntry] - # @RELATION DEPENDS_ON -> [SemanticCandidate] + # [DEF:resolve_from_dictionary:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Resolve candidates from connected tabular dictionary sources. + # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] + # @RELATION: [DEPENDS_ON] ->[SemanticCandidate] + # @PRE: dictionary source exists and fields contain stable field_name values. + # @POST: returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit. + # @SIDE_EFFECT: emits belief-state logs describing trusted-match and partial-recovery outcomes. + # @DATA_CONTRACT: Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult] def resolve_from_dictionary( self, source_payload: Mapping[str, Any], @@ -73,16 +72,19 @@ class SemanticSourceResolver: dictionary_rows = source_payload.get("rows") if not source_ref: - log.explore("Dictionary semantic source is missing source_ref", error="No source_ref in dictionary source") + logger.explore("Dictionary semantic source is missing source_ref") raise ValueError("Dictionary semantic source must include source_ref") if not isinstance(dictionary_rows, list) or not dictionary_rows: - log.explore("Dictionary semantic source has no usable rows", error="Dictionary rows is empty or not a list", payload={"source_ref": source_ref}) + logger.explore( + "Dictionary semantic source has no usable rows", + extra={"source_ref": source_ref}, + ) raise ValueError("Dictionary semantic source must include non-empty rows") - log.reason( + logger.reason( "Resolving semantics from trusted dictionary source", - payload={"source_ref": source_ref, "row_count": len(dictionary_rows)}, + extra={"source_ref": source_ref, "row_count": len(dictionary_rows)}, ) normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)] @@ -102,9 +104,9 @@ class SemanticSourceResolver: is_locked = bool(raw_field.get("is_locked")) if is_locked: - log.reason( + logger.reason( "Preserving manual lock during dictionary resolution", - payload={"field_name": field_name}, + extra={"field_name": field_name}, ) resolved_fields.append( { @@ -124,9 +126,9 @@ class SemanticSourceResolver: candidates: List[Dict[str, Any]] = [] if exact_match is not None: - log.reason( + logger.reason( "Resolved exact dictionary match", - payload={"field_name": field_name, "source_ref": source_ref}, + extra={"field_name": field_name, "source_ref": source_ref}, ) candidates.append( self._build_candidate_payload( @@ -162,7 +164,10 @@ class SemanticSourceResolver: "status": "unresolved", } ) - log.explore("No trusted dictionary match found for field", error="No dictionary match found for field", payload={"field_name": field_name, "source_ref": source_ref}) + logger.explore( + "No trusted dictionary match found for field", + extra={"field_name": field_name, "source_ref": source_ref}, + ) continue ranked_candidates = self.rank_candidates(candidates) @@ -194,9 +199,9 @@ class SemanticSourceResolver: unresolved_fields=unresolved_fields, partial_recovery=bool(unresolved_fields), ) - log.reflect( + logger.reflect( "Dictionary resolution completed", - payload={ + extra={ "source_ref": source_ref, "resolved_fields": len(resolved_fields), "unresolved_fields": len(unresolved_fields), @@ -204,21 +209,23 @@ class SemanticSourceResolver: }, ) return result - # #endregion resolve_from_dictionary + # [/DEF:resolve_from_dictionary:Function] - # #region resolve_from_reference_dataset [C:2] [TYPE Function] - # @BRIEF Reuse semantic metadata from trusted Superset datasets. + # [DEF:resolve_from_reference_dataset:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Reuse semantic metadata from trusted Superset datasets. def resolve_from_reference_dataset( self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]], ) -> DictionaryResolutionResult: return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "reference_dataset")) - # #endregion resolve_from_reference_dataset + # [/DEF:resolve_from_reference_dataset:Function] - # #region rank_candidates [C:2] [TYPE Function] - # @BRIEF Apply confidence ordering and determine best candidate per field. - # @RELATION DEPENDS_ON -> [SemanticCandidate] + # [DEF:rank_candidates:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Apply confidence ordering and determine best candidate per field. + # @RELATION: [DEPENDS_ON] ->[SemanticCandidate] def rank_candidates(self, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]: ranked = sorted( candidates, @@ -231,30 +238,33 @@ class SemanticSourceResolver: for index, candidate in enumerate(ranked, start=1): candidate["candidate_rank"] = index return ranked - # #endregion rank_candidates + # [/DEF:rank_candidates:Function] - # #region detect_conflicts [C:2] [TYPE Function] - # @BRIEF Mark competing candidate sets that require explicit user review. + # [DEF:detect_conflicts:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Mark competing candidate sets that require explicit user review. def detect_conflicts(self, candidates: List[Dict[str, Any]]) -> bool: return len(candidates) > 1 - # #endregion detect_conflicts + # [/DEF:detect_conflicts:Function] - # #region apply_field_decision [C:2] [TYPE Function] - # @BRIEF Accept, reject, or manually override a field-level semantic value. + # [DEF:apply_field_decision:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Accept, reject, or manually override a field-level semantic value. def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> Dict[str, Any]: merged = dict(field_state) merged.update(decision) return merged - # #endregion apply_field_decision + # [/DEF:apply_field_decision:Function] - # #region propagate_source_version_update [C:4] [TYPE Function] - # @BRIEF Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values. - # @PRE source is persisted and fields belong to the same session aggregate. - # @POST unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched. - # @SIDE_EFFECT mutates in-memory field state for the caller to persist. - # @DATA_CONTRACT Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]] - # @RELATION DEPENDS_ON -> [SemanticSource] - # @RELATION DEPENDS_ON -> [SemanticFieldEntry] + # [DEF:propagate_source_version_update:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values. + # @RELATION: [DEPENDS_ON] ->[SemanticSource] + # @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry] + # @PRE: source is persisted and fields belong to the same session aggregate. + # @POST: unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched. + # @SIDE_EFFECT: mutates in-memory field state for the caller to persist. + # @DATA_CONTRACT: Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]] def propagate_source_version_update( self, source: SemanticSource, @@ -264,7 +274,10 @@ class SemanticSourceResolver: source_id = str(source.source_id or "").strip() source_version = str(source.source_version or "").strip() if not source_id or not source_version: - log.explore("Semantic source version propagation rejected due to incomplete source metadata", error="Source metadata incomplete (missing source_id or source_version)", payload={"source_id": source_id, "source_version": source_version}) + logger.explore( + "Semantic source version propagation rejected due to incomplete source metadata", + extra={"source_id": source_id, "source_version": source_version}, + ) raise ValueError("Semantic source must provide source_id and source_version") propagated = 0 @@ -283,9 +296,9 @@ class SemanticSourceResolver: field.has_conflict = bool(getattr(field, "has_conflict", False)) propagated += 1 - log.reflect( + logger.reflect( "Semantic source version propagation completed", - payload={ + extra={ "source_id": source_id, "source_version": source_version, "propagated": propagated, @@ -298,10 +311,11 @@ class SemanticSourceResolver: "preserved_locked": preserved_locked, "untouched": untouched, } - # #endregion propagate_source_version_update + # [/DEF:propagate_source_version_update:Function] - # #region _normalize_dictionary_row [C:2] [TYPE Function] - # @BRIEF Normalize one dictionary row into a consistent lookup structure. + # [DEF:_normalize_dictionary_row:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize one dictionary row into a consistent lookup structure. def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> Dict[str, Any]: field_name = ( row.get("field_name") @@ -317,10 +331,11 @@ class SemanticSourceResolver: "description": row.get("description"), "display_format": row.get("display_format") or row.get("format"), } - # #endregion _normalize_dictionary_row + # [/DEF:_normalize_dictionary_row:Function] - # #region _find_fuzzy_matches [C:2] [TYPE Function] - # @BRIEF Produce confidence-scored fuzzy matches while keeping them reviewable. + # [DEF:_find_fuzzy_matches:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable. def _find_fuzzy_matches(self, field_name: str, rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: normalized_target = self._normalize_key(field_name) fuzzy_matches: List[Dict[str, Any]] = [] @@ -334,10 +349,11 @@ class SemanticSourceResolver: fuzzy_matches.append({"row": row, "score": round(score, 3)}) fuzzy_matches.sort(key=lambda item: item["score"], reverse=True) return fuzzy_matches[:3] - # #endregion _find_fuzzy_matches + # [/DEF:_find_fuzzy_matches:Function] - # #region _build_candidate_payload [C:2] [TYPE Function] - # @BRIEF Project normalized dictionary rows into semantic candidate payloads. + # [DEF:_build_candidate_payload:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Project normalized dictionary rows into semantic candidate payloads. def _build_candidate_payload( self, rank: int, @@ -354,10 +370,11 @@ class SemanticSourceResolver: "proposed_display_format": row.get("display_format"), "status": CandidateStatus.PROPOSED.value, } - # #endregion _build_candidate_payload + # [/DEF:_build_candidate_payload:Function] - # #region _match_priority [C:2] [TYPE Function] - # @BRIEF Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention. + # [DEF:_match_priority:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention. def _match_priority(self, match_type: Optional[str]) -> int: priority = { CandidateMatchType.EXACT.value: 0, @@ -366,11 +383,14 @@ class SemanticSourceResolver: CandidateMatchType.GENERATED.value: 3, } return priority.get(str(match_type or ""), 99) - # #endregion _match_priority + # [/DEF:_match_priority:Function] - # #region _normalize_key [C:2] [TYPE Function] - # @BRIEF Normalize field identifiers for stable exact/fuzzy comparisons. + # [DEF:_normalize_key:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons. def _normalize_key(self, value: str) -> str: return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch == "_") - # #endregion _normalize_key + # [/DEF:_normalize_key:Function] # #endregion SemanticSourceResolver + +# #endregion SemanticSourceResolver \ No newline at end of file diff --git a/backend/src/services/git/__init__.py b/backend/src/services/git/__init__.py index 20a2f78e..595e65f9 100644 --- a/backend/src/services/git/__init__.py +++ b/backend/src/services/git/__init__.py @@ -1,6 +1,6 @@ # #region GitServiceModule [C:3] [TYPE Module] [SEMANTICS git, service, decomposition, mixin, composition] +# @LAYER: Infra # @BRIEF Composed GitService via multiple inheritance from domain-specific mixins. -# @LAYER Infra # @RELATION DEPENDS_ON -> [GitServiceBase] # @RELATION DEPENDS_ON -> [GitServiceBranchMixin] # @RELATION DEPENDS_ON -> [GitServiceSyncMixin] @@ -10,8 +10,8 @@ # @RELATION DEPENDS_ON -> [GitServiceGiteaMixin] # @RELATION DEPENDS_ON -> [GitServiceGithubMixin] # @RELATION DEPENDS_ON -> [GitServiceGitlabMixin] -# @RATIONALE Decomposed from monolithic git_service.py (2101 lines) into # +# @RATIONALE: Decomposed from monolithic git_service.py (2101 lines) into # domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class # preserves the original public API surface — all consumers continue to import # `from src.services.git_service import GitService` without changes. @@ -32,15 +32,15 @@ __all__ = ["GitService"] # #region GitService [C:3] [TYPE Class] # @BRIEF Composed GitService class providing all Git operations — init, clone, branches, commits, push, pull, # merge, status, diff, history, and provider API operations (Gitea, GitHub, GitLab). -# @RELATION: INHERITS -> [GitServiceBase] -# @RELATION: INHERITS -> [GitServiceBranchMixin] -# @RELATION: INHERITS -> [GitServiceSyncMixin] -# @RELATION: INHERITS -> [GitServiceStatusMixin] -# @RELATION: INHERITS -> [GitServiceMergeMixin] -# @RELATION: INHERITS -> [GitServiceUrlMixin] -# @RELATION: INHERITS -> [GitServiceGiteaMixin] -# @RELATION: INHERITS -> [GitServiceGithubMixin] -# @RELATION: INHERITS -> [GitServiceGitlabMixin] +# @RELATION INHERITS -> [GitServiceBase] +# @RELATION INHERITS -> [GitServiceBranchMixin] +# @RELATION INHERITS -> [GitServiceSyncMixin] +# @RELATION INHERITS -> [GitServiceStatusMixin] +# @RELATION INHERITS -> [GitServiceMergeMixin] +# @RELATION INHERITS -> [GitServiceUrlMixin] +# @RELATION INHERITS -> [GitServiceGiteaMixin] +# @RELATION INHERITS -> [GitServiceGithubMixin] +# @RELATION INHERITS -> [GitServiceGitlabMixin] class GitService( GitServiceGiteaMixin, GitServiceGithubMixin, diff --git a/backend/src/services/git/_base.py b/backend/src/services/git/_base.py index c040bb0c..63b3d186 100644 --- a/backend/src/services/git/_base.py +++ b/backend/src/services/git/_base.py @@ -1,6 +1,6 @@ # #region GitServiceBase [C:3] [TYPE Module] [SEMANTICS git, service, repository, version_control, initialization] +# @LAYER: Infra # @BRIEF Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration. -# @LAYER Infra # @RELATION INHERITED_BY -> [GitService] # @RELATION DEPENDS_ON -> [SessionLocal] # @RELATION DEPENDS_ON -> [AppConfigRecord] @@ -14,10 +14,7 @@ from typing import Any, Dict, List, Optional from git import Repo from git.exc import InvalidGitRepositoryError, NoSuchPathError from fastapi import HTTPException -from src.core.logger import belief_scope -from src.core.cot_logger import MarkerLogger - -log = MarkerLogger("GitBase") +from src.core.logger import logger, belief_scope from src.models.git import GitRepository from src.models.config import AppConfigRecord from src.core.database import SessionLocal @@ -26,8 +23,8 @@ from src.core.database import SessionLocal # #region GitServiceBase [C:3] [TYPE Class] # @BRIEF Base class for GitService providing initialization, path resolution, repository lifecycle, and identity. class GitServiceBase: - # #region GitService_init [TYPE Function] - # @BRIEF Initializes the GitService with a base path for repositories. + # [DEF:GitService_init:Function] + # @PURPOSE: Initializes the GitService with a base path for repositories. # @PARAM: base_path (str) - Root directory for all Git clones. # @PRE: base_path is a valid string path. # @POST: GitService is initialized; base_path directory exists. @@ -38,12 +35,12 @@ class GitServiceBase: self._uses_default_base_path = base_path == "git_repos" self.base_path = self._resolve_base_path(base_path) self._ensure_base_path_exists() - # #endregion GitService_init + # [/DEF:GitService_init:Function] - # #region _ensure_base_path_exists [TYPE Function] - # @BRIEF Ensure the repositories root directory exists and is a directory. - # @PRE self.base_path is resolved to filesystem path. - # @POST self.base_path exists as directory or raises ValueError. + # [DEF:_ensure_base_path_exists:Function] + # @PURPOSE: Ensure the repositories root directory exists and is a directory. + # @PRE: self.base_path is resolved to filesystem path. + # @POST: self.base_path exists as directory or raises ValueError. def _ensure_base_path_exists(self) -> None: base = Path(self.base_path) if base.exists() and not base.is_dir(): @@ -51,15 +48,17 @@ class GitServiceBase: try: base.mkdir(parents=True, exist_ok=True) except (PermissionError, OSError) as e: - log.explore("Cannot create Git repositories base path", payload={"base_path": self.base_path}, error=str(e)) + logger.warning( + f"[_ensure_base_path_exists][Coherence:Failed] Cannot create Git repositories base path: {self.base_path}. Error: {e}" + ) raise ValueError(f"Cannot create Git repositories base path: {self.base_path}. {e}") - # #endregion _ensure_base_path_exists + # [/DEF:_ensure_base_path_exists:Function] - # #region _resolve_base_path [TYPE Function] - # @BRIEF Resolve base repository directory from explicit argument or global storage settings. - # @PRE base_path is a string path. - # @POST Returns absolute path for Git repositories root. - # @RETURN str + # [DEF:_resolve_base_path:Function] + # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings. + # @PRE: base_path is a string path. + # @POST: Returns absolute path for Git repositories root. + # @RETURN: str def _resolve_base_path(self, base_path: str) -> str: backend_root = Path(__file__).parents[3] fallback_path = str((backend_root / base_path).resolve()) @@ -86,25 +85,25 @@ class GitServiceBase: return str(repo_root.resolve()) return str((root / repo_root).resolve()) except Exception as e: - log.explore("Falling back to default base path", error=str(e)) + logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}") return fallback_path - # #endregion _resolve_base_path + # [/DEF:_resolve_base_path:Function] - # #region _normalize_repo_key [TYPE Function] - # @BRIEF Convert user/dashboard-provided key to safe filesystem directory name. - # @PRE repo_key can be None/empty. - # @POST Returns normalized non-empty key. - # @RETURN str + # [DEF:_normalize_repo_key:Function] + # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name. + # @PRE: repo_key can be None/empty. + # @POST: Returns normalized non-empty key. + # @RETURN: str def _normalize_repo_key(self, repo_key: Optional[str]) -> str: raw_key = str(repo_key or "").strip().lower() normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-") return normalized or "dashboard" - # #endregion _normalize_repo_key + # [/DEF:_normalize_repo_key:Function] - # #region _update_repo_local_path [TYPE Function] - # @BRIEF Persist repository local_path in GitRepository table when record exists. - # @PRE dashboard_id is valid integer. - # @POST local_path is updated for existing record. + # [DEF:_update_repo_local_path:Function] + # @PURPOSE: Persist repository local_path in GitRepository table when record exists. + # @PRE: dashboard_id is valid integer. + # @POST: local_path is updated for existing record. def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None: try: session = SessionLocal() @@ -120,21 +119,21 @@ class GitServiceBase: finally: session.close() except Exception as e: - log.explore("Failed to update repo local path", error=str(e)) - # #endregion _update_repo_local_path + logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}") + # [/DEF:_update_repo_local_path:Function] - # #region _migrate_repo_directory [TYPE Function] - # @BRIEF Move legacy repository directory to target path and sync DB metadata. - # @PRE source_path exists. - # @POST Repository content available at target_path. - # @RETURN str + # [DEF:_migrate_repo_directory:Function] + # @PURPOSE: Move legacy repository directory to target path and sync DB metadata. + # @PRE: source_path exists. + # @POST: Repository content available at target_path. + # @RETURN: str def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str: source_abs = os.path.abspath(source_path) target_abs = os.path.abspath(target_path) if source_abs == target_abs: return source_abs if os.path.exists(target_abs): - log.explore(f"Target already exists, keeping source path: {target_abs}", error="Target path exists") + logger.warning(f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}") return source_abs Path(target_abs).parent.mkdir(parents=True, exist_ok=True) try: @@ -142,12 +141,14 @@ class GitServiceBase: except OSError: shutil.move(source_abs, target_abs) self._update_repo_local_path(dashboard_id, target_abs) - log.reflect(f"Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}") + logger.info( + f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}" + ) return target_abs - # #endregion _migrate_repo_directory + # [/DEF:_migrate_repo_directory:Function] - # #region _get_repo_path [TYPE Function] - # @BRIEF Resolves the local filesystem path for a dashboard's repository. + # [DEF:_get_repo_path:Function] + # @PURPOSE: Resolves the local filesystem path for a dashboard's repository. # @PARAM: dashboard_id (int) # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent. # @PRE: dashboard_id is an integer. @@ -183,17 +184,17 @@ class GitServiceBase: return self._migrate_repo_directory(dashboard_id, db_path, target_path) return db_path except Exception as e: - log.explore("Could not resolve local_path from DB", error=str(e)) + logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}") legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id)) if os.path.exists(legacy_id_path) and not os.path.exists(target_path): return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path) if os.path.exists(target_path): self._update_repo_local_path(dashboard_id, target_path) return target_path - # #endregion _get_repo_path + # [/DEF:_get_repo_path:Function] - # #region init_repo [TYPE Function] - # @BRIEF Initialize or clone a repository for a dashboard. + # [DEF:init_repo:Function] + # @PURPOSE: Initialize or clone a repository for a dashboard. # @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]). # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided. # @POST: Repository is cloned or opened at the local path. @@ -209,11 +210,11 @@ class GitServiceBase: else: auth_url = remote_url if os.path.exists(repo_path): - log.reason(f"Opening existing repo at {repo_path}") + logger.info(f"[init_repo][Action] Opening existing repo at {repo_path}") try: repo = Repo(repo_path) except (InvalidGitRepositoryError, NoSuchPathError): - log.explore(f"Existing path is not a Git repository, recreating: {repo_path}", error="Not a git repository") + logger.warning(f"[init_repo][Action] Existing path is not a Git repository, recreating: {repo_path}") stale_path = Path(repo_path) if stale_path.exists(): shutil.rmtree(stale_path, ignore_errors=True) @@ -225,16 +226,16 @@ class GitServiceBase: repo = Repo.clone_from(auth_url, repo_path) self._ensure_gitflow_branches(repo, dashboard_id) return repo - log.reason(f"Cloning {remote_url} to {repo_path}") + logger.info(f"[init_repo][Action] Cloning {remote_url} to {repo_path}") repo = Repo.clone_from(auth_url, repo_path) self._ensure_gitflow_branches(repo, dashboard_id) return repo - # #endregion init_repo + # [/DEF:init_repo:Function] - # #region delete_repo [TYPE Function] - # @BRIEF Remove local repository and DB binding for a dashboard. - # @PRE dashboard_id is a valid integer. - # @POST Local path is deleted when present and GitRepository row is removed. + # [DEF:delete_repo:Function] + # @PURPOSE: Remove local repository and DB binding for a dashboard. + # @PRE: dashboard_id is a valid integer. + # @POST: Local path is deleted when present and GitRepository row is removed. def delete_repo(self, dashboard_id: int) -> None: with belief_scope("GitService.delete_repo"): repo_path = self._get_repo_path(dashboard_id) @@ -267,34 +268,34 @@ class GitServiceBase: raise except Exception as e: session.rollback() - log.explore(f"Failed to delete repository for dashboard {dashboard_id}", error=str(e)) + logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}") raise HTTPException(status_code=500, detail=f"Failed to delete repository: {str(e)}") finally: session.close() - # #endregion delete_repo + # [/DEF:delete_repo:Function] - # #region get_repo [TYPE Function] - # @BRIEF Get Repo object for a dashboard. - # @PRE Repository must exist on disk for the given dashboard_id. - # @POST Returns a GitPython Repo instance for the dashboard. - # @RETURN Repo + # [DEF:get_repo:Function] + # @PURPOSE: Get Repo object for a dashboard. + # @PRE: Repository must exist on disk for the given dashboard_id. + # @POST: Returns a GitPython Repo instance for the dashboard. + # @RETURN: Repo def get_repo(self, dashboard_id: int) -> Repo: with belief_scope("GitService.get_repo"): repo_path = self._get_repo_path(dashboard_id) if not os.path.exists(repo_path): - log.explore(f"Repository for dashboard {dashboard_id} does not exist", error="Repository not found") + logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist") raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found") try: return Repo(repo_path) except Exception as e: - log.explore(f"Failed to open repository at {repo_path}", error=str(e)) + logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}") raise HTTPException(status_code=500, detail="Failed to open local Git repository") - # #endregion get_repo + # [/DEF:get_repo:Function] - # #region configure_identity [TYPE Function] - # @BRIEF Configure repository-local Git committer identity for user-scoped operations. - # @PRE dashboard_id repository exists; git_username/git_email may be empty. - # @POST Repository config has user.name and user.email when both identity values are provided. + # [DEF:configure_identity:Function] + # @PURPOSE: Configure repository-local Git committer identity for user-scoped operations. + # @PRE: dashboard_id repository exists; git_username/git_email may be empty. + # @POST: Repository config has user.name and user.email when both identity values are provided. def configure_identity(self, dashboard_id: int, git_username: Optional[str], git_email: Optional[str]) -> None: with belief_scope("GitService.configure_identity"): normalized_username = str(git_username or "").strip() @@ -306,10 +307,10 @@ class GitServiceBase: with repo.config_writer(config_level="repository") as config_writer: config_writer.set_value("user", "name", normalized_username) config_writer.set_value("user", "email", normalized_email) - log.reason(f"Applied repository-local git identity for dashboard {dashboard_id}") + logger.info("[configure_identity][Action] Applied repository-local git identity for dashboard %s", dashboard_id) except Exception as e: - log.explore("Failed to configure git identity", error=str(e)) + logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}") raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {str(e)}") - # #endregion configure_identity + # [/DEF:configure_identity:Function] # #endregion GitServiceBase # #endregion GitServiceBase diff --git a/backend/src/services/git/_branch.py b/backend/src/services/git/_branch.py index 3f18af92..c886559a 100644 --- a/backend/src/services/git/_branch.py +++ b/backend/src/services/git/_branch.py @@ -1,6 +1,6 @@ # #region GitServiceBranchMixin [C:3] [TYPE Module] [SEMANTICS git, branches, commits, checkout, gitflow] +# @LAYER: Infra # @BRIEF Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes. -# @LAYER Infra # @RELATION USED_BY -> [GitService] import os @@ -9,19 +9,16 @@ from datetime import datetime from git import Repo from git.exc import GitCommandError from fastapi import HTTPException -from src.core.logger import belief_scope -from src.core.cot_logger import MarkerLogger - -log = MarkerLogger("GitBranch") +from src.core.logger import logger, belief_scope # #region GitServiceBranchMixin [C:3] [TYPE Class] # @BRIEF Mixin providing branch and commit operations for GitService. class GitServiceBranchMixin: - # #region _ensure_gitflow_branches [TYPE Function] - # @BRIEF Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin. - # @PRE repo is a valid GitPython Repo instance. - # @POST main, dev, preprod are available in local repository and pushed to origin when available. + # [DEF:_ensure_gitflow_branches:Function] + # @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin. + # @PRE: repo is a valid GitPython Repo instance. + # @POST: main, dev, preprod are available in local repository and pushed to origin when available. def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None: with belief_scope("GitService._ensure_gitflow_branches"): required_branches = ["main", "dev", "preprod"] @@ -34,20 +31,26 @@ class GitServiceBranchMixin: if "main" in local_heads: base_commit = local_heads["main"].commit if base_commit is None: - log.explore("Branch bootstrap skipped - no commits", payload={"dashboard_id": dashboard_id}, error="Repository has no initial commit to branch from") + logger.warning( + f"[_ensure_gitflow_branches][Action] Skipping branch bootstrap for dashboard {dashboard_id}: repository has no commits" + ) return if "main" not in local_heads: local_heads["main"] = repo.create_head("main", base_commit) - log.reason("Created local branch main", payload={"dashboard_id": dashboard_id}) + logger.info(f"[_ensure_gitflow_branches][Action] Created local branch main for dashboard {dashboard_id}") for branch_name in ("dev", "preprod"): if branch_name in local_heads: continue local_heads[branch_name] = repo.create_head(branch_name, local_heads["main"].commit) - log.reason("Created local branch", payload={"branch_name": branch_name, "dashboard_id": dashboard_id}) + logger.info( + f"[_ensure_gitflow_branches][Action] Created local branch {branch_name} for dashboard {dashboard_id}" + ) try: origin = repo.remote(name="origin") except ValueError: - log.reason("Remote origin not configured; skipping remote branch creation", payload={"dashboard_id": dashboard_id}) + logger.info( + f"[_ensure_gitflow_branches][Action] Remote origin is not configured for dashboard {dashboard_id}; skipping remote branch creation" + ) return remote_branch_names = set() try: @@ -57,35 +60,37 @@ class GitServiceBranchMixin: if remote_head: remote_branch_names.add(str(remote_head)) except Exception as e: - log.explore("Failed to fetch origin refs", error=str(e)) + logger.warning(f"[_ensure_gitflow_branches][Action] Failed to fetch origin refs: {e}") for branch_name in required_branches: if branch_name in remote_branch_names: continue try: origin.push(refspec=f"{branch_name}:{branch_name}") - log.reason("Pushed branch to origin", payload={"branch_name": branch_name, "dashboard_id": dashboard_id}) + logger.info( + f"[_ensure_gitflow_branches][Action] Pushed branch {branch_name} to origin for dashboard {dashboard_id}" + ) except Exception as e: - log.explore("Failed to push branch to origin", error=str(e)) + logger.error(f"[_ensure_gitflow_branches][Coherence:Failed] Failed to push branch {branch_name} to origin: {e}") raise HTTPException( status_code=500, detail=f"Failed to create default branch '{branch_name}' on remote: {str(e)}", ) try: repo.git.checkout("dev") - log.reason("Checked out default branch dev", payload={"dashboard_id": dashboard_id}) + logger.info(f"[_ensure_gitflow_branches][Action] Checked out default branch dev for dashboard {dashboard_id}") except Exception as e: - log.explore("Could not checkout dev branch", payload={"dashboard_id": dashboard_id}, error=str(e)) - # #endregion _ensure_gitflow_branches + logger.warning(f"[_ensure_gitflow_branches][Action] Could not checkout dev branch for dashboard {dashboard_id}: {e}") + # [/DEF:_ensure_gitflow_branches:Function] - # #region list_branches [TYPE Function] - # @BRIEF List all branches for a dashboard's repository. - # @PRE Repository for dashboard_id exists. - # @POST Returns a list of branch metadata dictionaries. - # @RETURN List[dict] + # [DEF:list_branches:Function] + # @PURPOSE: List all branches for a dashboard's repository. + # @PRE: Repository for dashboard_id exists. + # @POST: Returns a list of branch metadata dictionaries. + # @RETURN: List[dict] def list_branches(self, dashboard_id: int) -> List[dict]: with belief_scope("GitService.list_branches"): repo = self.get_repo(dashboard_id) - log.reason("Listing branches", payload={"dashboard_id": dashboard_id}) + logger.info(f"[list_branches][Action] Listing branches for {dashboard_id}. Refs: {repo.refs}") branches = [] for ref in repo.refs: try: @@ -99,7 +104,7 @@ class GitServiceBranchMixin: "last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow() }) except Exception as e: - log.explore("Skipping ref", payload={"ref": str(ref)}, error=str(e)) + logger.warning(f"[list_branches][Action] Skipping ref {ref}: {e}") try: active_name = repo.active_branch.name if not any(b['name'] == active_name for b in branches): @@ -108,17 +113,17 @@ class GitServiceBranchMixin: "is_remote": False, "last_updated": datetime.utcnow() }) except Exception as e: - log.explore("Could not determine active branch", error=str(e)) + logger.warning(f"[list_branches][Action] Could not determine active branch: {e}") if not branches: branches.append({ "name": "dev", "commit_hash": "0000000", "is_remote": False, "last_updated": datetime.utcnow() }) return branches - # #endregion list_branches + # [/DEF:list_branches:Function] - # #region create_branch [TYPE Function] - # @BRIEF Create a new branch from an existing one. + # [DEF:create_branch:Function] + # @PURPOSE: Create a new branch from an existing one. # @PARAM: name (str) - New branch name. # @PARAM: from_branch (str) - Source branch. # @PRE: Repository exists; name is valid; from_branch exists or repo is empty. @@ -126,9 +131,9 @@ class GitServiceBranchMixin: def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"): with belief_scope("GitService.create_branch"): repo = self.get_repo(dashboard_id) - log.reason("Creating branch", payload={"name": name, "from_branch": from_branch}) + logger.info(f"[create_branch][Action] Creating branch {name} from {from_branch}") if not repo.heads and not repo.remotes: - log.explore("Repository is empty; creating initial commit to enable branching", error="No branches or remotes exist; initializing from scratch") + logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.") readme_path = os.path.join(repo.working_dir, "README.md") if not os.path.exists(readme_path): with open(readme_path, "w") as f: @@ -138,29 +143,29 @@ class GitServiceBranchMixin: try: repo.commit(from_branch) except Exception: - log.explore("Source branch not found, using HEAD", payload={"from_branch": from_branch}, error=f"Branch '{from_branch}' does not exist, falling back to HEAD") + logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD") from_branch = repo.head try: new_branch = repo.create_head(name, from_branch) return new_branch except Exception as e: - log.explore("Failed to create branch", error=str(e)) + logger.error(f"[create_branch][Coherence:Failed] {e}") raise - # #endregion create_branch + # [/DEF:create_branch:Function] - # #region checkout_branch [TYPE Function] - # @BRIEF Switch to a specific branch. - # @PRE Repository exists and the specified branch name exists. - # @POST The repository working directory is updated to the specified branch. + # [DEF:checkout_branch:Function] + # @PURPOSE: Switch to a specific branch. + # @PRE: Repository exists and the specified branch name exists. + # @POST: The repository working directory is updated to the specified branch. def checkout_branch(self, dashboard_id: int, name: str): with belief_scope("GitService.checkout_branch"): repo = self.get_repo(dashboard_id) - log.reason("Checking out branch", payload={"name": name}) + logger.info(f"[checkout_branch][Action] Checking out branch {name}") repo.git.checkout(name) - # #endregion checkout_branch + # [/DEF:checkout_branch:Function] - # #region commit_changes [TYPE Function] - # @BRIEF Stage and commit changes. + # [DEF:commit_changes:Function] + # @PURPOSE: Stage and commit changes. # @PARAM: message (str) - Commit message. # @PARAM: files (List[str]) - Optional list of specific files to stage. # @PRE: Repository exists and has changes (dirty) or files are specified. @@ -169,16 +174,16 @@ class GitServiceBranchMixin: with belief_scope("GitService.commit_changes"): repo = self.get_repo(dashboard_id) if not repo.is_dirty(untracked_files=True) and not files: - log.reason("No changes to commit", payload={"dashboard_id": dashboard_id}) + logger.info(f"[commit_changes][Action] No changes to commit for dashboard {dashboard_id}") return if files: - log.reason("Staging files", payload={"files": files}) + logger.info(f"[commit_changes][Action] Staging files: {files}") repo.index.add(files) else: - log.reason("Staging all changes") + logger.info("[commit_changes][Action] Staging all changes") repo.git.add(A=True) repo.index.commit(message) - log.reflect("Committed changes", payload={"message": message}) - # #endregion commit_changes + logger.info(f"[commit_changes][Coherence:OK] Committed changes with message: {message}") + # [/DEF:commit_changes:Function] # #endregion GitServiceBranchMixin # #endregion GitServiceBranchMixin diff --git a/backend/src/services/git/_gitea.py b/backend/src/services/git/_gitea.py index a467f592..62f014dd 100644 --- a/backend/src/services/git/_gitea.py +++ b/backend/src/services/git/_gitea.py @@ -1,24 +1,21 @@ # #region GitServiceGiteaMixin [C:3] [TYPE Module] [SEMANTICS git, gitea, api, provider, connection_test] +# @LAYER: Infra # @BRIEF Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation. -# @LAYER Infra # @RELATION USED_BY -> [GitService] import httpx from typing import Any, Dict, List, Optional from urllib.parse import quote from fastapi import HTTPException -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope - -log = MarkerLogger("GitGitea") +from src.core.logger import logger, belief_scope from src.models.git import GitProvider # #region GitServiceGiteaMixin [C:3] [TYPE Class] # @BRIEF Mixin providing Gitea API operations for GitService. class GitServiceGiteaMixin: - # #region test_connection [TYPE Function] - # @BRIEF Test connection to Git provider using PAT. + # [DEF:test_connection:Function] + # @PURPOSE: Test connection to Git provider using PAT. # @PARAM: provider (GitProvider), url (str), pat (str) # @PRE: provider is valid; url is a valid HTTP(S) URL; pat is provided. # @POST: Returns True if connection to the provider's API succeeds. @@ -26,13 +23,13 @@ class GitServiceGiteaMixin: async def test_connection(self, provider: GitProvider, url: str, pat: str) -> bool: with belief_scope("GitService.test_connection"): if ".local" in url or "localhost" in url: - log.reason("Local/Offline mode detected for URL") + logger.info("[test_connection][Action] Local/Offline mode detected for URL") return True if not url.startswith(('http://', 'https://')): - log.explore(f"Invalid URL protocol: {url}", error="Invalid URL protocol") + logger.error(f"[test_connection][Coherence:Failed] Invalid URL protocol: {url}") return False if not pat or not pat.strip(): - log.explore("Git PAT is missing or empty", error="Git PAT is missing or empty") + logger.error("[test_connection][Coherence:Failed] Git PAT is missing or empty") return False pat = pat.strip() try: @@ -52,18 +49,18 @@ class GitServiceGiteaMixin: else: return False if resp.status_code != 200: - log.explore(f"Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}", error="Git connection test failed") + logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}") return resp.status_code == 200 except Exception as e: - log.explore(f"Error testing git connection: {e}", error=str(e)) + logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}") return False - # #endregion test_connection + # [/DEF:test_connection:Function] - # #region _gitea_headers [TYPE Function] - # @BRIEF Build Gitea API authorization headers. - # @PRE pat is provided. - # @POST Returns headers with token auth. - # @RETURN Dict[str, str] + # [DEF:_gitea_headers:Function] + # @PURPOSE: Build Gitea API authorization headers. + # @PRE: pat is provided. + # @POST: Returns headers with token auth. + # @RETURN: Dict[str, str] def _gitea_headers(self, pat: str) -> Dict[str, str]: token = (pat or "").strip() if not token: @@ -73,13 +70,13 @@ class GitServiceGiteaMixin: "Content-Type": "application/json", "Accept": "application/json", } - # #endregion _gitea_headers + # [/DEF:_gitea_headers:Function] - # #region _gitea_request [TYPE Function] - # @BRIEF Execute HTTP request against Gitea API with stable error mapping. - # @PRE method and endpoint are valid. - # @POST Returns decoded JSON payload. - # @RETURN Any + # [DEF:_gitea_request:Function] + # @PURPOSE: Execute HTTP request against Gitea API with stable error mapping. + # @PRE: method and endpoint are valid. + # @POST: Returns decoded JSON payload. + # @RETURN: Any async def _gitea_request( self, method: str, server_url: str, pat: str, endpoint: str, payload: Optional[Dict[str, Any]] = None, @@ -91,7 +88,7 @@ class GitServiceGiteaMixin: async with httpx.AsyncClient(timeout=20.0) as client: response = await client.request(method=method, url=url, headers=headers, json=payload) except Exception as e: - log.explore(f"Network error: {e}", error=str(e)) + logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}") raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {str(e)}") if response.status_code >= 400: detail = response.text @@ -100,43 +97,43 @@ class GitServiceGiteaMixin: detail = parsed.get("message") or parsed.get("error") or detail except Exception: pass - log.explore(f"Gitea API error ({endpoint})", payload={"method": method, "status": response.status_code}, error=detail) + logger.error(f"[gitea_request][Coherence:Failed] method={method} endpoint={endpoint} status={response.status_code} detail={detail}") raise HTTPException(status_code=response.status_code, detail=f"Gitea API error: {detail}") if response.status_code == 204: return None return response.json() - # #endregion _gitea_request + # [/DEF:_gitea_request:Function] - # #region get_gitea_current_user [TYPE Function] - # @BRIEF Resolve current Gitea user for PAT. - # @PRE server_url and pat are valid. - # @POST Returns current username. - # @RETURN str + # [DEF:get_gitea_current_user:Function] + # @PURPOSE: Resolve current Gitea user for PAT. + # @PRE: server_url and pat are valid. + # @POST: Returns current username. + # @RETURN: str async def get_gitea_current_user(self, server_url: str, pat: str) -> str: payload = await self._gitea_request("GET", server_url, pat, "/user") username = payload.get("login") or payload.get("username") if not username: raise HTTPException(status_code=500, detail="Failed to resolve Gitea username") return str(username) - # #endregion get_gitea_current_user + # [/DEF:get_gitea_current_user:Function] - # #region list_gitea_repositories [TYPE Function] - # @BRIEF List repositories visible to authenticated Gitea user. - # @PRE server_url and pat are valid. - # @POST Returns repository list from Gitea. - # @RETURN List[dict] + # [DEF:list_gitea_repositories:Function] + # @PURPOSE: List repositories visible to authenticated Gitea user. + # @PRE: server_url and pat are valid. + # @POST: Returns repository list from Gitea. + # @RETURN: List[dict] async def list_gitea_repositories(self, server_url: str, pat: str) -> List[dict]: payload = await self._gitea_request("GET", server_url, pat, "/user/repos?limit=100&page=1") if not isinstance(payload, list): return [] return payload - # #endregion list_gitea_repositories + # [/DEF:list_gitea_repositories:Function] - # #region create_gitea_repository [TYPE Function] - # @BRIEF Create repository in Gitea for authenticated user. - # @PRE name is non-empty and PAT has repo creation permission. - # @POST Returns created repository payload. - # @RETURN dict + # [DEF:create_gitea_repository:Function] + # @PURPOSE: Create repository in Gitea for authenticated user. + # @PRE: name is non-empty and PAT has repo creation permission. + # @POST: Returns created repository payload. + # @RETURN: dict async def create_gitea_repository( self, server_url: str, pat: str, name: str, private: bool = True, description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", @@ -150,23 +147,23 @@ class GitServiceGiteaMixin: if not isinstance(created, dict): raise HTTPException(status_code=500, detail="Unexpected Gitea response while creating repository") return created - # #endregion create_gitea_repository + # [/DEF:create_gitea_repository:Function] - # #region delete_gitea_repository [TYPE Function] - # @BRIEF Delete repository in Gitea. - # @PRE owner and repo_name are non-empty. - # @POST Repository deleted on Gitea server. + # [DEF:delete_gitea_repository:Function] + # @PURPOSE: Delete repository in Gitea. + # @PRE: owner and repo_name are non-empty. + # @POST: Repository deleted on Gitea server. async def delete_gitea_repository(self, server_url: str, pat: str, owner: str, repo_name: str) -> None: if not owner or not repo_name: raise HTTPException(status_code=400, detail="owner and repo_name are required") await self._gitea_request("DELETE", server_url, pat, f"/repos/{owner}/{repo_name}") - # #endregion delete_gitea_repository + # [/DEF:delete_gitea_repository:Function] - # #region _gitea_branch_exists [TYPE Function] - # @BRIEF Check whether a branch exists in Gitea repository. - # @PRE owner/repo/branch are non-empty. - # @POST Returns True when branch exists, False when 404. - # @RETURN bool + # [DEF:_gitea_branch_exists:Function] + # @PURPOSE: Check whether a branch exists in Gitea repository. + # @PRE: owner/repo/branch are non-empty. + # @POST: Returns True when branch exists, False when 404. + # @RETURN: bool async def _gitea_branch_exists(self, server_url: str, pat: str, owner: str, repo: str, branch: str) -> bool: if not owner or not repo or not branch: return False @@ -178,13 +175,13 @@ class GitServiceGiteaMixin: if exc.status_code == 404: return False raise - # #endregion _gitea_branch_exists + # [/DEF:_gitea_branch_exists:Function] - # #region _build_gitea_pr_404_detail [TYPE Function] - # @BRIEF Build actionable error detail for Gitea PR 404 responses. - # @PRE owner/repo/from_branch/to_branch are provided. - # @POST Returns specific branch-missing message when detected. - # @RETURN Optional[str] + # [DEF:_build_gitea_pr_404_detail:Function] + # @PURPOSE: Build actionable error detail for Gitea PR 404 responses. + # @PRE: owner/repo/from_branch/to_branch are provided. + # @POST: Returns specific branch-missing message when detected. + # @RETURN: Optional[str] async def _build_gitea_pr_404_detail( self, server_url: str, pat: str, owner: str, repo: str, from_branch: str, to_branch: str, ) -> Optional[str]: @@ -199,13 +196,13 @@ class GitServiceGiteaMixin: if not target_exists: return f"Gitea branch not found: target branch '{to_branch}' in {owner}/{repo}" return None - # #endregion _build_gitea_pr_404_detail + # [/DEF:_build_gitea_pr_404_detail:Function] - # #region create_gitea_pull_request [TYPE Function] - # @BRIEF Create pull request in Gitea. - # @PRE Config and remote URL are valid. - # @POST Returns normalized PR metadata. - # @RETURN Dict[str, Any] + # [DEF:create_gitea_pull_request:Function] + # @PURPOSE: Create pull request in Gitea. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized PR metadata. + # @RETURN: Dict[str, Any] async def create_gitea_pull_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, title: str, description: Optional[str] = None, @@ -223,7 +220,10 @@ class GitServiceGiteaMixin: exc.status_code == 404 and fallback_url and fallback_url != normalized_primary ) if should_retry_with_fallback: - log.explore(f"Primary Gitea URL not found, retrying with remote host: {fallback_url}", error=fallback_url) + logger.warning( + "[create_gitea_pull_request][Action] Primary Gitea URL not found, retrying with remote host: %s", + fallback_url, + ) active_server_url = fallback_url try: data = await self._gitea_request("POST", active_server_url, pat, endpoint, payload=req_payload) @@ -254,6 +254,6 @@ class GitServiceGiteaMixin: "url": data.get("html_url") or data.get("url"), "status": data.get("state") or "open", } - # #endregion create_gitea_pull_request + # [/DEF:create_gitea_pull_request:Function] # #endregion GitServiceGiteaMixin # #endregion GitServiceGiteaMixin diff --git a/backend/src/services/git/_merge.py b/backend/src/services/git/_merge.py index 0d307eb5..67a296a8 100644 --- a/backend/src/services/git/_merge.py +++ b/backend/src/services/git/_merge.py @@ -1,6 +1,6 @@ # #region GitServiceMergeMixin [C:3] [TYPE Module] [SEMANTICS git, merge, conflicts, resolution, promote] +# @LAYER: Infra # @BRIEF Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote. -# @LAYER Infra # @RELATION USED_BY -> [GitService] import os @@ -16,8 +16,8 @@ from src.core.logger import logger, belief_scope # #region GitServiceMergeMixin [C:3] [TYPE Class] # @BRIEF Mixin providing merge operations for GitService. class GitServiceMergeMixin: - # #region _read_blob_text [TYPE Function] - # @BRIEF Read text from a Git blob. + # [DEF:_read_blob_text:Function] + # @PURPOSE: Read text from a Git blob. def _read_blob_text(self, blob: Blob) -> str: with belief_scope("GitService._read_blob_text"): if blob is None: @@ -26,20 +26,20 @@ class GitServiceMergeMixin: return blob.data_stream.read().decode("utf-8", errors="replace") except Exception: return "" - # #endregion _read_blob_text + # [/DEF:_read_blob_text:Function] - # #region _get_unmerged_file_paths [TYPE Function] - # @BRIEF List files with merge conflicts. + # [DEF:_get_unmerged_file_paths:Function] + # @PURPOSE: List files with merge conflicts. def _get_unmerged_file_paths(self, repo: Repo) -> List[str]: with belief_scope("GitService._get_unmerged_file_paths"): try: return sorted(list(repo.index.unmerged_blobs().keys())) except Exception: return [] - # #endregion _get_unmerged_file_paths + # [/DEF:_get_unmerged_file_paths:Function] - # #region _build_unfinished_merge_payload [TYPE Function] - # @BRIEF Build payload for unfinished merge state. + # [DEF:_build_unfinished_merge_payload:Function] + # @PURPOSE: Build payload for unfinished merge state. def _build_unfinished_merge_payload(self, repo: Repo) -> Dict[str, Any]: with belief_scope("GitService._build_unfinished_merge_payload"): merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") @@ -83,10 +83,10 @@ class GitServiceMergeMixin: ], "manual_commands": ["git status", "git add ", 'git commit -m "resolve merge conflicts"', "git merge --abort"], } - # #endregion _build_unfinished_merge_payload + # [/DEF:_build_unfinished_merge_payload:Function] - # #region get_merge_status [TYPE Function] - # @BRIEF Get current merge status for a dashboard repository. + # [DEF:get_merge_status:Function] + # @PURPOSE: Get current merge status for a dashboard repository. def get_merge_status(self, dashboard_id: int) -> Dict[str, Any]: with belief_scope("GitService.get_merge_status"): repo = self.get_repo(dashboard_id) @@ -116,10 +116,10 @@ class GitServiceMergeMixin: "merge_message_preview": payload["merge_message_preview"], "conflicts_count": int(payload.get("conflicts_count") or 0), } - # #endregion get_merge_status + # [/DEF:get_merge_status:Function] - # #region get_merge_conflicts [TYPE Function] - # @BRIEF List all files with conflicts and their contents. + # [DEF:get_merge_conflicts:Function] + # @PURPOSE: List all files with conflicts and their contents. def get_merge_conflicts(self, dashboard_id: int) -> List[Dict[str, Any]]: with belief_scope("GitService.get_merge_conflicts"): repo = self.get_repo(dashboard_id) @@ -139,10 +139,10 @@ class GitServiceMergeMixin: "theirs": self._read_blob_text(theirs_blob) if theirs_blob else "", }) return sorted(conflicts, key=lambda item: item["file_path"]) - # #endregion get_merge_conflicts + # [/DEF:get_merge_conflicts:Function] - # #region resolve_merge_conflicts [TYPE Function] - # @BRIEF Resolve conflicts using specified strategy. + # [DEF:resolve_merge_conflicts:Function] + # @PURPOSE: Resolve conflicts using specified strategy. def resolve_merge_conflicts(self, dashboard_id: int, resolutions: List[Dict[str, Any]]) -> List[str]: with belief_scope("GitService.resolve_merge_conflicts"): repo = self.get_repo(dashboard_id) @@ -172,10 +172,10 @@ class GitServiceMergeMixin: repo.git.add(file_path) resolved_files.append(file_path) return resolved_files - # #endregion resolve_merge_conflicts + # [/DEF:resolve_merge_conflicts:Function] - # #region abort_merge [TYPE Function] - # @BRIEF Abort ongoing merge. + # [DEF:abort_merge:Function] + # @PURPOSE: Abort ongoing merge. def abort_merge(self, dashboard_id: int) -> Dict[str, Any]: with belief_scope("GitService.abort_merge"): repo = self.get_repo(dashboard_id) @@ -188,10 +188,10 @@ class GitServiceMergeMixin: return {"status": "no_merge_in_progress"} raise HTTPException(status_code=409, detail=f"Cannot abort merge: {details}") return {"status": "aborted"} - # #endregion abort_merge + # [/DEF:abort_merge:Function] - # #region continue_merge [TYPE Function] - # @BRIEF Finalize merge after conflict resolution. + # [DEF:continue_merge:Function] + # @PURPOSE: Finalize merge after conflict resolution. def continue_merge(self, dashboard_id: int, message: Optional[str] = None) -> Dict[str, Any]: with belief_scope("GitService.continue_merge"): repo = self.get_repo(dashboard_id) @@ -223,13 +223,13 @@ class GitServiceMergeMixin: except Exception: commit_hash = "" return {"status": "committed", "commit_hash": commit_hash} - # #endregion continue_merge + # [/DEF:continue_merge:Function] - # #region promote_direct_merge [TYPE Function] - # @BRIEF Perform direct merge between branches in local repo and push target branch. - # @PRE Repository exists and both branches are valid. - # @POST Target branch contains merged changes from source branch. - # @RETURN Dict[str, Any] + # [DEF:promote_direct_merge:Function] + # @PURPOSE: Perform direct merge between branches in local repo and push target branch. + # @PRE: Repository exists and both branches are valid. + # @POST: Target branch contains merged changes from source branch. + # @RETURN: Dict[str, Any] def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> Dict[str, Any]: with belief_scope("GitService.promote_direct_merge"): if not from_branch or not to_branch: @@ -270,6 +270,6 @@ class GitServiceMergeMixin: raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}") raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}") return {"mode": "direct", "from_branch": source, "to_branch": target, "status": "merged"} - # #endregion promote_direct_merge + # [/DEF:promote_direct_merge:Function] # #endregion GitServiceMergeMixin # #endregion GitServiceMergeMixin diff --git a/backend/src/services/git/_remote_providers.py b/backend/src/services/git/_remote_providers.py index 4723a347..a598b9cb 100644 --- a/backend/src/services/git/_remote_providers.py +++ b/backend/src/services/git/_remote_providers.py @@ -1,6 +1,6 @@ # #region GitServiceRemoteMixin [C:3] [TYPE Module] [SEMANTICS git, github, gitlab, api, provider, repository, pull_request, merge_request] +# @LAYER: Infra # @BRIEF GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation. -# @LAYER Infra # @RELATION USED_BY -> [GitService] import httpx @@ -10,15 +10,14 @@ from fastapi import HTTPException from src.core.logger import logger, belief_scope -# [DEF:GitServiceGithubMixin:Class] -# @COMPLEXITY: 3 -# @PURPOSE: Mixin providing GitHub API operations for GitService. +# #region GitServiceGithubMixin [C:3] [TYPE Class] +# @BRIEF Mixin providing GitHub API operations for GitService. class GitServiceGithubMixin: - # #region create_github_repository [TYPE Function] - # @BRIEF Create repository in GitHub or GitHub Enterprise. - # @PRE PAT has repository create permission. - # @POST Returns created repository payload. - # @RETURN dict + # [DEF:create_github_repository:Function] + # @PURPOSE: Create repository in GitHub or GitHub Enterprise. + # @PRE: PAT has repository create permission. + # @POST: Returns created repository payload. + # @RETURN: dict async def create_github_repository( self, server_url: str, pat: str, name: str, private: bool = True, description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", @@ -52,13 +51,13 @@ class GitServiceGithubMixin: pass raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") return response.json() - # #endregion create_github_repository + # [/DEF:create_github_repository:Function] - # #region create_github_pull_request [TYPE Function] - # @BRIEF Create pull request in GitHub or GitHub Enterprise. - # @PRE Config and remote URL are valid. - # @POST Returns normalized PR metadata. - # @RETURN Dict[str, Any] + # [DEF:create_github_pull_request:Function] + # @PURPOSE: Create pull request in GitHub or GitHub Enterprise. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized PR metadata. + # @RETURN: Dict[str, Any] async def create_github_pull_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, title: str, description: Optional[str] = None, draft: bool = False, @@ -92,17 +91,17 @@ class GitServiceGithubMixin: raise HTTPException(status_code=response.status_code, detail=f"GitHub API error: {detail}") data = response.json() return {"id": data.get("number") or data.get("id"), "url": data.get("html_url") or data.get("url"), "status": data.get("state") or "open"} - # #endregion create_github_pull_request + # [/DEF:create_github_pull_request:Function] # #region GitServiceGitlabMixin [C:3] [TYPE Class] # @BRIEF Mixin providing GitLab API operations for GitService. class GitServiceGitlabMixin: - # #region create_gitlab_repository [TYPE Function] - # @BRIEF Create repository(project) in GitLab. - # @PRE PAT has api scope. - # @POST Returns created repository payload. - # @RETURN dict + # [DEF:create_gitlab_repository:Function] + # @PURPOSE: Create repository(project) in GitLab. + # @PRE: PAT has api scope. + # @POST: Returns created repository payload. + # @RETURN: dict async def create_gitlab_repository( self, server_url: str, pat: str, name: str, private: bool = True, description: Optional[str] = None, auto_init: bool = True, default_branch: Optional[str] = "main", @@ -142,13 +141,13 @@ class GitServiceGitlabMixin: if "full_name" not in data: data["full_name"] = data.get("path_with_namespace") or data.get("name") return data - # #endregion create_gitlab_repository + # [/DEF:create_gitlab_repository:Function] - # #region create_gitlab_merge_request [TYPE Function] - # @BRIEF Create merge request in GitLab. - # @PRE Config and remote URL are valid. - # @POST Returns normalized MR metadata. - # @RETURN Dict[str, Any] + # [DEF:create_gitlab_merge_request:Function] + # @PURPOSE: Create merge request in GitLab. + # @PRE: Config and remote URL are valid. + # @POST: Returns normalized MR metadata. + # @RETURN: Dict[str, Any] async def create_gitlab_merge_request( self, server_url: str, pat: str, remote_url: str, from_branch: str, to_branch: str, title: str, description: Optional[str] = None, remove_source_branch: bool = False, @@ -178,6 +177,7 @@ class GitServiceGitlabMixin: raise HTTPException(status_code=response.status_code, detail=f"GitLab API error: {detail}") data = response.json() return {"id": data.get("iid") or data.get("id"), "url": data.get("web_url") or data.get("url"), "status": data.get("state") or "opened"} - # #endregion create_gitlab_merge_request + # [/DEF:create_gitlab_merge_request:Function] # #endregion GitServiceGitlabMixin # #endregion GitServiceRemoteMixin +# #endregion GitServiceRemoteMixin diff --git a/backend/src/services/git/_status.py b/backend/src/services/git/_status.py index 1356abe2..7adbf481 100644 --- a/backend/src/services/git/_status.py +++ b/backend/src/services/git/_status.py @@ -1,25 +1,23 @@ # #region GitServiceStatusMixin [C:3] [TYPE Module] [SEMANTICS git, status, diff, history, porcelain] +# @LAYER: Infra # @BRIEF Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues. -# @LAYER Infra # @RELATION USED_BY -> [GitService] from typing import Any, Dict, List from datetime import datetime from git import Repo -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope - -log = MarkerLogger("GitStatus") +from src.core.logger import logger, belief_scope # #region GitServiceStatusMixin [C:3] [TYPE Class] # @BRIEF Mixin providing repository status, diff, and commit history for GitService. class GitServiceStatusMixin: - # #region _parse_status_porcelain [C:2] [TYPE Function] - # @BRIEF Parse git status --porcelain output into staged, modified, and untracked file lists. - # @PRE `repo` is an open GitPython Repo instance. - # @POST Returns (staged, modified, untracked) tuple of file path lists. - # @RATIONALE Avoids repo.is_dirty() / repo.index.diff("HEAD") which internally + # [DEF:_parse_status_porcelain:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Parse git status --porcelain output into staged, modified, and untracked file lists. + # @PRE: `repo` is an open GitPython Repo instance. + # @POST: Returns (staged, modified, untracked) tuple of file path lists. + # @RATIONALE: Avoids repo.is_dirty() / repo.index.diff("HEAD") which internally # call git diff --cached, a flag unsupported in some Git environments (exit 129). # Using git status --porcelain is self-contained and avoids the --cached flag entirely. def _parse_status_porcelain(self, repo) -> tuple[list[str], list[str], list[str]]: @@ -30,7 +28,7 @@ class GitServiceStatusMixin: try: output = repo.git.status("--porcelain") except Exception: - log.explore("git status --porcelain failed", error="git status --porcelain command failed") + logger.warning("[_parse_status_porcelain][Coherence:Failed] git status --porcelain failed") return staged, modified, untracked for line in output.split("\n"): if not line: @@ -52,13 +50,13 @@ class GitServiceStatusMixin: if path not in modified: modified.append(path) return staged, modified, untracked - # #endregion _parse_status_porcelain + # [/DEF:_parse_status_porcelain:Function] - # #region get_status [TYPE Function] - # @BRIEF Get current repository status (dirty files, untracked, etc.) - # @PRE Repository for dashboard_id exists. - # @POST Returns a dictionary representing the Git status. - # @RETURN dict + # [DEF:get_status:Function] + # @PURPOSE: Get current repository status (dirty files, untracked, etc.) + # @PRE: Repository for dashboard_id exists. + # @POST: Returns a dictionary representing the Git status. + # @RETURN: dict def get_status(self, dashboard_id: int) -> dict: with belief_scope("GitService.get_status"): repo = self.get_repo(dashboard_id) @@ -112,10 +110,10 @@ class GitServiceStatusMixin: "is_diverged": is_diverged, "sync_state": sync_state, } - # #endregion get_status + # [/DEF:get_status:Function] - # #region get_diff [TYPE Function] - # @BRIEF Generate diff for a file or the whole repository. + # [DEF:get_diff:Function] + # @PURPOSE: Generate diff for a file or the whole repository. # @PARAM: file_path (str) - Optional specific file. # @PARAM: staged (bool) - Whether to show staged changes. # @PRE: Repository for dashboard_id exists. @@ -130,10 +128,10 @@ class GitServiceStatusMixin: if file_path: return repo.git.diff(*diff_args, "--", file_path) return repo.git.diff(*diff_args) - # #endregion get_diff + # [/DEF:get_diff:Function] - # #region get_commit_history [TYPE Function] - # @BRIEF Retrieve commit history for a repository. + # [DEF:get_commit_history:Function] + # @PURPOSE: Retrieve commit history for a repository. # @PARAM: limit (int) - Max number of commits to return. # @PRE: Repository for dashboard_id exists. # @POST: Returns a list of dictionaries for each commit in history. @@ -155,8 +153,9 @@ class GitServiceStatusMixin: "files_changed": list(commit.stats.files.keys()) }) except Exception as e: - log.explore(f"Could not retrieve commit history for dashboard {dashboard_id}: {e}", error=str(e)) + logger.warning(f"[get_commit_history][Action] Could not retrieve commit history for dashboard {dashboard_id}: {e}") return [] return commits - # #endregion get_commit_history + # [/DEF:get_commit_history:Function] +# #endregion GitServiceStatusMixin # #endregion GitServiceStatusMixin diff --git a/backend/src/services/git/_sync.py b/backend/src/services/git/_sync.py index 99d7ed05..d0d429d4 100644 --- a/backend/src/services/git/_sync.py +++ b/backend/src/services/git/_sync.py @@ -1,6 +1,6 @@ # #region GitServiceSyncMixin [C:3] [TYPE Module] [SEMANTICS git, push, pull, sync, remote] +# @LAYER: Infra # @BRIEF Push and pull operations for GitService with origin host auto-alignment. -# @LAYER Infra # @RELATION USED_BY -> [GitService] # @RELATION DEPENDS_ON -> [GitServiceUrlMixin] @@ -10,30 +10,27 @@ from git import Repo from git.exc import GitCommandError from fastapi import HTTPException from src.core.database import SessionLocal -from src.core.logger import belief_scope -from src.core.cot_logger import MarkerLogger - -log = MarkerLogger("GitSync") +from src.core.logger import logger, belief_scope from src.models.git import GitRepository, GitServerConfig # #region GitServiceSyncMixin [C:3] [TYPE Class] # @BRIEF Mixin providing push and pull operations with origin host alignment. class GitServiceSyncMixin: - # #region push_changes [TYPE Function] - # @BRIEF Push local commits to remote. - # @PRE Repository exists and has an 'origin' remote. - # @POST Local branch commits are pushed to origin. + # [DEF:push_changes:Function] + # @PURPOSE: Push local commits to remote. + # @PRE: Repository exists and has an 'origin' remote. + # @POST: Local branch commits are pushed to origin. def push_changes(self, dashboard_id: int): with belief_scope("GitService.push_changes"): repo = self.get_repo(dashboard_id) if not repo.heads: - log.explore("No local branches to push", payload={"dashboard_id": dashboard_id}, error="Repository has no local branch heads to push") + logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}") return try: origin = repo.remote(name='origin') except ValueError: - log.explore("Remote 'origin' not found", payload={"dashboard_id": dashboard_id}, error="Git remote 'origin' is not configured for this repository") + logger.error(f"[push_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") raise HTTPException(status_code=400, detail="Remote 'origin' not configured") try: origin_urls = list(origin.urls) @@ -63,7 +60,10 @@ class GitServiceSyncMixin: finally: session.close() except Exception as diag_error: - log.explore("Failed to load repository binding diagnostics", payload={"dashboard_id": dashboard_id}, error=str(diag_error)) + logger.warning( + "[push_changes][Action] Failed to load repository binding diagnostics for dashboard %s: %s", + dashboard_id, diag_error, + ) realigned_origin_url = self._align_origin_host_with_config( dashboard_id=dashboard_id, origin=origin, @@ -75,17 +75,13 @@ class GitServiceSyncMixin: origin_urls = list(origin.urls) except Exception: origin_urls = [] - log.reason( - "Push diagnostics", - payload={ - "dashboard_id": dashboard_id, "config_id": binding_config_id, - "config_url": binding_config_url, "binding_remote_url": binding_remote_url, - "origin_urls": origin_urls, "origin_realigned": bool(realigned_origin_url), - }, + logger.info( + "[push_changes][Action] Push diagnostics dashboard=%s config_id=%s config_url=%s binding_remote_url=%s origin_urls=%s origin_realigned=%s", + dashboard_id, binding_config_id, binding_config_url, binding_remote_url, origin_urls, bool(realigned_origin_url), ) try: current_branch = repo.active_branch - log.reason("Pushing branch", payload={"branch": current_branch.name, "origin": True}) + logger.info(f"[push_changes][Action] Pushing branch {current_branch.name} to origin") tracking_branch = None try: tracking_branch = current_branch.tracking_branch() @@ -97,7 +93,7 @@ class GitServiceSyncMixin: push_info = origin.push(refspec=f'{current_branch.name}:{current_branch.name}') for info in push_info: if info.flags & info.ERROR: - log.explore("Error pushing ref", payload={"ref": info.remote_ref_string, "summary": info.summary}, error=f"Git push error: {info.summary}") + logger.error(f"[push_changes][Coherence:Failed] Error pushing ref {info.remote_ref_string}: {info.summary}") raise Exception(f"Git push error for {info.remote_ref_string}: {info.summary}") except GitCommandError as e: details = str(e) @@ -107,31 +103,28 @@ class GitServiceSyncMixin: status_code=409, detail="Push rejected: remote branch contains newer commits. Run Pull first, resolve conflicts if any, then push again.", ) - log.explore("Failed to push changes", error=str(e)) + logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") raise HTTPException(status_code=500, detail=f"Git push failed: {details}") except Exception as e: - log.explore("Failed to push changes", error=str(e)) + logger.error(f"[push_changes][Coherence:Failed] Failed to push changes: {e}") raise HTTPException(status_code=500, detail=f"Git push failed: {str(e)}") - # #endregion push_changes + # [/DEF:push_changes:Function] - # #region pull_changes [TYPE Function] - # @BRIEF Pull changes from remote. - # @PRE Repository exists and has an 'origin' remote. - # @POST Changes from origin are pulled and merged into the active branch. + # [DEF:pull_changes:Function] + # @PURPOSE: Pull changes from remote. + # @PRE: Repository exists and has an 'origin' remote. + # @POST: Changes from origin are pulled and merged into the active branch. def pull_changes(self, dashboard_id: int): with belief_scope("GitService.pull_changes"): repo = self.get_repo(dashboard_id) merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD") if os.path.exists(merge_head_path): payload = self._build_unfinished_merge_payload(repo) - log.explore("Unfinished merge detected", error="Unfinished merge state found", - payload={ - "dashboard_id": dashboard_id, - "repo_path": payload["repository_path"], - "git_dir": payload["git_dir"], - "branch": payload["current_branch"], - "merge_head": payload["merge_head"], - }) + logger.warning( + "[pull_changes][Action] Unfinished merge detected for dashboard %s (repo_path=%s git_dir=%s branch=%s merge_head=%s merge_msg=%s)", + dashboard_id, payload["repository_path"], payload["git_dir"], + payload["current_branch"], payload["merge_head"], payload["merge_message_preview"], + ) raise HTTPException(status_code=409, detail=payload) try: origin = repo.remote(name='origin') @@ -140,36 +133,26 @@ class GitServiceSyncMixin: origin_urls = list(origin.urls) except Exception: origin_urls = [] - log.reason( - "Pull diagnostics", - payload={ - "dashboard_id": dashboard_id, - "repo_path": repo.working_tree_dir, - "branch": current_branch, - "origin_urls": origin_urls, - }, + logger.info( + "[pull_changes][Action] Pull diagnostics dashboard=%s repo_path=%s branch=%s origin_urls=%s", + dashboard_id, repo.working_tree_dir, current_branch, origin_urls, ) origin.fetch(prune=True) remote_ref = f"origin/{current_branch}" has_remote_branch = any(ref.name == remote_ref for ref in repo.refs) - log.reason( - "Pull remote branch check", - payload={ - "dashboard_id": dashboard_id, - "branch": current_branch, - "remote_ref": remote_ref, - "exists": has_remote_branch, - }, + logger.info( + "[pull_changes][Action] Pull remote branch check dashboard=%s branch=%s remote_ref=%s exists=%s", + dashboard_id, current_branch, remote_ref, has_remote_branch, ) if not has_remote_branch: raise HTTPException( status_code=409, detail=f"Remote branch '{current_branch}' does not exist yet. Push this branch first.", ) - log.reason("Pulling changes", payload={"branch": current_branch}) + logger.info(f"[pull_changes][Action] Pulling changes from origin/{current_branch}") repo.git.pull("--no-rebase", "origin", current_branch) except ValueError: - log.explore("Remote 'origin' not found", payload={"dashboard_id": dashboard_id}, error="Git remote 'origin' is not configured for this repository") + logger.error(f"[pull_changes][Coherence:Failed] Remote 'origin' not found for dashboard {dashboard_id}") raise HTTPException(status_code=400, detail="Remote 'origin' not configured") except GitCommandError as e: details = str(e) @@ -179,13 +162,13 @@ class GitServiceSyncMixin: status_code=409, detail="Pull requires conflict resolution. Resolve conflicts in repository and repeat operation.", ) - log.explore("Failed to pull changes", error=str(e)) + logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") raise HTTPException(status_code=500, detail=f"Git pull failed: {details}") except HTTPException: raise except Exception as e: - log.explore("Failed to pull changes", error=str(e)) + logger.error(f"[pull_changes][Coherence:Failed] Failed to pull changes: {e}") raise HTTPException(status_code=500, detail=f"Git pull failed: {str(e)}") - # #endregion pull_changes + # [/DEF:pull_changes:Function] # #endregion GitServiceSyncMixin # #endregion GitServiceSyncMixin diff --git a/backend/src/services/git/_url.py b/backend/src/services/git/_url.py index 70ad1e75..0c455325 100644 --- a/backend/src/services/git/_url.py +++ b/backend/src/services/git/_url.py @@ -1,6 +1,6 @@ # #region GitServiceUrlMixin [C:3] [TYPE Module] [SEMANTICS git, url, parsing, normalization, host_alignment] +# @LAYER: Infra # @BRIEF URL helper mixin for GitService — parse, normalize, align, and strip credentials from Git URLs. -# @LAYER Infra # @RELATION USED_BY -> [GitServiceSyncMixin] # @RELATION USED_BY -> [GitServiceGiteaMixin] # @RELATION USED_BY -> [GitServiceRemoteMixin] @@ -10,20 +10,17 @@ from typing import Any, Dict, List, Optional from urllib.parse import quote, urlparse from fastapi import HTTPException from src.core.database import SessionLocal -from src.core.cot_logger import MarkerLogger -from src.core.logger import belief_scope - -log = MarkerLogger("GitUrl") +from src.core.logger import logger, belief_scope from src.models.git import GitRepository, GitServerConfig # #region GitServiceUrlMixin [C:3] [TYPE Class] # @BRIEF URL helper methods for GitService — extract host, strip credentials, replace host, align origin, parse remote identity, derive server URL, normalize server URL. class GitServiceUrlMixin: - # #region _extract_http_host [TYPE Function] - # @BRIEF Extract normalized host[:port] from HTTP(S) URL. - # @PRE url_value may be empty. - # @POST Returns lowercase host token or None. - # @RETURN Optional[str] + # [DEF:_extract_http_host:Function] + # @PURPOSE: Extract normalized host[:port] from HTTP(S) URL. + # @PRE: url_value may be empty. + # @POST: Returns lowercase host token or None. + # @RETURN: Optional[str] def _extract_http_host(self, url_value: Optional[str]) -> Optional[str]: normalized = str(url_value or "").strip() if not normalized: @@ -40,13 +37,13 @@ class GitServiceUrlMixin: if parsed.port: return f"{host.lower()}:{parsed.port}" return host.lower() - # #endregion _extract_http_host + # [/DEF:_extract_http_host:Function] - # #region _strip_url_credentials [TYPE Function] - # @BRIEF Remove credentials from URL while preserving scheme/host/path. - # @PRE url_value may contain credentials. - # @POST Returns URL without username/password. - # @RETURN str + # [DEF:_strip_url_credentials:Function] + # @PURPOSE: Remove credentials from URL while preserving scheme/host/path. + # @PRE: url_value may contain credentials. + # @POST: Returns URL without username/password. + # @RETURN: str def _strip_url_credentials(self, url_value: str) -> str: normalized = str(url_value or "").strip() if not normalized: @@ -61,13 +58,13 @@ class GitServiceUrlMixin: if parsed.port: host = f"{host}:{parsed.port}" return parsed._replace(netloc=host).geturl() - # #endregion _strip_url_credentials + # [/DEF:_strip_url_credentials:Function] - # #region _replace_host_in_url [TYPE Function] - # @BRIEF Replace source URL host with host from configured server URL. - # @PRE source_url and config_url are HTTP(S) URLs. - # @POST Returns source URL with updated host (credentials preserved) or None. - # @RETURN Optional[str] + # [DEF:_replace_host_in_url:Function] + # @PURPOSE: Replace source URL host with host from configured server URL. + # @PRE: source_url and config_url are HTTP(S) URLs. + # @POST: Returns source URL with updated host (credentials preserved) or None. + # @RETURN: Optional[str] def _replace_host_in_url(self, source_url: Optional[str], config_url: Optional[str]) -> Optional[str]: source = str(source_url or "").strip() config = str(config_url or "").strip() @@ -93,13 +90,13 @@ class GitServiceUrlMixin: auth_part = f"{auth_part}@" new_netloc = f"{auth_part}{target_host}" return source_parsed._replace(netloc=new_netloc).geturl() - # #endregion _replace_host_in_url + # [/DEF:_replace_host_in_url:Function] - # #region _align_origin_host_with_config [TYPE Function] - # @BRIEF Auto-align local origin host to configured Git server host when they drift. - # @PRE origin remote exists. - # @POST origin URL host updated and DB binding normalized when mismatch detected. - # @RETURN Optional[str] + # [DEF:_align_origin_host_with_config:Function] + # @PURPOSE: Auto-align local origin host to configured Git server host when they drift. + # @PRE: origin remote exists. + # @POST: origin URL host updated and DB binding normalized when mismatch detected. + # @RETURN: Optional[str] def _align_origin_host_with_config( self, dashboard_id: int, @@ -118,11 +115,17 @@ class GitServiceUrlMixin: aligned_url = self._replace_host_in_url(source_origin_url, config_url) if not aligned_url: return None - log.explore(f"Host mismatch for dashboard {dashboard_id}: config_host={config_host} origin_host={origin_host}, applying origin.set_url", error=f"Host mismatch for dashboard {dashboard_id}") + logger.warning( + "[_align_origin_host_with_config][Action] Host mismatch for dashboard %s: config_host=%s origin_host=%s, applying origin.set_url", + dashboard_id, config_host, origin_host, + ) try: origin.set_url(aligned_url) except Exception as e: - log.explore(f"Failed to set origin URL for dashboard {dashboard_id}: {e}", error=str(e)) + logger.warning( + "[_align_origin_host_with_config][Coherence:Failed] Failed to set origin URL for dashboard %s: %s", + dashboard_id, e, + ) return None try: session = SessionLocal() @@ -138,15 +141,18 @@ class GitServiceUrlMixin: finally: session.close() except Exception as e: - log.explore(f"Failed to persist aligned remote_url for dashboard {dashboard_id}: {e}", error=str(e)) + logger.warning( + "[_align_origin_host_with_config][Action] Failed to persist aligned remote_url for dashboard %s: %s", + dashboard_id, e, + ) return aligned_url - # #endregion _align_origin_host_with_config + # [/DEF:_align_origin_host_with_config:Function] - # #region _parse_remote_repo_identity [TYPE Function] - # @BRIEF Parse owner/repo from remote URL for Git server API operations. - # @PRE remote_url is a valid git URL. - # @POST Returns owner/repo tokens. - # @RETURN Dict[str, str] + # [DEF:_parse_remote_repo_identity:Function] + # @PURPOSE: Parse owner/repo from remote URL for Git server API operations. + # @PRE: remote_url is a valid git URL. + # @POST: Returns owner/repo tokens. + # @RETURN: Dict[str, str] def _parse_remote_repo_identity(self, remote_url: str) -> Dict[str, str]: normalized = str(remote_url or "").strip() if not normalized: @@ -166,13 +172,13 @@ class GitServiceUrlMixin: repo = parts[-1] namespace = "/".join(parts[:-1]) return {"owner": owner, "repo": repo, "namespace": namespace, "full_name": f"{namespace}/{repo}"} - # #endregion _parse_remote_repo_identity + # [/DEF:_parse_remote_repo_identity:Function] - # #region _derive_server_url_from_remote [TYPE Function] - # @BRIEF Build API base URL from remote repository URL without credentials. - # @PRE remote_url may be any git URL. - # @POST Returns normalized http(s) base URL or None when derivation is impossible. - # @RETURN Optional[str] + # [DEF:_derive_server_url_from_remote:Function] + # @PURPOSE: Build API base URL from remote repository URL without credentials. + # @PRE: remote_url may be any git URL. + # @POST: Returns normalized http(s) base URL or None when derivation is impossible. + # @RETURN: Optional[str] def _derive_server_url_from_remote(self, remote_url: str) -> Optional[str]: normalized = str(remote_url or "").strip() if not normalized or normalized.startswith("git@"): @@ -186,18 +192,18 @@ class GitServiceUrlMixin: if parsed.port: netloc = f"{netloc}:{parsed.port}" return f"{parsed.scheme}://{netloc}".rstrip("/") - # #endregion _derive_server_url_from_remote + # [/DEF:_derive_server_url_from_remote:Function] - # #region _normalize_git_server_url [TYPE Function] - # @BRIEF Normalize Git server URL for provider API calls. - # @PRE raw_url is non-empty. - # @POST Returns URL without trailing slash. - # @RETURN str + # [DEF:_normalize_git_server_url:Function] + # @PURPOSE: Normalize Git server URL for provider API calls. + # @PRE: raw_url is non-empty. + # @POST: Returns URL without trailing slash. + # @RETURN: str def _normalize_git_server_url(self, raw_url: str) -> str: normalized = (raw_url or "").strip() if not normalized: raise HTTPException(status_code=400, detail="Git server URL is required") return normalized.rstrip("/") - # #endregion _normalize_git_server_url + # [/DEF:_normalize_git_server_url:Function] # #endregion GitServiceUrlMixin # #endregion GitServiceUrlMixin diff --git a/backend/src/services/git_service.py b/backend/src/services/git_service.py index 92328cc6..18cb9e7a 100644 --- a/backend/src/services/git_service.py +++ b/backend/src/services/git_service.py @@ -1,10 +1,11 @@ -# #region git_service [C:1] [TYPE Module:Tombstone] +# [DEF:git_service:Module:Tombstone] +# @COMPLEXITY: 1 # @BRIEF Re-export shim — GitService has been decomposed into services/git/ package. # All consumers continue to import from this same path without changes. -# @RELATION: REDIRECTS_TO -> [GitServiceModule] +# @RELATION REDIRECTS_TO -> [GitServiceModule] # @RATIONALE: Monolithic GitService (2101 lines) was decomposed into 8 domain-specific mixins # under services/git/ to satisfy INV_7 (< 400 lines per module). This shim preserves # the original import path for all 5 consumers. # @REJECTED: Breaking 5 consumer imports to remove this shim — unacceptable migration cost. from src.services.git import GitService # noqa: F401 -# #endregion git_service +# [/DEF:git_service:Module:Tombstone] diff --git a/backend/src/services/health_service.py b/backend/src/services/health_service.py index 5dcfe244..22e97bca 100644 --- a/backend/src/services/health_service.py +++ b/backend/src/services/health_service.py @@ -1,6 +1,6 @@ # #region health_service [C:3] [TYPE Module] [SEMANTICS health, aggregation, dashboards] # @BRIEF Business logic for aggregating dashboard health status from validation records. -# @LAYER Domain/Service +# @LAYER: Domain/Service # @RELATION DEPENDS_ON -> [ValidationRecord] # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [TaskCleanupService] @@ -25,10 +25,10 @@ def _empty_dashboard_meta() -> Dict[str, Optional[str]]: # #region HealthService [C:4] [TYPE Class] # @BRIEF Aggregate latest dashboard validation state and manage persisted health report lifecycle. -# @PRE Service is constructed with a live SQLAlchemy session and optional config manager. -# @POST Exposes health summary aggregation and validation report deletion operations. -# @SIDE_EFFECT Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators. -# @DATA_CONTRACT Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool] +# @PRE: Service is constructed with a live SQLAlchemy session and optional config manager. +# @POST: Exposes health summary aggregation and validation report deletion operations. +# @SIDE_EFFECT: Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators. +# @DATA_CONTRACT: Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool] # @RELATION DEPENDS_ON -> [ValidationRecord] # @RELATION DEPENDS_ON -> [DashboardHealthItem] # @RELATION DEPENDS_ON -> [HealthSummaryResponse] @@ -45,29 +45,31 @@ class HealthService: @PURPOSE: Service for managing and querying dashboard health data. """ - # #region HealthService_init [C:3] [TYPE Function] - # @BRIEF Initialize health service with DB session and optional config access for dashboard metadata resolution. - # @PRE db is a valid SQLAlchemy session. - # @POST Service is ready to aggregate summaries and delete health reports. - # @SIDE_EFFECT Initializes per-instance dashboard metadata cache. - # @DATA_CONTRACT Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService] - # @RELATION BINDS_TO -> [HealthService] + # [DEF:HealthService_init:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution. + # @PRE: db is a valid SQLAlchemy session. + # @POST: Service is ready to aggregate summaries and delete health reports. + # @SIDE_EFFECT: Initializes per-instance dashboard metadata cache. + # @DATA_CONTRACT: Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService] + # @RELATION: [BINDS_TO] ->[HealthService] def __init__(self, db: Session, config_manager=None): self.db = db self.config_manager = config_manager self._dashboard_meta_cache: Dict[Tuple[str, str], Dict[str, Optional[str]]] = {} - # #endregion HealthService_init + # [/DEF:HealthService_init:Function] - # #region _prime_dashboard_meta_cache [C:3] [TYPE Function] - # @BRIEF Warm dashboard slug/title cache with one Superset list fetch per environment. - # @PRE records may contain mixed numeric and slug dashboard identifiers. - # @POST Numeric dashboard ids for known environments are cached when discoverable. - # @SIDE_EFFECT May call Superset dashboard list API once per referenced environment. - # @DATA_CONTRACT Input[records: List[ValidationRecord]] -> Output[None] - # @RELATION DEPENDS_ON -> [ValidationRecord] - # @RELATION DEPENDS_ON -> [ConfigManager] - # @RELATION DEPENDS_ON -> [SupersetClient] + # [DEF:_prime_dashboard_meta_cache:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Warm dashboard slug/title cache with one Superset list fetch per environment. + # @PRE: records may contain mixed numeric and slug dashboard identifiers. + # @POST: Numeric dashboard ids for known environments are cached when discoverable. + # @SIDE_EFFECT: May call Superset dashboard list API once per referenced environment. + # @DATA_CONTRACT: Input[records: List[ValidationRecord]] -> Output[None] + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[ConfigManager] + # @RELATION: [DEPENDS_ON] ->[SupersetClient] def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None: if not self.config_manager or not records: return @@ -148,13 +150,14 @@ class HealthService: _empty_dashboard_meta() ) - # #endregion _prime_dashboard_meta_cache + # [/DEF:_prime_dashboard_meta_cache:Function] - # #region _resolve_dashboard_meta [C:1] [TYPE Function] - # @BRIEF Resolve slug/title for a dashboard referenced by persisted validation record. - # @PRE dashboard_id may be numeric or slug-like; environment_id may be empty. - # @POST Returns dict with `slug` and `title` keys, using cache when possible. - # @SIDE_EFFECT Writes default cache entries for unresolved numeric dashboard ids. + # [DEF:_resolve_dashboard_meta:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record. + # @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty. + # @POST: Returns dict with `slug` and `title` keys, using cache when possible. + # @SIDE_EFFECT: Writes default cache entries for unresolved numeric dashboard ids. def _resolve_dashboard_meta( self, dashboard_id: str, environment_id: Optional[str] ) -> Dict[str, Optional[str]]: @@ -178,16 +181,17 @@ class HealthService: self._dashboard_meta_cache[cache_key] = meta return meta - # #endregion _resolve_dashboard_meta + # [/DEF:_resolve_dashboard_meta:Function] - # #region get_health_summary [C:3] [TYPE Function] - # @BRIEF Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title. - # @PRE environment_id may be omitted to aggregate across all environments. - # @POST Returns HealthSummaryResponse with counts and latest record row per dashboard. - # @SIDE_EFFECT May call Superset API to resolve dashboard metadata. - # @DATA_CONTRACT Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse] - # @RELATION CALLS -> [_prime_dashboard_meta_cache] - # @RELATION CALLS -> [_resolve_dashboard_meta] + # [DEF:get_health_summary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Aggregate latest validation status per dashboard and enrich rows with dashboard slug/title. + # @PRE: environment_id may be omitted to aggregate across all environments. + # @POST: Returns HealthSummaryResponse with counts and latest record row per dashboard. + # @SIDE_EFFECT: May call Superset API to resolve dashboard metadata. + # @DATA_CONTRACT: Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse] + # @RELATION: [CALLS] ->[_prime_dashboard_meta_cache] + # @RELATION: [CALLS] ->[_resolve_dashboard_meta] async def get_health_summary( self, environment_id: str = "" ) -> HealthSummaryResponse: @@ -281,17 +285,18 @@ class HealthService: unknown_count=unknown_count, ) - # #endregion get_health_summary + # [/DEF:get_health_summary:Function] - # #region delete_validation_report [C:3] [TYPE Function] - # @BRIEF Delete one persisted health report and optionally clean linked task/log artifacts. - # @PRE record_id is a validation record identifier. - # @POST Returns True only when a matching record was deleted. - # @SIDE_EFFECT Deletes DB rows, optional screenshot file, and optional task/log persistence. - # @DATA_CONTRACT Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool] - # @RELATION DEPENDS_ON -> [ValidationRecord] - # @RELATION DEPENDS_ON -> [TaskManager] - # @RELATION DEPENDS_ON -> [TaskCleanupService] + # [DEF:delete_validation_report:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Delete one persisted health report and optionally clean linked task/log artifacts. + # @PRE: record_id is a validation record identifier. + # @POST: Returns True only when a matching record was deleted. + # @SIDE_EFFECT: Deletes DB rows, optional screenshot file, and optional task/log persistence. + # @DATA_CONTRACT: Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool] + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[TaskManager] + # @RELATION: [DEPENDS_ON] ->[TaskCleanupService] def delete_validation_report( self, record_id: str, task_manager: Optional[TaskManager] = None ) -> bool: @@ -368,7 +373,9 @@ class HealthService: return True - # #endregion delete_validation_report + # [/DEF:delete_validation_report:Function] # #endregion HealthService + +# #endregion health_service diff --git a/backend/src/services/llm_prompt_templates.py b/backend/src/services/llm_prompt_templates.py index 38933268..b390fdbb 100644 --- a/backend/src/services/llm_prompt_templates.py +++ b/backend/src/services/llm_prompt_templates.py @@ -1,8 +1,8 @@ # #region llm_prompt_templates [C:2] [TYPE Module] [SEMANTICS llm, prompts, templates, settings] # @BRIEF Provide default LLM prompt templates and normalization helpers for runtime usage. -# @LAYER Domain -# @INVARIANT All required prompt template keys are always present after normalization. +# @LAYER: Domain # @RELATION DEPENDS_ON -> [backend.src.core.config_manager:Function] +# @INVARIANT: All required prompt template keys are always present after normalization. from __future__ import annotations @@ -79,9 +79,9 @@ DEFAULT_LLM_ASSISTANT_SETTINGS: Dict[str, str] = { # #region normalize_llm_settings [C:3] [TYPE Function] # @BRIEF Ensure llm settings contain stable schema with prompts section and default templates. -# @PRE llm_settings is dictionary-like value or None. -# @POST Returned dict contains prompts with all required template keys. -# @RELATION DEPENDS_ON -> [LLMProviderService] +# @PRE: llm_settings is dictionary-like value or None. +# @POST: Returned dict contains prompts with all required template keys. +# @RELATION DEPENDS_ON -> LLMProviderService def normalize_llm_settings(llm_settings: Any) -> Dict[str, Any]: normalized: Dict[str, Any] = { "providers": [], @@ -123,9 +123,9 @@ def normalize_llm_settings(llm_settings: Any) -> Dict[str, Any]: # #region is_multimodal_model [C:3] [TYPE Function] # @BRIEF Heuristically determine whether model supports image input required for dashboard validation. -# @PRE model_name may be empty or mixed-case. -# @POST Returns True when model likely supports multimodal input. -# @RELATION DEPENDS_ON -> [LLMProviderService] +# @PRE: model_name may be empty or mixed-case. +# @POST: Returns True when model likely supports multimodal input. +# @RELATION DEPENDS_ON -> LLMProviderService def is_multimodal_model(model_name: str, provider_type: Optional[str] = None) -> bool: token = (model_name or "").strip().lower() if not token: @@ -165,9 +165,9 @@ def is_multimodal_model(model_name: str, provider_type: Optional[str] = None) -> # #region resolve_bound_provider_id [C:3] [TYPE Function] # @BRIEF Resolve provider id configured for a task binding with fallback to default provider. -# @PRE llm_settings is normalized or raw dict from config. -# @POST Returns configured provider id or fallback id/empty string when not defined. -# @RELATION DEPENDS_ON -> [LLMProviderService] +# @PRE: llm_settings is normalized or raw dict from config. +# @POST: Returns configured provider id or fallback id/empty string when not defined. +# @RELATION DEPENDS_ON -> LLMProviderService def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str: normalized = normalize_llm_settings(llm_settings) bindings = normalized.get("provider_bindings", {}) @@ -181,9 +181,9 @@ def resolve_bound_provider_id(llm_settings: Any, task_key: str) -> str: # #region render_prompt [C:3] [TYPE Function] # @BRIEF Render prompt template using deterministic placeholder replacement with graceful fallback. -# @PRE template is a string and variables values are already stringifiable. -# @POST Returns rendered prompt text with known placeholders substituted. -# @RELATION DEPENDS_ON -> [LLMProviderService] +# @PRE: template is a string and variables values are already stringifiable. +# @POST: Returns rendered prompt text with known placeholders substituted. +# @RELATION DEPENDS_ON -> LLMProviderService def render_prompt(template: str, variables: Dict[str, Any]) -> str: rendered = template for key, value in variables.items(): diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index bd899a7c..1bf11696 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -1,6 +1,6 @@ # #region llm_provider [C:3] [TYPE Module] [SEMANTICS service, llm, provider, encryption] # @BRIEF Service for managing LLM provider configurations with encrypted API keys. -# @LAYER Domain +# @LAYER: Domain # @RELATION DEPENDS_ON -> [LLMProvider] # @RELATION DEPENDS_ON -> [EncryptionManager] # @RELATION DEPENDS_ON -> [LLMProviderConfig] @@ -19,12 +19,12 @@ MASKED_API_KEY_PLACEHOLDER = "********" # #region _require_fernet_key [C:5] [TYPE Function] # @BRIEF Load and validate the Fernet key used for secret encryption. -# @PRE ENCRYPTION_KEY environment variable must be set to a valid Fernet key. -# @POST Returns validated key bytes ready for Fernet initialization. -# @SIDE_EFFECT Emits belief-state logs for missing or invalid encryption configuration. -# @DATA_CONTRACT Input[ENCRYPTION_KEY:str] -> Output[bytes] -# @INVARIANT Encryption initialization never falls back to a hardcoded secret. +# @PRE: ENCRYPTION_KEY environment variable must be set to a valid Fernet key. +# @POST: Returns validated key bytes ready for Fernet initialization. # @RELATION DEPENDS_ON -> [backend.src.core.logger:Function] +# @SIDE_EFFECT: Emits belief-state logs for missing or invalid encryption configuration. +# @DATA_CONTRACT: Input[ENCRYPTION_KEY:str] -> Output[bytes] +# @INVARIANT: Encryption initialization never falls back to a hardcoded secret. def _require_fernet_key() -> bytes: with belief_scope("_require_fernet_key"): raw_key = os.getenv("ENCRYPTION_KEY", "").strip() @@ -52,12 +52,12 @@ def _require_fernet_key() -> bytes: # #region EncryptionManager [C:5] [TYPE Class] # @BRIEF Handles encryption and decryption of sensitive data like API keys. -# @PRE ENCRYPTION_KEY is configured with a valid Fernet key before instantiation. -# @POST Manager exposes reversible encrypt/decrypt operations for persisted secrets. -# @SIDE_EFFECT Initializes Fernet cryptography state from process environment. -# @DATA_CONTRACT Input[str] -> Output[str] -# @INVARIANT Uses only a validated secret key from environment. # @RELATION CALLS -> [_require_fernet_key] +# @PRE: ENCRYPTION_KEY is configured with a valid Fernet key before instantiation. +# @POST: Manager exposes reversible encrypt/decrypt operations for persisted secrets. +# @SIDE_EFFECT: Initializes Fernet cryptography state from process environment. +# @DATA_CONTRACT: Input[str] -> Output[str] +# @INVARIANT: Uses only a validated secret key from environment. # # @TEST_CONTRACT: EncryptionManagerModel -> # { @@ -71,35 +71,35 @@ def _require_fernet_key() -> bytes: # @TEST_EDGE: empty_string_encryption -> {"data": ""} # @TEST_INVARIANT: symmetric_encryption -> verifies: [basic_encryption_cycle, empty_string_encryption] class EncryptionManager: - # #region EncryptionManager_init [TYPE Function] - # @BRIEF Initialize the encryption manager with a Fernet key. - # @PRE ENCRYPTION_KEY env var must be set to a valid Fernet key. - # @POST Fernet instance ready for encryption/decryption. + # [DEF:EncryptionManager_init:Function] + # @PURPOSE: Initialize the encryption manager with a Fernet key. + # @PRE: ENCRYPTION_KEY env var must be set to a valid Fernet key. + # @POST: Fernet instance ready for encryption/decryption. def __init__(self): self.key = _require_fernet_key() self.fernet = Fernet(self.key) - # #endregion EncryptionManager_init + # [/DEF:EncryptionManager_init:Function] - # #region encrypt [TYPE Function] - # @BRIEF Encrypt a plaintext string. - # @PRE data must be a non-empty string. - # @POST Returns encrypted string. + # [DEF:encrypt:Function] + # @PURPOSE: Encrypt a plaintext string. + # @PRE: data must be a non-empty string. + # @POST: Returns encrypted string. def encrypt(self, data: str) -> str: with belief_scope("encrypt"): return self.fernet.encrypt(data.encode()).decode() - # #endregion encrypt + # [/DEF:encrypt:Function] - # #region decrypt [TYPE Function] - # @BRIEF Decrypt an encrypted string. - # @PRE encrypted_data must be a valid Fernet-encrypted string. - # @POST Returns original plaintext string. + # [DEF:decrypt:Function] + # @PURPOSE: Decrypt an encrypted string. + # @PRE: encrypted_data must be a valid Fernet-encrypted string. + # @POST: Returns original plaintext string. def decrypt(self, encrypted_data: str) -> str: with belief_scope("decrypt"): return self.fernet.decrypt(encrypted_data.encode()).decode() - # #endregion decrypt + # [/DEF:decrypt:Function] # #endregion EncryptionManager @@ -111,48 +111,51 @@ class EncryptionManager: # @RELATION DEPENDS_ON -> [EncryptionManager] # @RELATION DEPENDS_ON -> [LLMProviderConfig] class LLMProviderService: - # #region LLMProviderService_init [TYPE Function] - # @BRIEF Initialize the service with database session. - # @PRE db must be a valid SQLAlchemy Session. - # @POST Service ready for provider operations. - # @RELATION DEPENDS_ON -> [EncryptionManager] + # [DEF:LLMProviderService_init:Function] + # @PURPOSE: Initialize the service with database session. + # @PRE: db must be a valid SQLAlchemy Session. + # @POST: Service ready for provider operations. + # @RELATION: [DEPENDS_ON] ->[EncryptionManager] def __init__(self, db: Session): self.db = db self.encryption = EncryptionManager() - # #endregion LLMProviderService_init + # [/DEF:LLMProviderService_init:Function] - # #region get_all_providers [C:3] [TYPE Function] - # @BRIEF Returns all configured LLM providers. - # @PRE Database connection must be active. - # @POST Returns list of all LLMProvider records. - # @RELATION DEPENDS_ON -> [LLMProvider] + # [DEF:get_all_providers:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Returns all configured LLM providers. + # @PRE: Database connection must be active. + # @POST: Returns list of all LLMProvider records. + # @RELATION: [DEPENDS_ON] ->[LLMProvider] def get_all_providers(self) -> List[LLMProvider]: with belief_scope("get_all_providers"): return self.db.query(LLMProvider).all() - # #endregion get_all_providers + # [/DEF:get_all_providers:Function] - # #region get_provider [C:3] [TYPE Function] - # @BRIEF Returns a single LLM provider by ID. - # @PRE provider_id must be a valid string. - # @POST Returns LLMProvider or None if not found. - # @RELATION DEPENDS_ON -> [LLMProvider] + # [DEF:get_provider:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Returns a single LLM provider by ID. + # @PRE: provider_id must be a valid string. + # @POST: Returns LLMProvider or None if not found. + # @RELATION: [DEPENDS_ON] ->[LLMProvider] def get_provider(self, provider_id: str) -> Optional[LLMProvider]: with belief_scope("get_provider"): return ( self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first() ) - # #endregion get_provider + # [/DEF:get_provider:Function] - # #region create_provider [C:3] [TYPE Function] - # @BRIEF Creates a new LLM provider with encrypted API key. - # @PRE config must contain valid provider configuration. - # @POST New provider created and persisted to database. - # @RELATION DEPENDS_ON -> [LLMProviderConfig] - # @RELATION DEPENDS_ON -> [LLMProvider] - # @RELATION CALLS -> [encrypt] + # [DEF:create_provider:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Creates a new LLM provider with encrypted API key. + # @PRE: config must contain valid provider configuration. + # @POST: New provider created and persisted to database. + # @RELATION: [DEPENDS_ON] ->[LLMProviderConfig] + # @RELATION: [DEPENDS_ON] ->[LLMProvider] + # @RELATION: [CALLS] ->[encrypt] def create_provider(self, config: "LLMProviderConfig") -> LLMProvider: with belief_scope("create_provider"): encrypted_key = self.encryption.encrypt(config.api_key) @@ -169,15 +172,16 @@ class LLMProviderService: self.db.refresh(db_provider) return db_provider - # #endregion create_provider + # [/DEF:create_provider:Function] - # #region update_provider [C:3] [TYPE Function] - # @BRIEF Updates an existing LLM provider. - # @PRE provider_id must exist, config must be valid. - # @POST Provider updated and persisted to database. - # @RELATION DEPENDS_ON -> [LLMProviderConfig] - # @RELATION DEPENDS_ON -> [LLMProvider] - # @RELATION CALLS -> [encrypt] + # [DEF:update_provider:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Updates an existing LLM provider. + # @PRE: provider_id must exist, config must be valid. + # @POST: Provider updated and persisted to database. + # @RELATION: [DEPENDS_ON] ->[LLMProviderConfig] + # @RELATION: [DEPENDS_ON] ->[LLMProvider] + # @RELATION: [CALLS] ->[encrypt] def update_provider( self, provider_id: str, config: "LLMProviderConfig" ) -> Optional[LLMProvider]: @@ -203,13 +207,14 @@ class LLMProviderService: self.db.refresh(db_provider) return db_provider - # #endregion update_provider + # [/DEF:update_provider:Function] - # #region delete_provider [C:3] [TYPE Function] - # @BRIEF Deletes an LLM provider. - # @PRE provider_id must exist. - # @POST Provider removed from database. - # @RELATION DEPENDS_ON -> [LLMProvider] + # [DEF:delete_provider:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Deletes an LLM provider. + # @PRE: provider_id must exist. + # @POST: Provider removed from database. + # @RELATION: [DEPENDS_ON] ->[LLMProvider] def delete_provider(self, provider_id: str) -> bool: with belief_scope("delete_provider"): db_provider = self.get_provider(provider_id) @@ -219,14 +224,15 @@ class LLMProviderService: self.db.commit() return True - # #endregion delete_provider + # [/DEF:delete_provider:Function] - # #region get_decrypted_api_key [C:3] [TYPE Function] - # @BRIEF Returns the decrypted API key for a provider. - # @PRE provider_id must exist with valid encrypted key. - # @POST Returns decrypted API key or None on failure. - # @RELATION DEPENDS_ON -> [LLMProvider] - # @RELATION CALLS -> [decrypt] + # [DEF:get_decrypted_api_key:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Returns the decrypted API key for a provider. + # @PRE: provider_id must exist with valid encrypted key. + # @POST: Returns decrypted API key or None on failure. + # @RELATION: [DEPENDS_ON] ->[LLMProvider] + # @RELATION: [CALLS] ->[decrypt] def get_decrypted_api_key(self, provider_id: str) -> Optional[str]: with belief_scope("get_decrypted_api_key"): db_provider = self.get_provider(provider_id) @@ -251,7 +257,9 @@ class LLMProviderService: logger.error(f"[get_decrypted_api_key] Decryption failed: {str(e)}") return None - # #endregion get_decrypted_api_key + # [/DEF:get_decrypted_api_key:Function] # #endregion LLMProviderService + +# #endregion llm_provider diff --git a/backend/src/services/mapping_service.py b/backend/src/services/mapping_service.py index 8a7a9d0e..6f0c254c 100644 --- a/backend/src/services/mapping_service.py +++ b/backend/src/services/mapping_service.py @@ -1,36 +1,35 @@ # #region mapping_service [C:3] [TYPE Module] [SEMANTICS service, mapping, fuzzy-matching, superset] +# # @BRIEF Orchestrates database fetching and fuzzy matching suggestions. -# @LAYER Service -# @PRE source/target environment identifiers are provided by caller. -# @POST Exposes stateless mapping suggestion orchestration over configured environments. -# @SIDE_EFFECT Performs remote metadata reads through Superset API clients. -# @DATA_CONTRACT Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]] -# @INVARIANT Suggestions are based on database names. -# @RELATION DEPENDS_ON -> [SupersetClient] -# @RELATION DEPENDS_ON -> [suggest_mappings] -# +# @LAYER: Service +# @PRE: source/target environment identifiers are provided by caller. +# @POST: Exposes stateless mapping suggestion orchestration over configured environments. +# @SIDE_EFFECT: Performs remote metadata reads through Superset API clients. +# @DATA_CONTRACT: Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]] +# @RELATION DEPENDS_ON -> SupersetClient +# @RELATION DEPENDS_ON -> suggest_mappings # +# @INVARIANT: Suggestions are based on database names. -# [SECTION: IMPORTS] from typing import List, Dict from ..core.logger import belief_scope from ..core.superset_client import SupersetClient from ..core.utils.matching import suggest_mappings -# [/SECTION] # #region MappingService [C:3] [TYPE Class] # @BRIEF Service for handling database mapping logic. -# @PRE config_manager exposes get_environments() with environment objects containing id. -# @POST Provides client resolution and mapping suggestion methods. -# @SIDE_EFFECT Instantiates Superset clients and performs upstream metadata reads. -# @DATA_CONTRACT Input[config_manager] -> Output[List[Dict]] -# @RELATION DEPENDS_ON -> [SupersetClient] -# @RELATION DEPENDS_ON -> [suggest_mappings] +# @PRE: config_manager exposes get_environments() with environment objects containing id. +# @POST: Provides client resolution and mapping suggestion methods. +# @SIDE_EFFECT: Instantiates Superset clients and performs upstream metadata reads. +# @DATA_CONTRACT: Input[config_manager] -> Output[List[Dict]] +# @RELATION DEPENDS_ON -> SupersetClient +# @RELATION DEPENDS_ON -> suggest_mappings class MappingService: - # #region init [C:3] [TYPE Function] - # @BRIEF Initializes the mapping service with a config manager. - # @PRE config_manager is provided. + # [DEF:init:Function] + # @PURPOSE: Initializes the mapping service with a config manager. + # @COMPLEXITY: 3 + # @PRE: config_manager is provided. # @PARAM: config_manager (ConfigManager) - The configuration manager. # @POST: Service is initialized. # @RELATION: DEPENDS_ON -> MappingService @@ -38,10 +37,11 @@ class MappingService: with belief_scope("MappingService.__init__"): self.config_manager = config_manager - # #endregion init + # [/DEF:init:Function] - # #region _get_client [C:3] [TYPE Function] - # @BRIEF Helper to get an initialized SupersetClient for an environment. + # [DEF:_get_client:Function] + # @PURPOSE: Helper to get an initialized SupersetClient for an environment. + # @COMPLEXITY: 3 # @PARAM: env_id (str) - The ID of the environment. # @PRE: environment must exist in config. # @POST: Returns an initialized SupersetClient. @@ -56,10 +56,11 @@ class MappingService: return SupersetClient(env) - # #endregion _get_client + # [/DEF:_get_client:Function] - # #region get_suggestions [C:3] [TYPE Function] - # @BRIEF Fetches databases from both environments and returns fuzzy matching suggestions. + # [DEF:get_suggestions:Function] + # @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions. + # @COMPLEXITY: 3 # @PARAM: source_env_id (str) - Source environment ID. # @PARAM: target_env_id (str) - Target environment ID. # @PRE: Both environments must be accessible. @@ -85,7 +86,9 @@ class MappingService: return suggest_mappings(source_dbs, target_dbs) - # #endregion get_suggestions + # [/DEF:get_suggestions:Function] # #endregion MappingService + +# #endregion mapping_service diff --git a/backend/src/services/notifications/__tests__/test_notification_service.py b/backend/src/services/notifications/__tests__/test_notification_service.py index 6fd80c86..e4be5663 100644 --- a/backend/src/services/notifications/__tests__/test_notification_service.py +++ b/backend/src/services/notifications/__tests__/test_notification_service.py @@ -1,6 +1,7 @@ -# #region test_notification_service [C:2] [TYPE Module] -# @BRIEF Unit tests for NotificationService routing and dispatch logic. -# @RELATION TESTS -> [NotificationService:Class] +# [DEF:test_notification_service:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Unit tests for NotificationService routing and dispatch logic. +# @RELATION: TESTS ->[NotificationService:Class] import pytest from unittest.mock import MagicMock, AsyncMock, patch @@ -117,4 +118,4 @@ async def test_dispatch_report_calls_providers(service, mock_db): service._providers["TELEGRAM"].send.assert_called_once() service._providers["SMTP"].send.assert_called_once() -# #endregion test_notification_service +# [/DEF:test_notification_service:Module] \ No newline at end of file diff --git a/backend/src/services/notifications/providers.py b/backend/src/services/notifications/providers.py index e426c731..995fa6c2 100644 --- a/backend/src/services/notifications/providers.py +++ b/backend/src/services/notifications/providers.py @@ -1,20 +1,20 @@ # #region providers [C:5] [TYPE Module] [SEMANTICS notifications, providers, smtp, slack, telegram, abstraction] +# # @BRIEF Defines abstract base and concrete implementations for external notification delivery. -# @LAYER Infra -# @PRE Provider configuration dictionaries are supplied by trusted configuration sources. -# @POST Each provider exposes async send contract returning boolean delivery outcome. -# @SIDE_EFFECT Performs outbound network I/O to SMTP or HTTP endpoints. -# @DATA_CONTRACT Input[target, subject, body, context?] -> Output[bool] -# @INVARIANT Concrete providers preserve boolean send contract and swallow transport exceptions into False. -# @INVARIANT Providers must be stateless and resilient to network failures. -# @INVARIANT Sensitive credentials must be handled via encrypted config. # @RELATION DEPENDED_ON_BY -> [NotificationService] # @RELATION DEPENDS_ON -> [NotificationProvider] # @RELATION DEPENDS_ON -> [SMTPProvider] # @RELATION DEPENDS_ON -> [TelegramProvider] # @RELATION DEPENDS_ON -> [SlackProvider] +# @LAYER: Infra +# @PRE: Provider configuration dictionaries are supplied by trusted configuration sources. +# @POST: Each provider exposes async send contract returning boolean delivery outcome. +# @SIDE_EFFECT: Performs outbound network I/O to SMTP or HTTP endpoints. +# @DATA_CONTRACT: Input[target, subject, body, context?] -> Output[bool] +# @INVARIANT: Concrete providers preserve boolean send contract and swallow transport exceptions into False. # -# +# @INVARIANT: Providers must be stateless and resilient to network failures. +# @INVARIANT: Sensitive credentials must be handled via encrypted config. from abc import ABC, abstractmethod from typing import Any, Dict, Optional diff --git a/backend/src/services/notifications/service.py b/backend/src/services/notifications/service.py index ff4aa0be..0564c9c2 100644 --- a/backend/src/services/notifications/service.py +++ b/backend/src/services/notifications/service.py @@ -1,11 +1,7 @@ # #region service [C:5] [TYPE Module] [SEMANTICS notifications, service, routing, dispatch, background-tasks] +# # @BRIEF Orchestrates notification routing based on user preferences and policy context. -# @LAYER Domain -# @PRE channel_config is loaded -# @POST Notification dispatched via configured providers -# @SIDE_EFFECT Sends notifications via configured providers -# @DATA_CONTRACT NotificationChannelConfig -> NotificationRecipient -# @INVARIANT NotificationService maintains singleton pattern for per-channel notifications +# @LAYER: Domain # @RELATION DEPENDS_ON -> [NotificationProvider] # @RELATION DEPENDS_ON -> [SMTPProvider] # @RELATION DEPENDS_ON -> [TelegramProvider] @@ -14,7 +10,11 @@ # @RELATION DEPENDS_ON -> [ValidationPolicy] # @RELATION DEPENDS_ON -> [UserDashboardPreference] # -# +# @INVARIANT: NotificationService maintains singleton pattern for per-channel notifications +# @DATA_CONTRACT: NotificationChannelConfig -> NotificationRecipient +# @PRE: channel_config is loaded +# @POST: Notification dispatched via configured providers +# @SIDE_EFFECT: Sends notifications via configured providers from typing import Any, Dict, List, Optional from fastapi import BackgroundTasks @@ -34,32 +34,34 @@ from .providers import ( # #region NotificationService [C:4] [TYPE Class] # @BRIEF Routes validation reports to appropriate users and channels. -# @PRE Service receives a live DB session and configuration manager with notification payload settings. -# @POST Service can resolve targets and dispatch provider sends without mutating validation records. -# @SIDE_EFFECT Reads notification configuration, queries user preferences, and dispatches provider I/O. -# @DATA_CONTRACT Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None] # @RELATION DEPENDS_ON -> [NotificationProvider] # @RELATION DEPENDS_ON -> [ValidationRecord] # @RELATION DEPENDS_ON -> [ValidationPolicy] # @RELATION DEPENDS_ON -> [UserDashboardPreference] +# @PRE: Service receives a live DB session and configuration manager with notification payload settings. +# @POST: Service can resolve targets and dispatch provider sends without mutating validation records. +# @SIDE_EFFECT: Reads notification configuration, queries user preferences, and dispatches provider I/O. +# @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None] class NotificationService: - # #region NotificationService_init [C:3] [TYPE Function] - # @BRIEF Bind DB and configuration collaborators used for provider initialization and routing. - # @RELATION BINDS_TO -> [NotificationService] - # @RELATION DEPENDS_ON -> [ValidationPolicy] + # [DEF:NotificationService_init:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Bind DB and configuration collaborators used for provider initialization and routing. + # @RELATION: [BINDS_TO] ->[NotificationService] + # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] def __init__(self, db: Session, config_manager: ConfigManager): self.db = db self.config_manager = config_manager self._providers: Dict[str, NotificationProvider] = {} self._initialized = False - # #endregion NotificationService_init + # [/DEF:NotificationService_init:Function] - # #region _initialize_providers [C:3] [TYPE Function] - # @BRIEF Materialize configured notification channel adapters once per service lifetime. - # @RELATION DEPENDS_ON -> [SMTPProvider] - # @RELATION DEPENDS_ON -> [TelegramProvider] - # @RELATION DEPENDS_ON -> [SlackProvider] + # [DEF:_initialize_providers:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Materialize configured notification channel adapters once per service lifetime. + # @RELATION: [DEPENDS_ON] ->[SMTPProvider] + # @RELATION: [DEPENDS_ON] ->[TelegramProvider] + # @RELATION: [DEPENDS_ON] ->[SlackProvider] def _initialize_providers(self): if self._initialized: return @@ -78,18 +80,19 @@ class NotificationService: self._initialized = True - # #endregion _initialize_providers + # [/DEF:_initialize_providers:Function] - # #region dispatch_report [C:4] [TYPE Function] - # @BRIEF Route one validation record to resolved owners and configured custom channels. - # @PRE record is persisted and providers can be initialized from configuration payload. - # @POST Eligible notification sends are scheduled in background or awaited inline. - # @SIDE_EFFECT Schedules or performs outbound provider sends and emits notification logs. - # @DATA_CONTRACT Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None] - # @RELATION CALLS -> [_initialize_providers] - # @RELATION CALLS -> [_should_notify] - # @RELATION CALLS -> [_resolve_targets] - # @RELATION CALLS -> [_build_body] + # [DEF:dispatch_report:Function] + # @COMPLEXITY: 4 + # @PURPOSE: Route one validation record to resolved owners and configured custom channels. + # @RELATION: [CALLS] ->[_initialize_providers] + # @RELATION: [CALLS] ->[_should_notify] + # @RELATION: [CALLS] ->[_resolve_targets] + # @RELATION: [CALLS] ->[_build_body] + # @PRE: record is persisted and providers can be initialized from configuration payload. + # @POST: Eligible notification sends are scheduled in background or awaited inline. + # @SIDE_EFFECT: Schedules or performs outbound provider sends and emits notification logs. + # @DATA_CONTRACT: Input[ValidationRecord, Optional[ValidationPolicy], Optional[BackgroundTasks]] -> Output[None] async def dispatch_report( self, record: ValidationRecord, @@ -135,12 +138,13 @@ class NotificationService: # Fallback to sync for tests or if no background_tasks provided await provider.send(recipient, subject, body) - # #endregion dispatch_report + # [/DEF:dispatch_report:Function] - # #region _should_notify [C:3] [TYPE Function] - # @BRIEF Evaluate record status against effective alert policy. - # @RELATION DEPENDS_ON -> [ValidationRecord] - # @RELATION DEPENDS_ON -> [ValidationPolicy] + # [DEF:_should_notify:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Evaluate record status against effective alert policy. + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] def _should_notify( self, record: ValidationRecord, policy: Optional[ValidationPolicy] ) -> bool: @@ -152,13 +156,14 @@ class NotificationService: return record.status in ("WARN", "FAIL") return record.status == "FAIL" - # #endregion _should_notify + # [/DEF:_should_notify:Function] - # #region _resolve_targets [C:3] [TYPE Function] - # @BRIEF Resolve owner and policy-defined delivery targets for one validation record. - # @RELATION CALLS -> [_find_dashboard_owners] - # @RELATION DEPENDS_ON -> [ValidationRecord] - # @RELATION DEPENDS_ON -> [ValidationPolicy] + # [DEF:_resolve_targets:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Resolve owner and policy-defined delivery targets for one validation record. + # @RELATION: [CALLS] ->[_find_dashboard_owners] + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[ValidationPolicy] def _resolve_targets( self, record: ValidationRecord, policy: Optional[ValidationPolicy] ) -> List[tuple]: @@ -188,12 +193,13 @@ class NotificationService: return targets - # #endregion _resolve_targets + # [/DEF:_resolve_targets:Function] - # #region _find_dashboard_owners [C:3] [TYPE Function] - # @BRIEF Load candidate dashboard owners from persisted profile preferences. - # @RELATION DEPENDS_ON -> [ValidationRecord] - # @RELATION DEPENDS_ON -> [UserDashboardPreference] + # [DEF:_find_dashboard_owners:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Load candidate dashboard owners from persisted profile preferences. + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] + # @RELATION: [DEPENDS_ON] ->[UserDashboardPreference] def _find_dashboard_owners( self, record: ValidationRecord ) -> List[UserDashboardPreference]: @@ -209,11 +215,12 @@ class NotificationService: .all() ) - # #endregion _find_dashboard_owners + # [/DEF:_find_dashboard_owners:Function] - # #region _build_body [C:2] [TYPE Function] - # @BRIEF Format one validation record into provider-ready body text. - # @RELATION DEPENDS_ON -> [ValidationRecord] + # [DEF:_build_body:Function] + # @COMPLEXITY: 2 + # @PURPOSE: Format one validation record into provider-ready body text. + # @RELATION: [DEPENDS_ON] ->[ValidationRecord] def _build_body(self, record: ValidationRecord) -> str: return ( f"Dashboard ID: {record.dashboard_id}\n" @@ -223,7 +230,9 @@ class NotificationService: f"Issues found: {len(record.issues)}" ) - # #endregion _build_body + # [/DEF:_build_body:Function] # #endregion NotificationService + +# #endregion service diff --git a/backend/src/services/profile_service.py b/backend/src/services/profile_service.py index 243e8e13..47555811 100644 --- a/backend/src/services/profile_service.py +++ b/backend/src/services/profile_service.py @@ -1,7 +1,7 @@ # #region profile_service [C:5] [TYPE Module] [SEMANTICS profile, service, validation, ownership, filtering, superset, preferences] +# # @BRIEF Orchestrates profile preference persistence, Superset account lookup, and deterministic actor matching. -# @LAYER Domain -# @INVARIANT Profile ID needs to unique per-user session +# @LAYER: Domain # @RELATION DEPENDS_ON -> [UserDashboardPreference] # @RELATION DEPENDS_ON -> [ProfilePreferenceResponse] # @RELATION DEPENDS_ON -> [SupersetClient] @@ -9,7 +9,7 @@ # @RELATION DEPENDS_ON -> [User] # @RELATION DEPENDS_ON -> [sqlalchemy.orm.Session] # -# +# @INVARIANT: Profile ID needs to unique per-user session # # @TEST_CONTRACT: ProfilePreferenceUpdateRequest -> ProfilePreferenceResponse # @TEST_FIXTURE: valid_profile_update -> {"user_id":"u-1","superset_username":"John_Doe","show_only_my_dashboards":true} @@ -22,7 +22,6 @@ # @POST: Profile with updated fields populated and # @SIDE_EFFECT: Database read/write operations -# [SECTION: IMPORTS] from datetime import datetime from typing import Any, Iterable, List, Optional, Sequence, Set, Tuple from sqlalchemy.orm import Session @@ -45,15 +44,14 @@ from ..schemas.profile import ( SupersetAccountLookupResponse, SupersetAccountCandidate, ) -# [/SECTION] SUPPORTED_START_PAGES = {"dashboards", "datasets", "reports"} SUPPORTED_DENSITIES = {"compact", "comfortable"} # #region ProfileValidationError [C:2] [TYPE Class] +# @RELATION INHERITS -> Exception # @BRIEF Domain validation error for profile preference update requests. -# @RELATION INHERITS -> [Exception] class ProfileValidationError(Exception): def __init__(self, errors: Sequence[str]): self.errors = list(errors) @@ -64,8 +62,8 @@ class ProfileValidationError(Exception): # #region EnvironmentNotFoundError [C:2] [TYPE Class] +# @RELATION INHERITS -> Exception # @BRIEF Raised when environment_id from lookup request is unknown in app configuration. -# @RELATION INHERITS -> [Exception] class EnvironmentNotFoundError(Exception): pass @@ -74,8 +72,8 @@ class EnvironmentNotFoundError(Exception): # #region ProfileAuthorizationError [C:2] [TYPE Class] +# @RELATION INHERITS -> Exception # @BRIEF Raised when caller attempts cross-user preference mutation. -# @RELATION INHERITS -> [Exception] class ProfileAuthorizationError(Exception): pass @@ -84,24 +82,24 @@ class ProfileAuthorizationError(Exception): # #region ProfileService [C:5] [TYPE Class] -# @BRIEF Implements profile preference read/update flow and Superset account lookup degradation strategy. -# @PRE Caller provides authenticated User context for external service methods. -# @POST Preference operations remain user-scoped and return normalized profile/lookup responses. -# @SIDE_EFFECT Writes preference records and encrypted tokens; performs external account lookups when requested. -# @DATA_CONTRACT Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool] -# @INVARIANT Profile data integrity maintained, cache consistency with database state # @RELATION DEPENDS_ON -> [sqlalchemy.orm.Session] # @RELATION DEPENDS_ON -> [AuthRepository] # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter] # @RELATION DEPENDS_ON -> [UserDashboardPreference] # @RELATION CALLS -> [discover_declared_permissions] +# @BRIEF Implements profile preference read/update flow and Superset account lookup degradation strategy. +# @PRE: Caller provides authenticated User context for external service methods. +# @POST: Preference operations remain user-scoped and return normalized profile/lookup responses. +# @SIDE_EFFECT: Writes preference records and encrypted tokens; performs external account lookups when requested. +# @DATA_CONTRACT: Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool] +# @INVARIANT: Profile data integrity maintained, cache consistency with database state class ProfileService: - # #region init [TYPE Function] - # @BRIEF Initialize service with DB session and config manager. - # @PRE db session is active and config_manager supports get_environments(). - # @POST Service is ready for preference persistence and lookup operations. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:init:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Initialize service with DB session and config manager. + # @PRE: db session is active and config_manager supports get_environments(). + # @POST: Service is ready for preference persistence and lookup operations. def __init__(self, db: Session, config_manager: Any, plugin_loader: Any = None): self.db = db self.config_manager = config_manager @@ -109,13 +107,13 @@ class ProfileService: self.auth_repository = AuthRepository(db) self.encryption = EncryptionManager() - # #endregion init + # [/DEF:init:Function] - # #region get_my_preference [TYPE Function] - # @BRIEF Return current user's persisted preference or default non-configured view. - # @PRE current_user is authenticated. - # @POST Returned payload belongs to current_user only. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:get_my_preference:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Return current user's persisted preference or default non-configured view. + # @PRE: current_user is authenticated. + # @POST: Returned payload belongs to current_user only. def get_my_preference(self, current_user: User) -> ProfilePreferenceResponse: with belief_scope( "ProfileService.get_my_preference", f"user_id={current_user.id}" @@ -140,13 +138,13 @@ class ProfileService: security=security_summary, ) - # #endregion get_my_preference + # [/DEF:get_my_preference:Function] - # #region get_dashboard_filter_binding [TYPE Function] - # @BRIEF Return only dashboard-filter fields required by dashboards listing hot path. - # @PRE current_user is authenticated. - # @POST Returns normalized username and profile-default filter toggles without security summary expansion. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:get_dashboard_filter_binding:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Return only dashboard-filter fields required by dashboards listing hot path. + # @PRE: current_user is authenticated. + # @POST: Returns normalized username and profile-default filter toggles without security summary expansion. def get_dashboard_filter_binding(self, current_user: User) -> dict: with belief_scope( "ProfileService.get_dashboard_filter_binding", f"user_id={current_user.id}" @@ -175,13 +173,13 @@ class ProfileService: ), } - # #endregion get_dashboard_filter_binding + # [/DEF:get_dashboard_filter_binding:Function] - # #region update_my_preference [TYPE Function] - # @BRIEF Validate and persist current user's profile preference in self-scoped mode. - # @PRE current_user is authenticated and payload is provided. - # @POST Preference row for current_user is created/updated when validation passes. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:update_my_preference:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Validate and persist current user's profile preference in self-scoped mode. + # @PRE: current_user is authenticated and payload is provided. + # @POST: Preference row for current_user is created/updated when validation passes. def update_my_preference( self, current_user: User, @@ -324,13 +322,13 @@ class ProfileService: security=self._build_security_summary(current_user), ) - # #endregion update_my_preference + # [/DEF:update_my_preference:Function] - # #region lookup_superset_accounts [TYPE Function] - # @BRIEF Query Superset users in selected environment and project canonical account candidates. - # @PRE current_user is authenticated and environment_id exists. - # @POST Returns success payload or degraded payload with warning while preserving manual fallback. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:lookup_superset_accounts:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Query Superset users in selected environment and project canonical account candidates. + # @PRE: current_user is authenticated and environment_id exists. + # @POST: Returns success payload or degraded payload with warning while preserving manual fallback. def lookup_superset_accounts( self, current_user: User, @@ -406,13 +404,13 @@ class ProfileService: items=[], ) - # #endregion lookup_superset_accounts + # [/DEF:lookup_superset_accounts:Function] - # #region matches_dashboard_actor [TYPE Function] - # @BRIEF Apply trim+case-insensitive actor match across owners OR modified_by. - # @PRE bound_username can be empty; owners may contain mixed payload. - # @POST Returns True when normalized username matches owners or modified_by. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:matches_dashboard_actor:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Apply trim+case-insensitive actor match across owners OR modified_by. + # @PRE: bound_username can be empty; owners may contain mixed payload. + # @POST: Returns True when normalized username matches owners or modified_by. def matches_dashboard_actor( self, bound_username: Optional[str], @@ -432,13 +430,13 @@ class ProfileService: return True return False - # #endregion matches_dashboard_actor + # [/DEF:matches_dashboard_actor:Function] - # #region _build_security_summary [TYPE Function] - # @BRIEF Build read-only security snapshot with role and permission badges. - # @PRE current_user is authenticated. - # @POST Returns deterministic security projection for profile UI. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_build_security_summary:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Build read-only security snapshot with role and permission badges. + # @PRE: current_user is authenticated. + # @POST: Returns deterministic security projection for profile UI. def _build_security_summary(self, current_user: User) -> ProfileSecuritySummary: role_names_set: Set[str] = set() roles = getattr(current_user, "roles", []) or [] @@ -496,13 +494,13 @@ class ProfileService: permissions=permission_states, ) - # #endregion _build_security_summary + # [/DEF:_build_security_summary:Function] - # #region _collect_user_permission_pairs [TYPE Function] - # @BRIEF Collect effective permission tuples from current user's roles. - # @PRE current_user can include role/permission graph. - # @POST Returns unique normalized (resource, ACTION) tuples. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_collect_user_permission_pairs:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Collect effective permission tuples from current user's roles. + # @PRE: current_user can include role/permission graph. + # @POST: Returns unique normalized (resource, ACTION) tuples. def _collect_user_permission_pairs( self, current_user: User ) -> Set[Tuple[str, str]]: @@ -517,13 +515,13 @@ class ProfileService: collected.add((resource, action)) return collected - # #endregion _collect_user_permission_pairs + # [/DEF:_collect_user_permission_pairs:Function] - # #region _format_permission_key [TYPE Function] - # @BRIEF Convert normalized permission pair to compact UI key. - # @PRE resource and action are normalized. - # @POST Returns user-facing badge key. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_format_permission_key:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Convert normalized permission pair to compact UI key. + # @PRE: resource and action are normalized. + # @POST: Returns user-facing badge key. def _format_permission_key(self, resource: str, action: str) -> str: normalized_resource = self._sanitize_text(resource) or "" normalized_action = str(action or "").strip().upper() @@ -531,13 +529,13 @@ class ProfileService: return normalized_resource return f"{normalized_resource}:{normalized_action.lower()}" - # #endregion _format_permission_key + # [/DEF:_format_permission_key:Function] - # #region _to_preference_payload [TYPE Function] - # @BRIEF Map ORM preference row to API DTO with token metadata. - # @PRE preference row can contain nullable optional fields. - # @POST Returns normalized ProfilePreference object. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_to_preference_payload:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Map ORM preference row to API DTO with token metadata. + # @PRE: preference row can contain nullable optional fields. + # @POST: Returns normalized ProfilePreference object. def _to_preference_payload( self, preference: UserDashboardPreference, @@ -591,13 +589,13 @@ class ProfileService: updated_at=updated_at, ) - # #endregion _to_preference_payload + # [/DEF:_to_preference_payload:Function] - # #region _mask_secret_value [TYPE Function] - # @BRIEF Build a safe display value for sensitive secrets. - # @PRE secret may be None or plaintext. - # @POST Returns masked representation or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_mask_secret_value:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Build a safe display value for sensitive secrets. + # @PRE: secret may be None or plaintext. + # @POST: Returns masked representation or None. def _mask_secret_value(self, secret: Optional[str]) -> Optional[str]: sanitized_secret = self._sanitize_secret(secret) if sanitized_secret is None: @@ -606,26 +604,26 @@ class ProfileService: return "***" return f"{sanitized_secret[:2]}***{sanitized_secret[-2:]}" - # #endregion _mask_secret_value + # [/DEF:_mask_secret_value:Function] - # #region _sanitize_text [TYPE Function] - # @BRIEF Normalize optional text into trimmed form or None. - # @PRE value may be empty or None. - # @POST Returns trimmed value or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_sanitize_text:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize optional text into trimmed form or None. + # @PRE: value may be empty or None. + # @POST: Returns trimmed value or None. def _sanitize_text(self, value: Optional[str]) -> Optional[str]: normalized = str(value or "").strip() if not normalized: return None return normalized - # #endregion _sanitize_text + # [/DEF:_sanitize_text:Function] - # #region _sanitize_secret [TYPE Function] - # @BRIEF Normalize secret input into trimmed form or None. - # @PRE value may be None or blank. - # @POST Returns trimmed secret or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_sanitize_secret:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize secret input into trimmed form or None. + # @PRE: value may be None or blank. + # @POST: Returns trimmed secret or None. def _sanitize_secret(self, value: Optional[str]) -> Optional[str]: if value is None: return None @@ -634,13 +632,13 @@ class ProfileService: return None return normalized - # #endregion _sanitize_secret + # [/DEF:_sanitize_secret:Function] - # #region _normalize_start_page [TYPE Function] - # @BRIEF Normalize supported start page aliases to canonical values. - # @PRE value may be None or alias. - # @POST Returns one of SUPPORTED_START_PAGES. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_normalize_start_page:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize supported start page aliases to canonical values. + # @PRE: value may be None or alias. + # @POST: Returns one of SUPPORTED_START_PAGES. def _normalize_start_page(self, value: Optional[str]) -> str: normalized = str(value or "").strip().lower() if normalized == "reports-logs": @@ -649,13 +647,13 @@ class ProfileService: return normalized return "dashboards" - # #endregion _normalize_start_page + # [/DEF:_normalize_start_page:Function] - # #region _normalize_density [TYPE Function] - # @BRIEF Normalize supported density aliases to canonical values. - # @PRE value may be None or alias. - # @POST Returns one of SUPPORTED_DENSITIES. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_normalize_density:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize supported density aliases to canonical values. + # @PRE: value may be None or alias. + # @POST: Returns one of SUPPORTED_DENSITIES. def _normalize_density(self, value: Optional[str]) -> str: normalized = str(value or "").strip().lower() if normalized == "free": @@ -664,13 +662,13 @@ class ProfileService: return normalized return "comfortable" - # #endregion _normalize_density + # [/DEF:_normalize_density:Function] - # #region _resolve_environment [TYPE Function] - # @BRIEF Resolve environment model from configured environments by id. - # @PRE environment_id is provided. - # @POST Returns environment object when found else None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_resolve_environment:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Resolve environment model from configured environments by id. + # @PRE: environment_id is provided. + # @POST: Returns environment object when found else None. def _resolve_environment(self, environment_id: str): environments = self.config_manager.get_environments() for env in environments: @@ -678,36 +676,36 @@ class ProfileService: return env return None - # #endregion _resolve_environment + # [/DEF:_resolve_environment:Function] - # #region _get_preference_row [TYPE Function] - # @BRIEF Return persisted preference row for user or None. - # @PRE user_id is provided. - # @POST Returns matching row or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_get_preference_row:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Return persisted preference row for user or None. + # @PRE: user_id is provided. + # @POST: Returns matching row or None. def _get_preference_row(self, user_id: str) -> Optional[UserDashboardPreference]: return self.auth_repository.get_user_dashboard_preference(str(user_id)) - # #endregion _get_preference_row + # [/DEF:_get_preference_row:Function] - # #region _get_or_create_preference_row [TYPE Function] - # @BRIEF Return existing preference row or create new unsaved row. - # @PRE user_id is provided. - # @POST Returned row always contains user_id. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_get_or_create_preference_row:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Return existing preference row or create new unsaved row. + # @PRE: user_id is provided. + # @POST: Returned row always contains user_id. def _get_or_create_preference_row(self, user_id: str) -> UserDashboardPreference: existing = self._get_preference_row(user_id) if existing is not None: return existing return UserDashboardPreference(user_id=str(user_id)) - # #endregion _get_or_create_preference_row + # [/DEF:_get_or_create_preference_row:Function] - # #region _build_default_preference [TYPE Function] - # @BRIEF Build non-persisted default preference DTO for unconfigured users. - # @PRE user_id is provided. - # @POST Returns ProfilePreference with disabled toggle and empty username. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_build_default_preference:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Build non-persisted default preference DTO for unconfigured users. + # @PRE: user_id is provided. + # @POST: Returns ProfilePreference with disabled toggle and empty username. def _build_default_preference(self, user_id: str) -> ProfilePreference: now = datetime.utcnow() return ProfilePreference( @@ -730,13 +728,13 @@ class ProfileService: updated_at=now, ) - # #endregion _build_default_preference + # [/DEF:_build_default_preference:Function] - # #region _validate_update_payload [TYPE Function] - # @BRIEF Validate username/toggle constraints for preference mutation. - # @PRE payload is provided. - # @POST Returns validation errors list; empty list means valid. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_validate_update_payload:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Validate username/toggle constraints for preference mutation. + # @PRE: payload is provided. + # @POST: Returns validation errors list; empty list means valid. def _validate_update_payload( self, superset_username: Optional[str], @@ -786,36 +784,36 @@ class ProfileService: return errors - # #endregion _validate_update_payload + # [/DEF:_validate_update_payload:Function] - # #region _sanitize_username [TYPE Function] - # @BRIEF Normalize raw username into trimmed form or None for empty input. - # @PRE value can be empty or None. - # @POST Returns trimmed username or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_sanitize_username:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize raw username into trimmed form or None for empty input. + # @PRE: value can be empty or None. + # @POST: Returns trimmed username or None. def _sanitize_username(self, value: Optional[str]) -> Optional[str]: return self._sanitize_text(value) - # #endregion _sanitize_username + # [/DEF:_sanitize_username:Function] - # #region _normalize_username [TYPE Function] - # @BRIEF Apply deterministic trim+lower normalization for actor matching. - # @PRE value can be empty or None. - # @POST Returns lowercase normalized token or None. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_normalize_username:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Apply deterministic trim+lower normalization for actor matching. + # @PRE: value can be empty or None. + # @POST: Returns lowercase normalized token or None. def _normalize_username(self, value: Optional[str]) -> Optional[str]: sanitized = self._sanitize_username(value) if sanitized is None: return None return sanitized.lower() - # #endregion _normalize_username + # [/DEF:_normalize_username:Function] - # #region _normalize_owner_tokens [TYPE Function] - # @BRIEF Normalize owners payload into deduplicated lower-cased tokens. - # @PRE owners can be iterable of scalars or dict-like values. - # @POST Returns list of unique normalized owner tokens. - # @RELATION BINDS_TO -> [ProfileService] + # [DEF:_normalize_owner_tokens:Function] + # @RELATION: BINDS_TO -> ProfileService + # @PURPOSE: Normalize owners payload into deduplicated lower-cased tokens. + # @PRE: owners can be iterable of scalars or dict-like values. + # @POST: Returns list of unique normalized owner tokens. def _normalize_owner_tokens(self, owners: Optional[Iterable[Any]]) -> List[str]: if owners is None: return [] @@ -851,7 +849,7 @@ class ProfileService: normalized.append(token) return normalized - # #endregion _normalize_owner_tokens + # [/DEF:_normalize_owner_tokens:Function] # #endregion ProfileService diff --git a/backend/src/services/rbac_permission_catalog.py b/backend/src/services/rbac_permission_catalog.py index 491308e7..40ec3240 100644 --- a/backend/src/services/rbac_permission_catalog.py +++ b/backend/src/services/rbac_permission_catalog.py @@ -1,8 +1,7 @@ # #region rbac_permission_catalog [C:2] [TYPE Module] -# @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database. # +# @BRIEF Discovers declared RBAC permissions from API routes/plugins and synchronizes them into auth database. -# [SECTION: IMPORTS] import re from functools import lru_cache from pathlib import Path @@ -29,9 +28,8 @@ ROUTES_DIR = Path(__file__).resolve().parent.parent / "api" / "routes" # #region _iter_route_files [TYPE Function] # @BRIEF Iterates API route files that may contain RBAC declarations. -# @PRE ROUTES_DIR points to backend/src/api/routes. -# @POST Yields Python files excluding test and cache directories. -# @RETURN Iterable[Path] - Route file paths for permission extraction. +# @PRE: ROUTES_DIR points to backend/src/api/routes. +# @POST: Yields Python files excluding test and cache directories. def _iter_route_files() -> Iterable[Path]: with belief_scope("rbac_permission_catalog._iter_route_files"): if not ROUTES_DIR.exists(): @@ -49,9 +47,8 @@ def _iter_route_files() -> Iterable[Path]: # #region _discover_route_permissions [TYPE Function] # @BRIEF Extracts explicit has_permission declarations from API route source code. -# @PRE Route files are readable UTF-8 text files. -# @POST Returns unique set of (resource, action) pairs declared in route guards. -# @RETURN Set[Tuple[str, str]] - Permission pairs from route-level RBAC declarations. +# @PRE: Route files are readable UTF-8 text files. +# @POST: Returns unique set of (resource, action) pairs declared in route guards. def _discover_route_permissions() -> Set[Tuple[str, str]]: with belief_scope("rbac_permission_catalog._discover_route_permissions"): discovered: Set[Tuple[str, str]] = set() @@ -77,8 +74,8 @@ def _discover_route_permissions() -> Set[Tuple[str, str]]: # #region _discover_route_permissions_cached [TYPE Function] # @BRIEF Cache route permission discovery because route source files are static during normal runtime. -# @PRE None. -# @POST Returns stable discovered route permission pairs without repeated filesystem scans. +# @PRE: None. +# @POST: Returns stable discovered route permission pairs without repeated filesystem scans. @lru_cache(maxsize=1) def _discover_route_permissions_cached() -> Tuple[Tuple[str, str], ...]: with belief_scope("rbac_permission_catalog._discover_route_permissions_cached"): @@ -88,9 +85,8 @@ def _discover_route_permissions_cached() -> Tuple[Tuple[str, str], ...]: # #region _discover_plugin_execute_permissions [TYPE Function] # @BRIEF Derives dynamic task permissions of form plugin:{plugin_id}:EXECUTE from plugin registry. -# @PRE plugin_loader is optional and may expose get_all_plugin_configs. -# @POST Returns unique plugin EXECUTE permissions if loader is available. -# @RETURN Set[Tuple[str, str]] - Permission pairs derived from loaded plugin IDs. +# @PRE: plugin_loader is optional and may expose get_all_plugin_configs. +# @POST: Returns unique plugin EXECUTE permissions if loader is available. def _discover_plugin_execute_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: with belief_scope("rbac_permission_catalog._discover_plugin_execute_permissions"): discovered: Set[Tuple[str, str]] = set() @@ -116,8 +112,8 @@ def _discover_plugin_execute_permissions(plugin_loader=None) -> Set[Tuple[str, s # #region _discover_plugin_execute_permissions_cached [TYPE Function] # @BRIEF Cache dynamic plugin EXECUTE permission pairs by normalized plugin id tuple. -# @PRE plugin_ids is a deterministic tuple of plugin ids. -# @POST Returns stable permission tuple without repeated plugin catalog expansion. +# @PRE: plugin_ids is a deterministic tuple of plugin ids. +# @POST: Returns stable permission tuple without repeated plugin catalog expansion. @lru_cache(maxsize=8) def _discover_plugin_execute_permissions_cached( plugin_ids: Tuple[str, ...], @@ -129,9 +125,8 @@ def _discover_plugin_execute_permissions_cached( # #region discover_declared_permissions [TYPE Function] # @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry. -# @PRE plugin_loader may be provided for dynamic task plugin permission discovery. -# @POST Returns union of route-declared and dynamic plugin EXECUTE permissions. -# @RETURN Set[Tuple[str, str]] - Complete discovered permission set. +# @PRE: plugin_loader may be provided for dynamic task plugin permission discovery. +# @POST: Returns union of route-declared and dynamic plugin EXECUTE permissions. def discover_declared_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: with belief_scope("rbac_permission_catalog.discover_declared_permissions"): permissions = set(_discover_route_permissions_cached()) @@ -151,11 +146,10 @@ def discover_declared_permissions(plugin_loader=None) -> Set[Tuple[str, str]]: # #region sync_permission_catalog [TYPE Function] # @BRIEF Persists missing RBAC permission pairs into auth database. -# @PRE db is a valid SQLAlchemy session bound to auth database. -# @PRE declared_permissions is an iterable of (resource, action) tuples. -# @POST Missing permissions are inserted; existing permissions remain untouched. -# @SIDE_EFFECT Commits auth database transaction when new permissions are added. -# @RETURN int - Number of inserted permission records. +# @PRE: db is a valid SQLAlchemy session bound to auth database. +# @PRE: declared_permissions is an iterable of (resource, action) tuples. +# @POST: Missing permissions are inserted; existing permissions remain untouched. +# @SIDE_EFFECT: Commits auth database transaction when new permissions are added. def sync_permission_catalog( db: Session, declared_permissions: Iterable[Tuple[str, str]], diff --git a/backend/src/services/reports/__tests__/test_report_normalizer.py b/backend/src/services/reports/__tests__/test_report_normalizer.py index aa226839..5562ed4e 100644 --- a/backend/src/services/reports/__tests__/test_report_normalizer.py +++ b/backend/src/services/reports/__tests__/test_report_normalizer.py @@ -1,8 +1,10 @@ -# #region test_report_normalizer [C:2] [TYPE Module] [SEMANTICS tests, reports, normalizer, fallback] -# @BRIEF Validate unknown task type fallback and partial payload normalization behavior. -# @LAYER Domain (Tests) -# @INVARIANT Unknown plugin types are mapped to canonical unknown task type. -# @RELATION TESTS -> [normalize_report:Function] +# [DEF:test_report_normalizer:Module] +# @COMPLEXITY: 2 +# @SEMANTICS: tests, reports, normalizer, fallback +# @PURPOSE: Validate unknown task type fallback and partial payload normalization behavior. +# @RELATION: TESTS ->[normalize_report:Function] +# @LAYER: Domain (Tests) +# @INVARIANT: Unknown plugin types are mapped to canonical unknown task type. from datetime import datetime @@ -10,9 +12,9 @@ from src.core.task_manager.models import Task, TaskStatus from src.services.reports.normalizer import normalize_task_report -# #region test_unknown_type_maps_to_unknown_profile [TYPE Function] -# @BRIEF Ensure unknown plugin IDs map to unknown profile with populated summary and error context. -# @RELATION BINDS_TO -> [test_report_normalizer] +# [DEF:test_unknown_type_maps_to_unknown_profile:Function] +# @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure unknown plugin IDs map to unknown profile with populated summary and error context. def test_unknown_type_maps_to_unknown_profile(): task = Task( id="unknown-1", @@ -31,12 +33,12 @@ def test_unknown_type_maps_to_unknown_profile(): assert report.error_context is not None -# #endregion test_unknown_type_maps_to_unknown_profile +# [/DEF:test_unknown_type_maps_to_unknown_profile:Function] -# #region test_partial_payload_keeps_report_visible_with_placeholders [TYPE Function] -# @BRIEF Ensure missing result payload still yields visible report details with result placeholder. -# @RELATION BINDS_TO -> [test_report_normalizer] +# [DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function] +# @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure missing result payload still yields visible report details with result placeholder. def test_partial_payload_keeps_report_visible_with_placeholders(): task = Task( id="partial-1", @@ -55,12 +57,12 @@ def test_partial_payload_keeps_report_visible_with_placeholders(): assert "result" in report.details -# #endregion test_partial_payload_keeps_report_visible_with_placeholders +# [/DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function] -# #region test_clean_release_plugin_maps_to_clean_release_task_type [TYPE Function] -# @BRIEF Ensure clean-release plugin ID maps to clean_release task profile and summary passthrough. -# @RELATION BINDS_TO -> [test_report_normalizer] +# [DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function] +# @RELATION: BINDS_TO -> test_report_normalizer +# @PURPOSE: Ensure clean-release plugin ID maps to clean_release task profile and summary passthrough. def test_clean_release_plugin_maps_to_clean_release_task_type(): task = Task( id="clean-release-1", @@ -78,5 +80,5 @@ def test_clean_release_plugin_maps_to_clean_release_task_type(): assert report.summary == "Clean release compliance passed" -# #endregion test_clean_release_plugin_maps_to_clean_release_task_type -# #endregion test_report_normalizer +# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function] +# [/DEF:test_report_normalizer:Module] diff --git a/backend/src/services/reports/__tests__/test_report_service.py b/backend/src/services/reports/__tests__/test_report_service.py index 5f033879..49560848 100644 --- a/backend/src/services/reports/__tests__/test_report_service.py +++ b/backend/src/services/reports/__tests__/test_report_service.py @@ -1,7 +1,8 @@ -# #region test_report_service [C:2] [TYPE Module] -# @BRIEF Unit tests for ReportsService list/detail operations -# @LAYER Domain -# @RELATION TESTS -> [ReportsService:Class] +# [DEF:test_report_service:Module] +# @COMPLEXITY: 2 +# @PURPOSE: Unit tests for ReportsService list/detail operations +# @RELATION: TESTS ->[ReportsService:Class] +# @LAYER: Domain import sys from pathlib import Path @@ -12,8 +13,8 @@ from unittest.mock import MagicMock, patch from datetime import datetime, timezone, timedelta -# #region _make_task [TYPE Function] -# @RELATION BINDS_TO -> [test_report_service] +# [DEF:_make_task:Function] +# @RELATION: BINDS_TO -> test_report_service def _make_task(task_id="task-1", plugin_id="superset-backup", status_value="SUCCESS", started_at=None, finished_at=None, result=None, params=None, logs=None): """Create a mock Task object matching the Task model interface.""" @@ -29,7 +30,7 @@ def _make_task(task_id="task-1", plugin_id="superset-backup", status_value="SUCC return task -# #endregion _make_task +# [/DEF:_make_task:Function] class TestReportsServiceList: """Tests for ReportsService.list_reports.""" @@ -183,4 +184,4 @@ class TestReportsServiceDetail: detail = svc.get_report_detail("ok-task") assert detail.next_actions == [] -# #endregion test_report_service +# [/DEF:test_report_service:Module] diff --git a/backend/src/services/reports/__tests__/test_type_profiles.py b/backend/src/services/reports/__tests__/test_type_profiles.py index 0fae7b9a..74f3fdad 100644 --- a/backend/src/services/reports/__tests__/test_type_profiles.py +++ b/backend/src/services/reports/__tests__/test_type_profiles.py @@ -1,7 +1,7 @@ -# #region __tests__/test_report_type_profiles [TYPE Module] -# @BRIEF Contract testing for task type profiles and resolution logic. -# @RELATION VERIFIES -> [../type_profiles.py] -# #endregion __tests__/test_report_type_profiles +# [DEF:__tests__/test_report_type_profiles:Module] +# @RELATION: VERIFIES -> ../type_profiles.py +# @PURPOSE: Contract testing for task type profiles and resolution logic. +# [/DEF:__tests__/test_report_type_profiles:Module] import pytest from src.models.report import TaskType @@ -9,9 +9,9 @@ from src.services.reports.type_profiles import resolve_task_type, get_type_profi # @TEST_CONTRACT: ResolveTaskType -> Invariants # @TEST_INVARIANT: fallback_to_unknown -# #region test_resolve_task_type_fallbacks [TYPE Function] -# @BRIEF Verify resolve_task_type_fallbacks returns correct fallback type when primary is missing. -# @RELATION BINDS_TO -> [__tests__/test_report_type_profiles] +# [DEF:test_resolve_task_type_fallbacks:Function] +# @RELATION: BINDS_TO -> __tests__/test_report_type_profiles +# @PURPOSE: Verify resolve_task_type_fallbacks returns correct fallback type when primary is missing. def test_resolve_task_type_fallbacks(): """Verify missing/unmapped plugin_id returns TaskType.UNKNOWN.""" assert resolve_task_type(None) == TaskType.UNKNOWN @@ -20,11 +20,11 @@ def test_resolve_task_type_fallbacks(): assert resolve_task_type("invalid_plugin") == TaskType.UNKNOWN # @TEST_FIXTURE: valid_plugin -# #endregion test_resolve_task_type_fallbacks +# [/DEF:test_resolve_task_type_fallbacks:Function] -# #region test_resolve_task_type_valid [TYPE Function] -# @BRIEF Verify resolve_task_type_valid returns the correct type when valid input is provided. -# @RELATION BINDS_TO -> [__tests__/test_report_type_profiles] +# [DEF:test_resolve_task_type_valid:Function] +# @RELATION: BINDS_TO -> __tests__/test_report_type_profiles +# @PURPOSE: Verify resolve_task_type_valid returns the correct type when valid input is provided. def test_resolve_task_type_valid(): """Verify known plugin IDs map correctly.""" assert resolve_task_type("superset-migration") == TaskType.MIGRATION @@ -33,11 +33,11 @@ def test_resolve_task_type_valid(): assert resolve_task_type("documentation") == TaskType.DOCUMENTATION # @TEST_FIXTURE: valid_profile -# #endregion test_resolve_task_type_valid +# [/DEF:test_resolve_task_type_valid:Function] -# #region test_get_type_profile_valid [TYPE Function] -# @BRIEF Verify get_type_profile_valid returns the correct profile for a valid task type. -# @RELATION BINDS_TO -> [__tests__/test_report_type_profiles] +# [DEF:test_get_type_profile_valid:Function] +# @RELATION: BINDS_TO -> __tests__/test_report_type_profiles +# @PURPOSE: Verify get_type_profile_valid returns the correct profile for a valid task type. def test_get_type_profile_valid(): """Verify known task types return correct profile metadata.""" profile = get_type_profile(TaskType.MIGRATION) @@ -47,11 +47,11 @@ def test_get_type_profile_valid(): # @TEST_INVARIANT: always_returns_dict # @TEST_EDGE: missing_profile -# #endregion test_get_type_profile_valid +# [/DEF:test_get_type_profile_valid:Function] -# #region test_get_type_profile_fallback [TYPE Function] -# @BRIEF Verify get_type_profile_fallback returns default profile when type is unknown. -# @RELATION BINDS_TO -> [__tests__/test_report_type_profiles] +# [DEF:test_get_type_profile_fallback:Function] +# @RELATION: BINDS_TO -> __tests__/test_report_type_profiles +# @PURPOSE: Verify get_type_profile_fallback returns default profile when type is unknown. def test_get_type_profile_fallback(): """Verify unknown task type returns fallback profile.""" # Assuming TaskType.UNKNOWN or any non-mapped value @@ -63,4 +63,4 @@ def test_get_type_profile_fallback(): profile_fallback = get_type_profile("non-enum-value") assert profile_fallback["display_label"] == "Other / Unknown" assert profile_fallback["fallback"] is True -# #endregion test_get_type_profile_fallback +# [/DEF:test_get_type_profile_fallback:Function] diff --git a/backend/src/services/reports/normalizer.py b/backend/src/services/reports/normalizer.py index 0f65d194..abbcf7de 100644 --- a/backend/src/services/reports/normalizer.py +++ b/backend/src/services/reports/normalizer.py @@ -1,16 +1,15 @@ # #region normalizer [C:5] [TYPE Module] [SEMANTICS reports, normalization, tasks, fallback] # @BRIEF Convert task manager task objects into canonical unified TaskReport entities with deterministic fallback behavior. -# @LAYER Domain -# @PRE session is active and valid -# @POST Returns Normalizer output with normalized fields -# @SIDE_EFFECT Read-only database operations -# @DATA_CONTRACT ReportRow -> NormalizerInput; session_id -> valid UUID -# @INVARIANT Normalizer instance maintains consistent field order +# @LAYER: Domain # @RELATION DEPENDS_ON -> [backend.src.core.task_manager.models.Task:Function] # @RELATION DEPENDS_ON -> [backend.src.models.report:Function] # @RELATION DEPENDS_ON -> [backend.src.services.reports.type_profiles:Function] +# @INVARIANT: Normalizer instance maintains consistent field order +# @DATA_CONTRACT: ReportRow -> NormalizerInput; session_id -> valid UUID +# @PRE: session is active and valid +# @POST: Returns Normalizer output with normalized fields +# @SIDE_EFFECT: Read-only database operations -# [SECTION: IMPORTS] from datetime import datetime from typing import Any, Dict, Optional @@ -18,15 +17,12 @@ from ...core.logger import belief_scope from ...core.task_manager.models import Task, TaskStatus from ...models.report import ErrorContext, ReportStatus, TaskReport from .type_profiles import get_type_profile, resolve_task_type -# [/SECTION] # #region status_to_report_status [TYPE Function] # @BRIEF Normalize internal task status to canonical report status. -# @PRE status may be known or unknown string/enum value. -# @POST Always returns one of canonical ReportStatus values. -# @PARAM: status (Any) - Internal task status value. -# @RETURN: ReportStatus - Canonical report status. +# @PRE: status may be known or unknown string/enum value. +# @POST: Always returns one of canonical ReportStatus values. def status_to_report_status(status: Any) -> ReportStatus: with belief_scope("status_to_report_status"): raw = str(status.value if isinstance(status, TaskStatus) else status).upper() @@ -42,11 +38,8 @@ def status_to_report_status(status: Any) -> ReportStatus: # #region build_summary [TYPE Function] # @BRIEF Build deterministic user-facing summary from task payload and status. -# @PRE report_status is canonical; plugin_id may be unknown. -# @POST Returns non-empty summary text. -# @PARAM: task (Task) - Source task object. -# @PARAM: report_status (ReportStatus) - Canonical status. -# @RETURN: str - Normalized summary. +# @PRE: report_status is canonical; plugin_id may be unknown. +# @POST: Returns non-empty summary text. def build_summary(task: Task, report_status: ReportStatus) -> str: with belief_scope("build_summary"): result = task.result @@ -67,11 +60,8 @@ def build_summary(task: Task, report_status: ReportStatus) -> str: # #region extract_error_context [TYPE Function] # @BRIEF Extract normalized error context and next actions for failed/partial reports. -# @PRE task is a valid Task object. -# @POST Returns ErrorContext for failed/partial when context exists; otherwise None. -# @PARAM: task (Task) - Source task. -# @PARAM: report_status (ReportStatus) - Canonical status. -# @RETURN: Optional[ErrorContext] - Error context block. +# @PRE: task is a valid Task object. +# @POST: Returns ErrorContext for failed/partial when context exists; otherwise None. def extract_error_context(task: Task, report_status: ReportStatus) -> Optional[ErrorContext]: with belief_scope("extract_error_context"): if report_status not in {ReportStatus.FAILED, ReportStatus.PARTIAL}: @@ -111,10 +101,8 @@ def extract_error_context(task: Task, report_status: ReportStatus) -> Optional[E # #region normalize_task_report [TYPE Function] # @BRIEF Convert one Task to canonical TaskReport envelope. -# @PRE task has valid id and plugin_id fields. -# @POST Returns TaskReport with required fields and deterministic fallback behavior. -# @PARAM: task (Task) - Source task. -# @RETURN: TaskReport - Canonical normalized report. +# @PRE: task has valid id and plugin_id fields. +# @POST: Returns TaskReport with required fields and deterministic fallback behavior. # # @TEST_CONTRACT: NormalizeTaskReport -> # { @@ -170,4 +158,4 @@ def normalize_task_report(task: Task) -> TaskReport: ) # #endregion normalize_task_report -# #endregion normalizer +# #endregion normalizer \ No newline at end of file diff --git a/backend/src/services/reports/report_service.py b/backend/src/services/reports/report_service.py index ab975d23..50a0c1a3 100644 --- a/backend/src/services/reports/report_service.py +++ b/backend/src/services/reports/report_service.py @@ -1,11 +1,6 @@ # #region report_service [C:5] [TYPE Module] [SEMANTICS reports, service, aggregation, filtering, pagination, detail] # @BRIEF Aggregate, normalize, filter, and paginate task reports for unified list/detail API use cases. -# @LAYER Domain -# @PRE session is active and valid -# @POST Returns Report with generated summary -# @SIDE_EFFECT Read-only database operations; logs report generation -# @DATA_CONTRACT ReportQuery -> ReportRow; session_id -> valid UUID -# @INVARIANT ReportService maintains consistent report structure +# @LAYER: Domain # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [TaskReport] # @RELATION DEPENDS_ON -> [ReportQuery] @@ -13,8 +8,12 @@ # @RELATION DEPENDS_ON -> [ReportDetailView] # @RELATION DEPENDS_ON -> [normalize_task_report] # @RELATION DEPENDS_ON -> [CleanReleaseRepository] +# @INVARIANT: ReportService maintains consistent report structure +# @DATA_CONTRACT: ReportQuery -> ReportRow; session_id -> valid UUID +# @PRE: session is active and valid +# @POST: Returns Report with generated summary +# @SIDE_EFFECT: Read-only database operations; logs report generation -# [SECTION: IMPORTS] from datetime import datetime, timezone from typing import List, Optional @@ -31,19 +30,18 @@ from ...models.report import ( ) from ..clean_release.repository import CleanReleaseRepository from .normalizer import normalize_task_report -# [/SECTION] # #region ReportsService [C:5] [TYPE Class] # @BRIEF Service layer for list/detail report retrieval and normalization. -# @PRE TaskManager dependency is initialized. -# @POST Provides deterministic list/detail report responses. -# @SIDE_EFFECT Reads task history and optional clean-release repository state without mutating source records. -# @DATA_CONTRACT Input[TaskManager, Optional[CleanReleaseRepository], ReportQuery|report_id] -> Output[ReportCollection|ReportDetailView|None] -# @INVARIANT Service methods are read-only over task history source. +# @PRE: TaskManager dependency is initialized. +# @POST: Provides deterministic list/detail report responses. # @RELATION DEPENDS_ON -> [TaskManager] # @RELATION DEPENDS_ON -> [CleanReleaseRepository] # @RELATION CALLS -> [normalize_task_report] +# @SIDE_EFFECT: Reads task history and optional clean-release repository state without mutating source records. +# @DATA_CONTRACT: Input[TaskManager, Optional[CleanReleaseRepository], ReportQuery|report_id] -> Output[ReportCollection|ReportDetailView|None] +# @INVARIANT: Service methods are read-only over task history source. # # @TEST_CONTRACT: ReportsServiceModel -> # { @@ -58,16 +56,17 @@ from .normalizer import normalize_task_report # @TEST_EDGE: report_not_found -> get_report_detail returns None # @TEST_INVARIANT: consistent_pagination -> verifies: [valid_service] class ReportsService: - # #region init [C:5] [TYPE Function] - # @BRIEF Initialize service with TaskManager dependency. - # @PRE task_manager is a live TaskManager instance. - # @POST self.task_manager is assigned and ready for read operations. - # @SIDE_EFFECT Stores collaborator references for later read-only report projections. - # @DATA_CONTRACT Input[TaskManager, Optional[CleanReleaseRepository]] -> Output[ReportsService] - # @INVARIANT Constructor performs no task mutations. - # @RELATION BINDS_TO -> [ReportsService] - # @RELATION DEPENDS_ON -> [TaskManager] - # @RELATION DEPENDS_ON -> [CleanReleaseRepository] + # [DEF:init:Function] + # @COMPLEXITY: 5 + # @PURPOSE: Initialize service with TaskManager dependency. + # @PRE: task_manager is a live TaskManager instance. + # @POST: self.task_manager is assigned and ready for read operations. + # @INVARIANT: Constructor performs no task mutations. + # @RELATION: [BINDS_TO] ->[ReportsService] + # @RELATION: [DEPENDS_ON] ->[TaskManager] + # @RELATION: [DEPENDS_ON] ->[CleanReleaseRepository] + # @SIDE_EFFECT: Stores collaborator references for later read-only report projections. + # @DATA_CONTRACT: Input[TaskManager, Optional[CleanReleaseRepository]] -> Output[ReportsService] # @PARAM: task_manager (TaskManager) - Task manager providing source task history. def __init__( self, @@ -78,27 +77,27 @@ class ReportsService: self.task_manager = task_manager self.clean_release_repository = clean_release_repository - # #endregion init + # [/DEF:init:Function] - # #region _load_normalized_reports [TYPE Function] - # @BRIEF Build normalized reports from all available tasks. - # @PRE Task manager returns iterable task history records. - # @POST Returns normalized report list preserving source cardinality. - # @INVARIANT Every returned item is a TaskReport. - # @RETURN List[TaskReport] - Reports sorted later by list logic. + # [DEF:_load_normalized_reports:Function] + # @PURPOSE: Build normalized reports from all available tasks. + # @PRE: Task manager returns iterable task history records. + # @POST: Returns normalized report list preserving source cardinality. + # @INVARIANT: Every returned item is a TaskReport. + # @RETURN: List[TaskReport] - Reports sorted later by list logic. def _load_normalized_reports(self) -> List[TaskReport]: with belief_scope("_load_normalized_reports"): tasks = self.task_manager.get_all_tasks() reports = [normalize_task_report(task) for task in tasks] return reports - # #endregion _load_normalized_reports + # [/DEF:_load_normalized_reports:Function] - # #region _to_utc_datetime [TYPE Function] - # @BRIEF Normalize naive/aware datetime values to UTC-aware datetime for safe comparisons. - # @PRE value is either datetime or None. - # @POST Returns UTC-aware datetime or None. - # @INVARIANT Naive datetimes are interpreted as UTC to preserve deterministic ordering/filtering. + # [DEF:_to_utc_datetime:Function] + # @PURPOSE: Normalize naive/aware datetime values to UTC-aware datetime for safe comparisons. + # @PRE: value is either datetime or None. + # @POST: Returns UTC-aware datetime or None. + # @INVARIANT: Naive datetimes are interpreted as UTC to preserve deterministic ordering/filtering. # @PARAM: value (Optional[datetime]) - Source datetime value. # @RETURN: Optional[datetime] - UTC-aware datetime or None. def _to_utc_datetime(self, value: Optional[datetime]) -> Optional[datetime]: @@ -109,13 +108,13 @@ class ReportsService: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) - # #endregion _to_utc_datetime + # [/DEF:_to_utc_datetime:Function] - # #region _datetime_sort_key [TYPE Function] - # @BRIEF Produce stable numeric sort key for report timestamps. - # @PRE report contains updated_at datetime. - # @POST Returns float timestamp suitable for deterministic sorting. - # @INVARIANT Mixed naive/aware datetimes never raise TypeError. + # [DEF:_datetime_sort_key:Function] + # @PURPOSE: Produce stable numeric sort key for report timestamps. + # @PRE: report contains updated_at datetime. + # @POST: Returns float timestamp suitable for deterministic sorting. + # @INVARIANT: Mixed naive/aware datetimes never raise TypeError. # @PARAM: report (TaskReport) - Report item. # @RETURN: float - UTC timestamp key. def _datetime_sort_key(self, report: TaskReport) -> float: @@ -125,13 +124,13 @@ class ReportsService: return 0.0 return updated.timestamp() - # #endregion _datetime_sort_key + # [/DEF:_datetime_sort_key:Function] - # #region _matches_query [TYPE Function] - # @BRIEF Apply query filtering to a report. - # @PRE report and query are normalized schema instances. - # @POST Returns True iff report satisfies all active query filters. - # @INVARIANT Filter evaluation is side-effect free. + # [DEF:_matches_query:Function] + # @PURPOSE: Apply query filtering to a report. + # @PRE: report and query are normalized schema instances. + # @POST: Returns True iff report satisfies all active query filters. + # @INVARIANT: Filter evaluation is side-effect free. # @PARAM: report (TaskReport) - Candidate report. # @PARAM: query (ReportQuery) - Applied query. # @RETURN: bool - True if report matches all filters. @@ -164,13 +163,13 @@ class ReportsService: return False return True - # #endregion _matches_query + # [/DEF:_matches_query:Function] - # #region _sort_reports [TYPE Function] - # @BRIEF Sort reports deterministically according to query settings. - # @PRE reports contains only TaskReport items. - # @POST Returns reports ordered by selected sort field and order. - # @INVARIANT Sorting criteria are deterministic for equal input. + # [DEF:_sort_reports:Function] + # @PURPOSE: Sort reports deterministically according to query settings. + # @PRE: reports contains only TaskReport items. + # @POST: Returns reports ordered by selected sort field and order. + # @INVARIANT: Sorting criteria are deterministic for equal input. # @PARAM: reports (List[TaskReport]) - Filtered reports. # @PARAM: query (ReportQuery) - Sort config. # @RETURN: List[TaskReport] - Sorted reports. @@ -189,12 +188,12 @@ class ReportsService: return reports - # #endregion _sort_reports + # [/DEF:_sort_reports:Function] - # #region list_reports [TYPE Function] - # @BRIEF Return filtered, sorted, paginated report collection. - # @PRE query has passed schema validation. - # @POST Returns {items,total,page,page_size,has_next,applied_filters}. + # [DEF:list_reports:Function] + # @PURPOSE: Return filtered, sorted, paginated report collection. + # @PRE: query has passed schema validation. + # @POST: Returns {items,total,page,page_size,has_next,applied_filters}. # @PARAM: query (ReportQuery) - List filters and pagination. # @RETURN: ReportCollection - Paginated unified reports payload. def list_reports(self, query: ReportQuery) -> ReportCollection: @@ -220,12 +219,12 @@ class ReportsService: applied_filters=query, ) - # #endregion list_reports + # [/DEF:list_reports:Function] - # #region get_report_detail [TYPE Function] - # @BRIEF Return one normalized report with timeline/diagnostics/next actions. - # @PRE report_id exists in normalized report set. - # @POST Returns normalized detail envelope with diagnostics and next actions where applicable. + # [DEF:get_report_detail:Function] + # @PURPOSE: Return one normalized report with timeline/diagnostics/next actions. + # @PRE: report_id exists in normalized report set. + # @POST: Returns normalized detail envelope with diagnostics and next actions where applicable. # @PARAM: report_id (str) - Stable report identifier. # @RETURN: Optional[ReportDetailView] - Detailed report or None if not found. def get_report_detail(self, report_id: str) -> Optional[ReportDetailView]: @@ -298,8 +297,9 @@ class ReportsService: next_actions=next_actions, ) - # #endregion get_report_detail + # [/DEF:get_report_detail:Function] # #endregion ReportsService + # #endregion report_service diff --git a/backend/src/services/reports/type_profiles.py b/backend/src/services/reports/type_profiles.py index da958dfe..9951611a 100644 --- a/backend/src/services/reports/type_profiles.py +++ b/backend/src/services/reports/type_profiles.py @@ -1,12 +1,10 @@ # #region type_profiles [C:2] [TYPE Module] # @BRIEF Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata. -# [SECTION: IMPORTS] from typing import Any, Dict, Optional from ...core.logger import belief_scope from ...models.report import TaskType -# [/SECTION] # #region PLUGIN_TO_TASK_TYPE [TYPE Data] # @BRIEF Maps plugin identifiers to normalized report task types. @@ -71,10 +69,8 @@ TASK_TYPE_PROFILES: Dict[TaskType, Dict[str, Any]] = { # #region resolve_task_type [TYPE Function] # @BRIEF Resolve canonical task type from plugin/task identifier with guaranteed fallback. -# @PRE plugin_id may be None or unknown. -# @POST Always returns one of TaskType enum values. -# @PARAM: plugin_id (Optional[str]) - Source plugin/task identifier from task record. -# @RETURN: TaskType - Resolved canonical type or UNKNOWN fallback. +# @PRE: plugin_id may be None or unknown. +# @POST: Always returns one of TaskType enum values. # # @TEST_CONTRACT: ResolveTaskType -> # { @@ -99,10 +95,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType: # #region get_type_profile [TYPE Function] # @BRIEF Return deterministic profile metadata for a task type. -# @PRE task_type may be known or unknown. -# @POST Returns a profile dict and never raises for unknown types. -# @PARAM: task_type (TaskType) - Canonical task type. -# @RETURN: Dict[str, Any] - Profile metadata used by normalization and UI contracts. +# @PRE: task_type may be known or unknown. +# @POST: Returns a profile dict and never raises for unknown types. # # @TEST_CONTRACT: GetTypeProfile -> # { diff --git a/backend/src/services/resource_service.py b/backend/src/services/resource_service.py index bce4ac93..ccfcaff6 100644 --- a/backend/src/services/resource_service.py +++ b/backend/src/services/resource_service.py @@ -1,44 +1,41 @@ # #region ResourceServiceModule [C:3] [TYPE Module] [SEMANTICS service, resources, dashboards, datasets, tasks, git] # @BRIEF Shared service for fetching resource data with Git status and task status -# @LAYER Service -# @INVARIANT All resources include metadata about their current state +# @LAYER: Service # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [TaskManagerPackage] # @RELATION DEPENDS_ON -> [TaskManagerModels] # @RELATION DEPENDS_ON -> [GitService] +# @INVARIANT: All resources include metadata about their current state -# [SECTION: IMPORTS] from typing import List, Dict, Optional, Any from datetime import datetime, timezone from ..core.superset_client import SupersetClient from ..core.task_manager.models import Task from ..services.git_service import GitService -from ..core.cot_logger import MarkerLogger from ..core.logger import logger, belief_scope -log = MarkerLogger("ResourceService") -# [/SECTION] - # #region ResourceService [C:3] [TYPE Class] # @BRIEF Provides centralized access to resource data with enhanced metadata # @RELATION DEPENDS_ON -> [SupersetClient] # @RELATION DEPENDS_ON -> [GitService] class ResourceService: - # #region ResourceService_init [C:1] [TYPE Function] - # @BRIEF Initialize the resource service with dependencies - # @PRE None - # @POST ResourceService is ready to fetch resources + # [DEF:ResourceService_init:Function] + # @COMPLEXITY: 1 + # @PURPOSE: Initialize the resource service with dependencies + # @PRE: None + # @POST: ResourceService is ready to fetch resources def __init__(self): with belief_scope("ResourceService.__init__"): self.git_service = GitService() - log.reason("Initialized ResourceService") - # #endregion ResourceService_init + logger.info("[ResourceService][Action] Initialized ResourceService") + # [/DEF:ResourceService_init:Function] - # #region get_dashboards_with_status [C:3] [TYPE Function] - # @BRIEF Fetch dashboards from environment with Git status and last task status - # @PRE env is a valid Environment object - # @POST Returns list of dashboards with enhanced metadata + # [DEF:get_dashboards_with_status:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch dashboards from environment with Git status and last task status + # @PRE: env is a valid Environment object + # @POST: Returns list of dashboards with enhanced metadata # @PARAM: env (Environment) - The environment to fetch from # @PARAM: tasks (List[Task]) - List of tasks to check for status # @RETURN: List[Dict] - Dashboards with git_status and last_task fields @@ -80,14 +77,15 @@ class ResourceService: result.append(dashboard_dict) - log.reflect(f"Fetched {len(result)} dashboards with status") + logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status") return result - # #endregion get_dashboards_with_status + # [/DEF:get_dashboards_with_status:Function] - # #region get_dashboards_page_with_status [C:3] [TYPE Function] - # @BRIEF Fetch one dashboard page from environment and enrich only that page with status metadata. - # @PRE env is valid; page >= 1; page_size > 0. - # @POST Returns page items plus total counters without scanning all pages locally. + # [DEF:get_dashboards_page_with_status:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch one dashboard page from environment and enrich only that page with status metadata. + # @PRE: env is valid; page >= 1; page_size > 0. + # @POST: Returns page items plus total counters without scanning all pages locally. # @PARAM: env (Environment) - Source environment. # @PARAM: tasks (Optional[List[Task]]) - Tasks for latest LLM status. # @PARAM: page (int) - 1-based page number. @@ -136,18 +134,25 @@ class ResourceService: result.append(dashboard_dict) total_pages = (total + page_size - 1) // page_size if total > 0 else 1 - log.reflect(f"Fetched dashboards page {page}/{total_pages} ({len(result)} items, total={total})") + logger.info( + "[ResourceService][Coherence:OK] Fetched dashboards page %s/%s (%s items, total=%s)", + page, + total_pages, + len(result), + total, + ) return { "dashboards": result, "total": total, "total_pages": total_pages, } - # #endregion get_dashboards_page_with_status + # [/DEF:get_dashboards_page_with_status:Function] - # #region _get_last_llm_task_for_dashboard [C:3] [TYPE Function] - # @BRIEF Get most recent LLM validation task for a dashboard in an environment - # @PRE dashboard_id is a valid integer identifier - # @POST Returns the newest llm_dashboard_validation task summary or None + # [DEF:_get_last_llm_task_for_dashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get most recent LLM validation task for a dashboard in an environment + # @PRE: dashboard_id is a valid integer identifier + # @POST: Returns the newest llm_dashboard_validation task summary or None # @PARAM: dashboard_id (int) - The dashboard ID # @PARAM: env_id (Optional[str]) - Environment ID to match task params # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search @@ -225,12 +230,13 @@ class ResourceService: "status": self._normalize_task_status(getattr(latest_task, "status", "")), "validation_status": validation_status, } - # #endregion _get_last_llm_task_for_dashboard + # [/DEF:_get_last_llm_task_for_dashboard:Function] - # #region _normalize_task_status [C:3] [TYPE Function] - # @BRIEF Normalize task status to stable uppercase values for UI/API projections - # @PRE raw_status can be enum or string - # @POST Returns uppercase status without enum class prefix + # [DEF:_normalize_task_status:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize task status to stable uppercase values for UI/API projections + # @PRE: raw_status can be enum or string + # @POST: Returns uppercase status without enum class prefix # @PARAM: raw_status (Any) - Raw task status object/value # @RETURN: str - Normalized status token # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard] @@ -242,12 +248,13 @@ class ResourceService: if "." in status_text: status_text = status_text.split(".")[-1] return status_text.upper() - # #endregion _normalize_task_status + # [/DEF:_normalize_task_status:Function] - # #region _normalize_validation_status [C:3] [TYPE Function] - # @BRIEF Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN - # @PRE raw_status can be any scalar type - # @POST Returns normalized validation status token or None + # [DEF:_normalize_validation_status:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize LLM validation status to PASS/FAIL/WARN/UNKNOWN + # @PRE: raw_status can be any scalar type + # @POST: Returns normalized validation status token or None # @PARAM: raw_status (Any) - Raw validation status from task result # @RETURN: Optional[str] - PASS|FAIL|WARN|UNKNOWN # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard] @@ -258,12 +265,13 @@ class ResourceService: if status_text in {"PASS", "FAIL", "WARN"}: return status_text return "UNKNOWN" - # #endregion _normalize_validation_status + # [/DEF:_normalize_validation_status:Function] - # #region _normalize_datetime_for_compare [C:3] [TYPE Function] - # @BRIEF Normalize datetime values to UTC-aware values for safe comparisons. - # @PRE value may be datetime or any scalar. - # @POST Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime. + # [DEF:_normalize_datetime_for_compare:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Normalize datetime values to UTC-aware values for safe comparisons. + # @PRE: value may be datetime or any scalar. + # @POST: Returns UTC-aware datetime; non-datetime values map to minimal UTC datetime. # @PARAM: value (Any) - Candidate datetime-like value. # @RETURN: datetime - UTC-aware comparable datetime. # @RELATION: USED_BY ->[_get_last_llm_task_for_dashboard] @@ -274,12 +282,13 @@ class ResourceService: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) return datetime.min.replace(tzinfo=timezone.utc) - # #endregion _normalize_datetime_for_compare + # [/DEF:_normalize_datetime_for_compare:Function] - # #region get_datasets_with_status [C:3] [TYPE Function] - # @BRIEF Fetch datasets from environment with mapping progress and last task status - # @PRE env is a valid Environment object - # @POST Returns list of datasets with enhanced metadata + # [DEF:get_datasets_with_status:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Fetch datasets from environment with mapping progress and last task status + # @PRE: env is a valid Environment object + # @POST: Returns list of datasets with enhanced metadata # @PARAM: env (Environment) - The environment to fetch from # @PARAM: tasks (List[Task]) - List of tasks to check for status # @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields @@ -310,14 +319,15 @@ class ResourceService: result.append(dataset_dict) - log.reflect(f"Fetched {len(result)} datasets with status") + logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status") return result - # #endregion get_datasets_with_status + # [/DEF:get_datasets_with_status:Function] - # #region get_activity_summary [C:3] [TYPE Function] - # @BRIEF Get summary of active and recent tasks for the activity indicator - # @PRE tasks is a list of Task objects - # @POST Returns summary with active_count and recent_tasks + # [DEF:get_activity_summary:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get summary of active and recent tasks for the activity indicator + # @PRE: tasks is a list of Task objects + # @POST: Returns summary with active_count and recent_tasks # @PARAM: tasks (List[Task]) - List of tasks to summarize # @RETURN: Dict - Activity summary # @RELATION: CALLS ->[_extract_resource_name_from_task] @@ -353,12 +363,13 @@ class ResourceService: 'active_count': len(active_tasks), 'recent_tasks': recent_tasks_formatted } - # #endregion get_activity_summary + # [/DEF:get_activity_summary:Function] - # #region _get_git_status_for_dashboard [C:3] [TYPE Function] - # @BRIEF Get Git sync status for a dashboard - # @PRE dashboard_id is a valid integer - # @POST Returns git status or None if no repo exists + # [DEF:_get_git_status_for_dashboard:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get Git sync status for a dashboard + # @PRE: dashboard_id is a valid integer + # @POST: Returns git status or None if no repo exists # @PARAM: dashboard_id (int) - The dashboard ID # @RETURN: Optional[Dict] - Git status with branch and sync_status # @RELATION: CALLS ->[get_repo] @@ -397,7 +408,7 @@ class ResourceService: 'has_changes_for_commit': has_changes_for_commit } except Exception: - log.explore(f"Failed to get git status for dashboard {dashboard_id}", error="Git status check failed") + logger.warning(f"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}") return { 'branch': None, 'sync_status': 'ERROR', @@ -412,12 +423,13 @@ class ResourceService: 'has_repo': False, 'has_changes_for_commit': False } - # #endregion _get_git_status_for_dashboard + # [/DEF:_get_git_status_for_dashboard:Function] - # #region _get_last_task_for_resource [C:3] [TYPE Function] - # @BRIEF Get the most recent task for a specific resource - # @PRE resource_id is a valid string - # @POST Returns task summary or None if no tasks found + # [DEF:_get_last_task_for_resource:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Get the most recent task for a specific resource + # @PRE: resource_id is a valid string + # @POST: Returns task summary or None if no tasks found # @PARAM: resource_id (str) - The resource identifier (e.g., "dashboard-123") # @PARAM: tasks (Optional[List[Task]]) - List of tasks to search # @RETURN: Optional[Dict] - Task summary with task_id and status @@ -450,29 +462,32 @@ class ResourceService: 'task_id': str(last_task.id), 'status': last_task.status } - # #endregion _get_last_task_for_resource + # [/DEF:_get_last_task_for_resource:Function] - # #region _extract_resource_name_from_task [C:3] [TYPE Function] - # @BRIEF Extract resource name from task params - # @PRE task is a valid Task object - # @POST Returns resource name or task ID + # [DEF:_extract_resource_name_from_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract resource name from task params + # @PRE: task is a valid Task object + # @POST: Returns resource name or task ID # @PARAM: task (Task) - The task to extract from # @RETURN: str - Resource name or fallback # @RELATION: USED_BY ->[get_activity_summary] def _extract_resource_name_from_task(self, task: Task) -> str: params = task.params or {} return params.get('resource_name', f"Task {task.id}") - # #endregion _extract_resource_name_from_task + # [/DEF:_extract_resource_name_from_task:Function] - # #region _extract_resource_type_from_task [C:3] [TYPE Function] - # @BRIEF Extract resource type from task params - # @PRE task is a valid Task object - # @POST Returns resource type or 'unknown' + # [DEF:_extract_resource_type_from_task:Function] + # @COMPLEXITY: 3 + # @PURPOSE: Extract resource type from task params + # @PRE: task is a valid Task object + # @POST: Returns resource type or 'unknown' # @PARAM: task (Task) - The task to extract from # @RETURN: str - Resource type # @RELATION: USED_BY ->[get_activity_summary] def _extract_resource_type_from_task(self, task: Task) -> str: params = task.params or {} return params.get('resource_type', 'unknown') - # #endregion _extract_resource_type_from_task + # [/DEF:_extract_resource_type_from_task:Function] # #endregion ResourceService +# #endregion ResourceServiceModule diff --git a/frontend/src/lib/api/__tests__/reports_api.test.js b/frontend/src/lib/api/__tests__/reports_api.test.js index 7c6073a2..48ba814b 100644 --- a/frontend/src/lib/api/__tests__/reports_api.test.js +++ b/frontend/src/lib/api/__tests__/reports_api.test.js @@ -1,3 +1,7 @@ +// #region ReportsApiTest [C:3] [TYPE Module] [SEMANTICS test,reports,api] +// @BRIEF Unit tests for reports API client functions: query string building, error normalization, and fetch wrappers. +// @RELATION DEPENDS_ON -> [ReportsApi] + // [DEF:ReportsApiTest:Module] // @COMPLEXITY: 3 // @SEMANTICS: tests, reports, api-client, query-string, error-normalization @@ -211,3 +215,4 @@ describe('getReportDetail', () => { // [/DEF:TestGetReportsAsync:Class] // [/DEF:ReportsApiTest:Module] +// #endregion ReportsApiTest diff --git a/frontend/src/lib/auth/__tests__/permissions.test.js b/frontend/src/lib/auth/__tests__/permissions.test.js index 77d9886a..87ee1325 100644 --- a/frontend/src/lib/auth/__tests__/permissions.test.js +++ b/frontend/src/lib/auth/__tests__/permissions.test.js @@ -1,3 +1,7 @@ +// #region PermissionsTest [C:3] [TYPE Module] [SEMANTICS test,auth,permissions,rbac] +// @BRIEF Verifies frontend RBAC permission parsing and access checks. +// @RELATION DEPENDS_ON -> [Permissions] + // [DEF:PermissionsTest:Module] // @COMPLEXITY: 3 // @SEMANTICS: tests, auth, permissions, rbac @@ -101,3 +105,4 @@ describe("auth.permissions", () => { }); // [/DEF:PermissionsTest:Module] +// #endregion PermissionsTest diff --git a/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.js b/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.js index 6edd552c..6745aabc 100644 --- a/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.js +++ b/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.js @@ -47,14 +47,14 @@ describe('AssistantChatPanel integration contract', () => { it('contains semantic anchors and UX contract tags', () => { const source = fs.readFileSync(COMPONENT_PATH, 'utf-8'); - expect(source).toContain(''); + expect(source).toContain(''); + expect(source).toContain(''); }); it('keeps confirmation/task-tracking action hooks in place', () => { diff --git a/frontend/src/lib/stores/__tests__/mocks/env_public.js b/frontend/src/lib/stores/__tests__/mocks/env_public.js index 341de4ae..86c7aa86 100644 --- a/frontend/src/lib/stores/__tests__/mocks/env_public.js +++ b/frontend/src/lib/stores/__tests__/mocks/env_public.js @@ -1,3 +1,5 @@ +// #region MockEnvPublic [C:1] [TYPE Module] [SEMANTICS mock,env] + // [DEF:mock_env_public:Module] // @RELATION: DEPENDS_ON -> [$env/static/public] // @COMPLEXITY: 3 @@ -5,3 +7,4 @@ // @LAYER: UI (Tests) export const PUBLIC_WS_URL = 'ws://localhost:8000'; // [/DEF:mock_env_public:Module] +// #endregion MockEnvPublic diff --git a/frontend/src/lib/stores/__tests__/mocks/environment.js b/frontend/src/lib/stores/__tests__/mocks/environment.js index 1f9d8339..5582f31e 100644 --- a/frontend/src/lib/stores/__tests__/mocks/environment.js +++ b/frontend/src/lib/stores/__tests__/mocks/environment.js @@ -1,3 +1,6 @@ +// #region EnvironmentMock [C:2] [TYPE Module] [SEMANTICS mock,environment] +// @BRIEF Mock for $app/environment in vitest, supplying browser, dev, and building flags. + // [DEF:EnvironmentMock:Module] // @COMPLEXITY: 2 // @PURPOSE: Mock for $app/environment in tests @@ -9,3 +12,4 @@ export const dev = true; export const building = false; // [/DEF:EnvironmentMock:Module] +// #endregion EnvironmentMock diff --git a/frontend/src/lib/stores/__tests__/mocks/navigation.js b/frontend/src/lib/stores/__tests__/mocks/navigation.js index 287b3be7..949e566b 100644 --- a/frontend/src/lib/stores/__tests__/mocks/navigation.js +++ b/frontend/src/lib/stores/__tests__/mocks/navigation.js @@ -1,3 +1,6 @@ +// #region NavigationMock [C:2] [TYPE Module] [SEMANTICS mock,navigation] +// @BRIEF Mock for $app/navigation in vitest, supplying goto, push, replace, prefetch, and prefetchRoutes stubs. + // [DEF:NavigationMock:Module] // @COMPLEXITY: 1 // @SEMANTICS: mock, navigation, sveltekit-v1-legacy-surface @@ -13,3 +16,4 @@ export const prefetch = () => Promise.resolve(); export const prefetchRoutes = () => Promise.resolve(); // [/DEF:NavigationMock:Module] +// #endregion NavigationMock diff --git a/frontend/src/lib/stores/__tests__/mocks/state.js b/frontend/src/lib/stores/__tests__/mocks/state.js index 6fe22814..960fa5d1 100644 --- a/frontend/src/lib/stores/__tests__/mocks/state.js +++ b/frontend/src/lib/stores/__tests__/mocks/state.js @@ -1,3 +1,6 @@ +// #region StateMock [C:2] [TYPE Module] [SEMANTICS mock,state] +// @BRIEF Mock for $app/state page data in vitest route/component tests, supplying default params, route, URL, status, and data. + // [DEF:state_mock:Module] // @COMPLEXITY: 2 // @PURPOSE: Mock for AppState in vitest route/component tests. @@ -13,4 +16,5 @@ export const page = { form: null, }; -// [/DEF:state_mock:Module] \ No newline at end of file +// [/DEF:state_mock:Module] +// #endregion StateMock \ No newline at end of file diff --git a/frontend/src/lib/stores/__tests__/sidebar.test.js b/frontend/src/lib/stores/__tests__/sidebar.test.js index a576ea96..ad0a1457 100644 --- a/frontend/src/lib/stores/__tests__/sidebar.test.js +++ b/frontend/src/lib/stores/__tests__/sidebar.test.js @@ -1,3 +1,7 @@ +// #region SidebarTest [C:3] [TYPE Module] [SEMANTICS test,sidebar,store,mobile,navigation] +// @BRIEF Unit tests for sidebar store validating expansion toggles, active item selection, and mobile open/close transitions. +// @RELATION DEPENDS_ON -> [sidebar] + // [DEF:SidebarTest:Module] // @COMPLEXITY: 3 // @SEMANTICS: sidebar, store, tests, mobile, navigation @@ -132,3 +136,4 @@ describe('SidebarStore', () => { }); // [/DEF:SidebarTest:Module] +// #endregion SidebarTest diff --git a/frontend/src/lib/stores/__tests__/taskDrawer.test.js b/frontend/src/lib/stores/__tests__/taskDrawer.test.js index 5cdf3465..2f422b18 100644 --- a/frontend/src/lib/stores/__tests__/taskDrawer.test.js +++ b/frontend/src/lib/stores/__tests__/taskDrawer.test.js @@ -1,3 +1,7 @@ +// #region TaskDrawerTests [C:3] [TYPE Module] [SEMANTICS test,store,task-drawer] +// @BRIEF Unit tests for task drawer store validating open/close, preference-aware opening, resource task mapping, and status-driven cleanup. +// @RELATION DEPENDS_ON -> [taskDrawer] + import { describe, it, expect, beforeEach, vi } from 'vitest'; import { get } from 'svelte/store'; import { @@ -74,3 +78,4 @@ describe('taskDrawerStore', () => { expect(state.resourceTaskMap['dash-1']).toBeUndefined(); }); }); +// #endregion TaskDrawerTests diff --git a/frontend/src/services/__tests__/gitService.test.js b/frontend/src/services/__tests__/gitService.test.js index 6d139ef9..a4c854e5 100644 --- a/frontend/src/services/__tests__/gitService.test.js +++ b/frontend/src/services/__tests__/gitService.test.js @@ -1,3 +1,7 @@ +// #region GitServiceContractTests [C:3] [TYPE Module] [SEMANTICS test,git-service,api] +// @BRIEF API client tests ensuring correct endpoints are called per contract for Git service operations. +// @RELATION DEPENDS_ON -> [GitServiceClient] + // [DEF:gitServiceContractTests:Module] // @COMPLEXITY: 3 // @SEMANTICS: git-service, api-client, contract-tests @@ -246,3 +250,4 @@ describe('gitService', () => { }); }); // [/DEF:gitServiceContractTests:Module] +// #endregion GitServiceContractTests