busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
backend/src/core/__tests__/test_config_manager_compat.py
View File

@@ -1,7 +1,9 @@
# #region TestConfigManagerCompat [C:3] [TYPE Module] [SEMANTICS config-manager, compatibility, payload, tests]
# @BRIEF Verifies ConfigManager compatibility wrappers preserve legacy payload sections.
# @LAYER Domain
# @RELATION VERIFIES -> [ConfigManager]
# [DEF:TestConfigManagerCompat:Module]
# @COMPLEXITY: 3
# @SEMANTICS: config-manager, compatibility, payload, tests
# @PURPOSE: Verifies ConfigManager compatibility wrappers preserve legacy payload sections.
# @LAYER: Domain
# @RELATION: VERIFIES -> ConfigManager
from types import SimpleNamespace
@@ -9,9 +11,9 @@ from src.core.config_manager import ConfigManager
from src.core.config_models import AppConfig, Environment, GlobalSettings
# #region test_get_payload_preserves_legacy_sections [TYPE Function]
# @BRIEF Ensure get_payload merges typed config into raw payload without dropping legacy sections.
# @RELATION BINDS_TO -> [TestConfigManagerCompat]
# [DEF:test_get_payload_preserves_legacy_sections:Function]
# @RELATION: BINDS_TO -> TestConfigManagerCompat
# @PURPOSE: Ensure get_payload merges typed config into raw payload without dropping legacy sections.
def test_get_payload_preserves_legacy_sections():
manager = ConfigManager.__new__(ConfigManager)
manager.raw_payload = {"notifications": {"smtp": {"host": "mail.local"}}}
@@ -23,7 +25,7 @@ def test_get_payload_preserves_legacy_sections():
assert payload["notifications"]["smtp"]["host"] == "mail.local"
# #endregion test_get_payload_preserves_legacy_sections
# [/DEF:test_get_payload_preserves_legacy_sections:Function]
# [DEF:test_save_config_accepts_raw_payload_and_keeps_extras:Function]
@@ -54,9 +56,13 @@ def test_save_config_accepts_raw_payload_and_keeps_extras(monkeypatch):
assert persisted["payload"]["notifications"]["telegram"]["bot_token"] == "secret"
# #region test_save_config_syncs_environment_records_for_fk_backed_flows [TYPE Function]
# @BRIEF Ensure saving config mirrors typed environments into relational records required by FK-backed session persistence.
# @RELATION BINDS_TO -> [TestConfigManagerCompat]
# [/DEF:test_save_config_accepts_raw_payload_and_keeps_extras:Function]
# [DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
# @RELATION: BINDS_TO -> TestConfigManagerCompat
# @PURPOSE: Ensure saving config mirrors typed environments into relational records required by FK-backed session persistence.
# Deletion of stale records is tested separately in test_save_config_syncs_deletions_to_persistence.
def test_save_config_syncs_environment_records_for_fk_backed_flows():
manager = ConfigManager.__new__(ConfigManager)
manager.raw_payload = {}
@@ -71,20 +77,22 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
credentials_id="legacy-user",
)
# #region _FakeQuery [C:1] [TYPE Class]
# @BRIEF Minimal query stub returning hardcoded existing environment record list for sync tests.
# @INVARIANT all() always returns [existing_record]; no parameterization or filtering.
# @RELATION BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# [DEF:_FakeQuery:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal query stub returning hardcoded existing environment record list for sync tests.
# @INVARIANT: all() always returns [existing_record]; no parameterization or filtering.
class _FakeQuery:
def all(self):
return [existing_record]
# #endregion _FakeQuery
# [/DEF:_FakeQuery:Class]
# #region _FakeSession [C:1] [TYPE Class]
# @BRIEF Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions.
# @INVARIANT query() always returns _FakeQuery; no real DB interaction.
# @RELATION BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions.
# @INVARIANT: query() always returns _FakeQuery; no real DB interaction.
class _FakeSession:
def query(self, model):
return _FakeQuery()
@@ -95,7 +103,7 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
def delete(self, value):
deleted_records.append(value)
# #endregion _FakeSession
# [/DEF:_FakeSession:Class]
session = _FakeSession()
config = AppConfig(
@@ -118,13 +126,91 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
assert added_records[0].name == "DEV"
assert added_records[0].url == "http://superset.local"
assert added_records[0].credentials_id == "demo"
assert deleted_records == [existing_record]
# Sync does NOT delete; deletion is in _delete_stale_environment_records
assert len(deleted_records) == 0
# #endregion test_save_config_syncs_environment_records_for_fk_backed_flows
# #region test_load_config_syncs_environment_records_from_existing_db_payload [TYPE Function]
# @BRIEF Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows.
# @RELATION BINDS_TO -> [TestConfigManagerCompat]
# [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
# [DEF:test_save_config_syncs_deletions_to_persistence:Function]
# @RELATION: BINDS_TO -> TestConfigManagerCompat
# @PURPOSE: Ensure stale environment records are deleted during save (in _delete_stale_environment_records)
# and NOT during _load_config → _sync_environment_records (which would violate FK constraints
# from task_records, database_mappings, etc.).
def test_save_config_syncs_deletions_to_persistence():
manager = ConfigManager.__new__(ConfigManager)
manager.raw_payload = {}
manager.config = AppConfig(environments=[], settings=GlobalSettings())
added_records = []
deleted_records = []
existing_record = SimpleNamespace(
id="legacy-env",
name="Legacy",
url="http://legacy.local",
credentials_id="legacy-user",
)
# [DEF:_FakeQueryDel:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_deletions_to_persistence]
# @COMPLEXITY: 1
# @PURPOSE: Minimal query stub for deletion test.
class _FakeQuery:
def all(self):
return [existing_record]
# [/DEF:_FakeQueryDel:Class]
# [DEF:_FakeSessionDel:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_deletions_to_persistence]
# @COMPLEXITY: 1
# @PURPOSE: Minimal session stub that captures add/delete for deletion assertions.
class _FakeSession:
def query(self, model):
return _FakeQuery()
def add(self, value):
added_records.append(value)
def delete(self, value):
deleted_records.append(value)
# [/DEF:_FakeSessionDel:Class]
session = _FakeSession()
config = AppConfig(
environments=[
Environment(
id="dev",
name="DEV",
url="http://superset.local",
username="demo",
password="secret",
)
],
settings=GlobalSettings(),
)
# Step 1: sync creates/updates (no deletion)
manager._sync_environment_records(session, config)
assert len(added_records) == 1
assert added_records[0].id == "dev"
assert len(deleted_records) == 0
# Step 2: stale deletion removes envs not in the configured set
manager._delete_stale_environment_records(session, config)
assert len(deleted_records) == 1
assert deleted_records[0] == existing_record
assert deleted_records[0].id == "legacy-env"
# [/DEF:test_save_config_syncs_deletions_to_persistence:Function]
# [DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
# @RELATION: BINDS_TO -> TestConfigManagerCompat
# @PURPOSE: Ensure loading an existing DB-backed config also mirrors environment rows required by FK-backed runtime flows.
def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypatch):
manager = ConfigManager.__new__(ConfigManager)
manager.config_path = None
@@ -135,10 +221,11 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
closed = {"value": False}
committed = {"value": False}
# #region _FakeSession [C:1] [TYPE Class]
# @BRIEF Minimal session stub tracking commit/close signals for config load lifecycle assertions.
# @INVARIANT No query or add semantics; only lifecycle signal tracking.
# @RELATION BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload]
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload]
# @COMPLEXITY: 1
# @PURPOSE: Minimal session stub tracking commit/close signals for config load lifecycle assertions.
# @INVARIANT: No query or add semantics; only lifecycle signal tracking.
class _FakeSession:
def commit(self):
committed["value"] = True
@@ -146,7 +233,7 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
def close(self):
closed["value"] = True
# #endregion _FakeSession
# [/DEF:_FakeSession:Class]
fake_session = _FakeSession()
fake_record = SimpleNamespace(
@@ -183,5 +270,6 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
assert closed["value"] is True
# #endregion test_load_config_syncs_environment_records_from_existing_db_payload
# #endregion TestConfigManagerCompat
# [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
# [/DEF:TestConfigManagerCompat:Module]

190
backend/src/core/__tests__/test_native_filters.py
View File

@@ -1,9 +1,11 @@
# #region NativeFilterExtractionTests [C:3] [TYPE Module] [SEMANTICS tests, superset, native, filters, permalink, filter_state]
# @BRIEF Verify native filter extraction from permalinks and native_filters_key URLs.
# @LAYER Domain
# @RELATION BINDS_TO -> [SupersetClient]
# @RELATION BINDS_TO -> [AsyncSupersetClient]
# @RELATION BINDS_TO -> [FilterState, ParsedNativeFilters, ExtraFormDataMerge]
# [DEF:NativeFilterExtractionTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, superset, native, filters, permalink, filter_state
# @PURPOSE: Verify native filter extraction from permalinks and native_filters_key URLs.
# @LAYER: Domain
# @RELATION: [BINDS_TO] ->[SupersetClient]
# @RELATION: [BINDS_TO] ->[AsyncSupersetClient]
# @RELATION: [BINDS_TO] ->[FilterState, ParsedNativeFilters, ExtraFormDataMerge]
import json
from unittest.mock import MagicMock
@@ -26,8 +28,8 @@ from src.models.filter_state import (
)
# #region _make_environment [TYPE Function]
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:_make_environment:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
def _make_environment() -> Environment:
return Environment(
id="env-1",
@@ -38,12 +40,12 @@ def _make_environment() -> Environment:
)
# #endregion _make_environment
# [/DEF:_make_environment:Function]
# #region test_extract_native_filters_from_permalink [TYPE Function]
# @BRIEF Extract native filters from a permalink key.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_native_filters_from_permalink:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Extract native filters from a permalink key.
def test_extract_native_filters_from_permalink():
client = SupersetClient(_make_environment())
client.get_dashboard_permalink_state = MagicMock(
@@ -87,12 +89,12 @@ def test_extract_native_filters_from_permalink():
assert result["anchor"] == "SECTION1"
# #endregion test_extract_native_filters_from_permalink
# [/DEF:test_extract_native_filters_from_permalink]
# #region test_extract_native_filters_from_permalink_direct_response [TYPE Function]
# @BRIEF Handle permalink response without result wrapper.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_native_filters_from_permalink_direct_response:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Handle permalink response without result wrapper.
def test_extract_native_filters_from_permalink_direct_response():
client = SupersetClient(_make_environment())
client.get_dashboard_permalink_state = MagicMock(
@@ -115,12 +117,12 @@ def test_extract_native_filters_from_permalink_direct_response():
assert "filter_1" in result["dataMask"]
# #endregion test_extract_native_filters_from_permalink_direct_response
# [/DEF:test_extract_native_filters_from_permalink_direct_response]
# #region test_extract_native_filters_from_key [TYPE Function]
# @BRIEF Extract native filters from a native_filters_key.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_native_filters_from_key:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Extract native filters from a native_filters_key.
def test_extract_native_filters_from_key():
client = SupersetClient(_make_environment())
client.get_native_filter_state = MagicMock(
@@ -154,12 +156,12 @@ def test_extract_native_filters_from_key():
] == ["EMEA"]
# #endregion test_extract_native_filters_from_key
# [/DEF:test_extract_native_filters_from_key]
# #region test_extract_native_filters_from_key_single_filter [TYPE Function]
# @BRIEF Handle single filter format in native filter state.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_native_filters_from_key_single_filter:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Handle single filter format in native filter state.
def test_extract_native_filters_from_key_single_filter():
client = SupersetClient(_make_environment())
client.get_native_filter_state = MagicMock(
@@ -188,12 +190,12 @@ def test_extract_native_filters_from_key_single_filter():
)
# #endregion test_extract_native_filters_from_key_single_filter
# [/DEF:test_extract_native_filters_from_key_single_filter]
# #region test_extract_native_filters_from_key_dict_value [TYPE Function]
# @BRIEF Handle filter state value as dict instead of JSON string.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_native_filters_from_key_dict_value:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Handle filter state value as dict instead of JSON string.
def test_extract_native_filters_from_key_dict_value():
client = SupersetClient(_make_environment())
client.get_native_filter_state = MagicMock(
@@ -215,12 +217,12 @@ def test_extract_native_filters_from_key_dict_value():
assert "filter_id" in result["dataMask"]
# #endregion test_extract_native_filters_from_key_dict_value
# [/DEF:test_extract_native_filters_from_key_dict_value]
# #region test_parse_dashboard_url_for_filters_permalink [TYPE Function]
# @BRIEF Parse permalink URL format.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_permalink:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Parse permalink URL format.
def test_parse_dashboard_url_for_filters_permalink():
client = SupersetClient(_make_environment())
client.extract_native_filters_from_permalink = MagicMock(
@@ -235,12 +237,12 @@ def test_parse_dashboard_url_for_filters_permalink():
assert result["filters"]["dataMask"]["f1"] == {}
# #endregion test_parse_dashboard_url_for_filters_permalink
# [/DEF:test_parse_dashboard_url_for_filters_permalink]
# #region test_parse_dashboard_url_for_filters_native_key [TYPE Function]
# @BRIEF Parse native_filters_key URL format with numeric dashboard ID.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_native_key:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Parse native_filters_key URL format with numeric dashboard ID.
def test_parse_dashboard_url_for_filters_native_key():
client = SupersetClient(_make_environment())
client.extract_native_filters_from_key = MagicMock(
@@ -260,12 +262,12 @@ def test_parse_dashboard_url_for_filters_native_key():
assert result["filters"]["dataMask"]["f2"] == {}
# #endregion test_parse_dashboard_url_for_filters_native_key
# [/DEF:test_parse_dashboard_url_for_filters_native_key]
# #region test_parse_dashboard_url_for_filters_native_key_slug [TYPE Function]
# @BRIEF Parse native_filters_key URL format when dashboard reference is a slug, not a numeric ID.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_native_key_slug:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Parse native_filters_key URL format when dashboard reference is a slug, not a numeric ID.
def test_parse_dashboard_url_for_filters_native_key_slug():
client = SupersetClient(_make_environment())
# Simulate slug resolution: get_dashboard returns the dashboard with numeric ID
@@ -293,12 +295,12 @@ def test_parse_dashboard_url_for_filters_native_key_slug():
client.extract_native_filters_from_key.assert_called_once_with(99, "abc123")
# #endregion test_parse_dashboard_url_for_filters_native_key_slug
# [/DEF:test_parse_dashboard_url_for_filters_native_key_slug]
# #region test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails [TYPE Function]
# @BRIEF Gracefully handle slug resolution failure for native_filters_key URL.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Gracefully handle slug resolution failure for native_filters_key URL.
def test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails():
client = SupersetClient(_make_environment())
client.get_dashboard = MagicMock(side_effect=Exception("Not found"))
@@ -311,12 +313,12 @@ def test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails():
assert result["dashboard_id"] is None
# #endregion test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails
# [/DEF:test_parse_dashboard_url_for_filters_native_key_slug_resolution_fails]
# #region test_parse_dashboard_url_for_filters_native_filters_direct [TYPE Function]
# @BRIEF Parse native_filters direct query param.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_native_filters_direct:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Parse native_filters direct query param.
def test_parse_dashboard_url_for_filters_native_filters_direct():
client = SupersetClient(_make_environment())
@@ -329,12 +331,12 @@ def test_parse_dashboard_url_for_filters_native_filters_direct():
assert "dataMask" in result["filters"]
# #endregion test_parse_dashboard_url_for_filters_native_filters_direct
# [/DEF:test_parse_dashboard_url_for_filters_native_filters_direct]
# #region test_parse_dashboard_url_for_filters_no_filters [TYPE Function]
# @BRIEF Return empty result when no filters present.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parse_dashboard_url_for_filters_no_filters:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Return empty result when no filters present.
def test_parse_dashboard_url_for_filters_no_filters():
client = SupersetClient(_make_environment())
@@ -346,12 +348,12 @@ def test_parse_dashboard_url_for_filters_no_filters():
assert result["filters"] == {}
# #endregion test_parse_dashboard_url_for_filters_no_filters
# [/DEF:test_parse_dashboard_url_for_filters_no_filters]
# #region test_extra_form_data_merge [TYPE Function]
# @BRIEF Test ExtraFormDataMerge correctly merges dictionaries.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extra_form_data_merge:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Test ExtraFormDataMerge correctly merges dictionaries.
def test_extra_form_data_merge():
merger = ExtraFormDataMerge()
@@ -384,12 +386,12 @@ def test_extra_form_data_merge():
assert result["columns"] == ["col1", "col2"]
# #endregion test_extra_form_data_merge
# [/DEF:test_extra_form_data_merge]
# #region test_filter_state_model [TYPE Function]
# @BRIEF Test FilterState Pydantic model.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_filter_state_model:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Test FilterState Pydantic model.
def test_filter_state_model():
state = FilterState(
extraFormData={"filters": [{"col": "x", "op": "==", "val": "y"}]},
@@ -402,12 +404,12 @@ def test_filter_state_model():
assert state.ownState["selectedValues"] == ["y"]
# #endregion test_filter_state_model
# [/DEF:test_filter_state_model]
# #region test_parsed_native_filters_model [TYPE Function]
# @BRIEF Test ParsedNativeFilters Pydantic model.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parsed_native_filters_model:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Test ParsedNativeFilters Pydantic model.
def test_parsed_native_filters_model():
filters = ParsedNativeFilters(
dataMask={"f1": {"extraFormData": {}, "filterState": {}}},
@@ -421,12 +423,12 @@ def test_parsed_native_filters_model():
assert filters.filter_type == "permalink"
# #endregion test_parsed_native_filters_model
# [/DEF:test_parsed_native_filters_model]
# #region test_parsed_native_filters_empty [TYPE Function]
# @BRIEF Test ParsedNativeFilters with no filters.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_parsed_native_filters_empty:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Test ParsedNativeFilters with no filters.
def test_parsed_native_filters_empty():
filters = ParsedNativeFilters()
@@ -434,12 +436,12 @@ def test_parsed_native_filters_empty():
assert filters.get_filter_count() == 0
# #endregion test_parsed_native_filters_empty
# [/DEF:test_parsed_native_filters_empty]
# #region test_native_filter_data_mask_model [TYPE Function]
# @BRIEF Test NativeFilterDataMask model.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_native_filter_data_mask_model:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Test NativeFilterDataMask model.
def test_native_filter_data_mask_model():
data_mask = NativeFilterDataMask(
filters={
@@ -455,12 +457,12 @@ def test_native_filter_data_mask_model():
assert data_mask.get_extra_form_data("nonexistent") == {}
# #endregion test_native_filter_data_mask_model
# [/DEF:test_native_filter_data_mask_model]
# #region test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names [TYPE Function]
# @BRIEF Reconcile raw native filter ids from state to canonical metadata filter names.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Reconcile raw native filter ids from state to canonical metadata filter names.
def test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names():
client = MagicMock()
client.get_dashboard.return_value = {
@@ -522,12 +524,12 @@ def test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_n
}
# #endregion test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names
# [/DEF:test_recover_imported_filters_reconciles_raw_native_filter_ids_to_metadata_names:Function]
# #region test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter [TYPE Function]
# @BRIEF Collapse raw-id state entries and metadata entries into one canonical filter.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Collapse raw-id state entries and metadata entries into one canonical filter.
def test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter():
client = MagicMock()
client.get_dashboard.return_value = {
@@ -580,12 +582,12 @@ def test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_o
assert region_filter["recovery_status"] == "partial"
# #endregion test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter
# [/DEF:test_recover_imported_filters_collapses_state_and_metadata_duplicates_into_one_canonical_filter:Function]
# #region test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids [TYPE Function]
# @BRIEF Preserve unmatched raw native filter ids as fallback diagnostics when metadata mapping is unavailable.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Preserve unmatched raw native filter ids as fallback diagnostics when metadata mapping is unavailable.
def test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids():
client = MagicMock()
client.get_dashboard.return_value = {
@@ -637,12 +639,12 @@ def test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids():
)
# #endregion test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids
# [/DEF:test_recover_imported_filters_preserves_unmatched_raw_native_filter_ids:Function]
# #region test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview [TYPE Function]
# @BRIEF Recovered native filter state should preserve exact Superset clause payload and time extras for preview compilation.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Recovered native filter state should preserve exact Superset clause payload and time extras for preview compilation.
def test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview():
extractor = SupersetContextExtractor(_make_environment(), client=MagicMock())
@@ -682,12 +684,12 @@ def test_extract_imported_filters_preserves_clause_level_native_filter_payload_f
]
# #endregion test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview
# [/DEF:test_extract_imported_filters_preserves_clause_level_native_filter_payload_for_preview:Function]
# #region test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values [TYPE Function]
# @BRIEF Assistant-facing imported-filter payload should redact likely PII while preserving structure metadata.
# @RELATION BINDS_TO -> [NativeFilterExtractionTests]
# [DEF:test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values:Function]
# @RELATION: BINDS_TO -> NativeFilterExtractionTests
# @PURPOSE: Assistant-facing imported-filter payload should redact likely PII while preserving structure metadata.
def test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values():
result = sanitize_imported_filter_for_assistant(
{
@@ -703,7 +705,7 @@ def test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values():
assert result["normalized_value"] == {"filter_clauses": []}
# #endregion test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values
# [/DEF:test_sanitize_imported_filter_for_assistant_masks_sensitive_raw_values:Function]
# #endregion NativeFilterExtractionTests
# [/DEF:NativeFilterExtractionTests:Module]

110
backend/src/core/__tests__/test_superset_preview_pipeline.py
View File

@@ -1,7 +1,9 @@
# #region SupersetPreviewPipelineTests [C:3] [TYPE Module] [SEMANTICS tests, superset, preview, chart_data, network, 404-mapping]
# @BRIEF Verify explicit chart-data preview compilation and ensure non-dashboard 404 errors remain generic across sync and async clients.
# @LAYER Domain
# @RELATION BINDS_TO -> [AsyncNetworkModule]
# [DEF:SupersetPreviewPipelineTests:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, superset, preview, chart_data, network, 404-mapping
# @PURPOSE: Verify explicit chart-data preview compilation and ensure non-dashboard 404 errors remain generic across sync and async clients.
# @LAYER: Domain
# @RELATION: [BINDS_TO] ->[AsyncNetworkModule]
import json
from unittest.mock import MagicMock
@@ -16,8 +18,8 @@ from src.core.utils.async_network import AsyncAPIClient
from src.core.utils.network import APIClient, DashboardNotFoundError, SupersetAPIError
# #region _make_environment [TYPE Function]
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:_make_environment:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
def _make_environment() -> Environment:
return Environment(
id="env-1",
@@ -28,11 +30,11 @@ def _make_environment() -> Environment:
)
# #endregion _make_environment
# [/DEF:_make_environment:Function]
# #region _make_requests_http_error [TYPE Function]
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:_make_requests_http_error:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
def _make_requests_http_error(
status_code: int, url: str
) -> requests.exceptions.HTTPError:
@@ -45,11 +47,11 @@ def _make_requests_http_error(
return requests.exceptions.HTTPError(response=response, request=request)
# #endregion _make_requests_http_error
# [/DEF:_make_requests_http_error:Function]
# #region _make_httpx_status_error [TYPE Function]
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:_make_httpx_status_error:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
def _make_httpx_status_error(status_code: int, url: str) -> httpx.HTTPStatusError:
request = httpx.Request("GET", url)
response = httpx.Response(
@@ -58,12 +60,12 @@ def _make_httpx_status_error(status_code: int, url: str) -> httpx.HTTPStatusErro
return httpx.HTTPStatusError("upstream error", request=request, response=response)
# #endregion _make_httpx_status_error
# [/DEF:_make_httpx_status_error:Function]
# #region test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy [TYPE Function]
# @BRIEF Superset preview compilation should prefer the legacy form_data transport inferred from browser traffic before falling back to chart-data.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Superset preview compilation should prefer the legacy form_data transport inferred from browser traffic before falling back to chart-data.
def test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy():
client = SupersetClient(_make_environment())
client.get_dataset = MagicMock(
@@ -144,12 +146,12 @@ def test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy():
]
# #endregion test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy
# [/DEF:test_compile_dataset_preview_prefers_legacy_explore_form_data_strategy:Function]
# #region test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures [TYPE Function]
# @BRIEF Superset preview compilation should fall back to chart-data when legacy form_data strategies are rejected.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Superset preview compilation should fall back to chart-data when legacy form_data strategies are rejected.
def test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures():
client = SupersetClient(_make_environment())
client.get_dataset = MagicMock(
@@ -241,12 +243,12 @@ def test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures(
}
# #endregion test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures
# [/DEF:test_compile_dataset_preview_falls_back_to_chart_data_after_legacy_failures:Function]
# #region test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data [TYPE Function]
# @BRIEF Preview query context should mirror chart-style filter transport so recovered native filters reach Superset compilation.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Preview query context should mirror chart-style filter transport so recovered native filters reach Superset compilation.
def test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data():
client = SupersetClient(_make_environment())
@@ -304,12 +306,12 @@ def test_build_dataset_preview_query_context_places_recovered_filters_in_chart_s
assert query_context["form_data"]["url_params"] == {"country": "DE"}
# #endregion test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data
# [/DEF:test_build_dataset_preview_query_context_places_recovered_filters_in_chart_style_form_data:Function]
# #region test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values [TYPE Function]
# @BRIEF Preview query context should merge dataset template params for parity with real dataset definitions while preserving explicit session overrides.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Preview query context should merge dataset template params for parity with real dataset definitions while preserving explicit session overrides.
def test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values():
client = SupersetClient(_make_environment())
@@ -335,12 +337,12 @@ def test_build_dataset_preview_query_context_merges_dataset_template_params_and_
}
# #endregion test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values
# [/DEF:test_build_dataset_preview_query_context_merges_dataset_template_params_and_preserves_user_values:Function]
# #region test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload [TYPE Function]
# @BRIEF Preview query context should preserve time-range native filter extras even when dataset defaults differ.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Preview query context should preserve time-range native filter extras even when dataset defaults differ.
def test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload():
client = SupersetClient(_make_environment())
@@ -374,12 +376,12 @@ def test_build_dataset_preview_query_context_preserves_time_range_from_native_fi
assert query_context["queries"][0]["filters"] == []
# #endregion test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload
# [/DEF:test_build_dataset_preview_query_context_preserves_time_range_from_native_filter_payload:Function]
# #region test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses [TYPE Function]
# @BRIEF Legacy preview form_data should preserve recovered native filter clauses in browser-style fields without duplicating datasource for QueryObjectFactory.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Legacy preview form_data should preserve recovered native filter clauses in browser-style fields without duplicating datasource for QueryObjectFactory.
def test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses():
client = SupersetClient(_make_environment())
@@ -428,12 +430,12 @@ def test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses(
assert legacy_form_data["result_type"] == "query"
# #endregion test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses
# [/DEF:test_build_dataset_preview_legacy_form_data_preserves_native_filter_clauses:Function]
# #region test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function]
# @BRIEF Sync network client should reserve dashboard-not-found translation for dashboard endpoints only.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Sync network client should reserve dashboard-not-found translation for dashboard endpoints only.
def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic():
client = APIClient(
config={
@@ -452,12 +454,12 @@ def test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic():
assert "API resource not found at endpoint '/chart/data'" in str(exc_info.value)
# #endregion test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic
# [/DEF:test_sync_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
# #region test_sync_network_404_mapping_translates_dashboard_endpoints [TYPE Function]
# @BRIEF Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Sync network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
def test_sync_network_404_mapping_translates_dashboard_endpoints():
client = APIClient(
config={
@@ -475,12 +477,12 @@ def test_sync_network_404_mapping_translates_dashboard_endpoints():
assert "Dashboard '/dashboard/10' Dashboard not found" in str(exc_info.value)
# #endregion test_sync_network_404_mapping_translates_dashboard_endpoints
# [/DEF:test_sync_network_404_mapping_translates_dashboard_endpoints:Function]
# #region test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic [TYPE Function]
# @BRIEF Async network client should reserve dashboard-not-found translation for dashboard endpoints only.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Async network client should reserve dashboard-not-found translation for dashboard endpoints only.
@pytest.mark.asyncio
async def test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic():
client = AsyncAPIClient(
@@ -505,12 +507,12 @@ async def test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic()
await client.aclose()
# #endregion test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic
# [/DEF:test_async_network_404_mapping_keeps_non_dashboard_endpoints_generic:Function]
# #region test_async_network_404_mapping_translates_dashboard_endpoints [TYPE Function]
# @BRIEF Async network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
# @RELATION BINDS_TO -> [SupersetPreviewPipelineTests]
# [DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function]
# @RELATION: BINDS_TO -> SupersetPreviewPipelineTests
# @PURPOSE: Async network client should still translate dashboard endpoint 404 responses into dashboard-not-found errors.
@pytest.mark.asyncio
async def test_async_network_404_mapping_translates_dashboard_endpoints():
client = AsyncAPIClient(
@@ -534,7 +536,7 @@ async def test_async_network_404_mapping_translates_dashboard_endpoints():
await client.aclose()
# #endregion test_async_network_404_mapping_translates_dashboard_endpoints
# [/DEF:test_async_network_404_mapping_translates_dashboard_endpoints:Function]
# #endregion SupersetPreviewPipelineTests
# [/DEF:SupersetPreviewPipelineTests:Module]

79
backend/src/core/__tests__/test_superset_profile_lookup.py
View File

@@ -1,7 +1,9 @@
# #region TestSupersetProfileLookup [C:3] [TYPE Module] [SEMANTICS tests, superset, profile, lookup, fallback, sorting]
# @BRIEF Verifies Superset profile lookup adapter payload normalization and fallback error precedence.
# @LAYER Domain
# @RELATION BELONGS_TO -> [SrcRoot]
# [DEF:TestSupersetProfileLookup:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, superset, profile, lookup, fallback, sorting
# @PURPOSE: Verifies Superset profile lookup adapter payload normalization and fallback error precedence.
# @LAYER: Domain
# [SECTION: IMPORTS]
import json
@@ -20,25 +22,26 @@ from src.core.utils.network import AuthenticationError, SupersetAPIError
# [/SECTION]
# #region _RecordingNetworkClient [C:2] [TYPE Class]
# @BRIEF Records request payloads and returns scripted responses for deterministic adapter tests.
# @INVARIANT Each request consumes one scripted response in call order and persists call metadata.
# @RELATION BINDS_TO -> [TestSupersetProfileLookup]
# [DEF:_RecordingNetworkClient:Class]
# @RELATION: BINDS_TO -> TestSupersetProfileLookup
# @COMPLEXITY: 2
# @PURPOSE: Records request payloads and returns scripted responses for deterministic adapter tests.
# @INVARIANT: Each request consumes one scripted response in call order and persists call metadata.
class _RecordingNetworkClient:
# #region __init__ [TYPE Function]
# @BRIEF Initializes scripted network responses.
# @PRE scripted_responses is ordered per expected request sequence.
# @POST Instance stores response script and captures subsequent request calls.
# [DEF:__init__:Function]
# @PURPOSE: Initializes scripted network responses.
# @PRE: scripted_responses is ordered per expected request sequence.
# @POST: Instance stores response script and captures subsequent request calls.
def __init__(self, scripted_responses: List[Any]):
self._scripted_responses = scripted_responses
self.calls: List[Dict[str, Any]] = []
# #endregion __init__
# [/DEF:__init__:Function]
# #region request [TYPE Function]
# @BRIEF Mimics APIClient.request while capturing call arguments.
# @PRE method and endpoint are provided.
# @POST Returns scripted response or raises scripted exception.
# [DEF:request:Function]
# @PURPOSE: Mimics APIClient.request while capturing call arguments.
# @PRE: method and endpoint are provided.
# @POST: Returns scripted response or raises scripted exception.
def request(
self,
method: str,
@@ -59,17 +62,17 @@ class _RecordingNetworkClient:
raise response
return response
# #endregion request
# [/DEF:request:Function]
# #endregion _RecordingNetworkClient
# [/DEF:_RecordingNetworkClient:Class]
# #region test_get_users_page_sends_lowercase_order_direction [TYPE Function]
# @BRIEF Ensures adapter sends lowercase order_direction compatible with Superset rison schema.
# @PRE Adapter is initialized with recording network client.
# @POST First request query payload contains order_direction='asc' for asc sort.
# @RELATION BINDS_TO -> [TestSupersetProfileLookup]
# [DEF:test_get_users_page_sends_lowercase_order_direction:Function]
# @RELATION: BINDS_TO -> TestSupersetProfileLookup
# @PURPOSE: Ensures adapter sends lowercase order_direction compatible with Superset rison schema.
# @PRE: Adapter is initialized with recording network client.
# @POST: First request query payload contains order_direction='asc' for asc sort.
def test_get_users_page_sends_lowercase_order_direction():
client = _RecordingNetworkClient(
scripted_responses=[{"result": [{"username": "admin"}], "count": 1}]
@@ -90,14 +93,14 @@ def test_get_users_page_sends_lowercase_order_direction():
assert sent_query["order_direction"] == "asc"
# #endregion test_get_users_page_sends_lowercase_order_direction
# [/DEF:test_get_users_page_sends_lowercase_order_direction:Function]
# #region test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error [TYPE Function]
# @BRIEF Ensures fallback auth error does not mask primary schema/query failure.
# @PRE Primary endpoint fails with SupersetAPIError and fallback fails with AuthenticationError.
# @POST Raised exception remains primary SupersetAPIError (non-auth) to preserve root cause.
# @RELATION BINDS_TO -> [TestSupersetProfileLookup]
# [DEF:test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error:Function]
# @RELATION: BINDS_TO -> TestSupersetProfileLookup
# @PURPOSE: Ensures fallback auth error does not mask primary schema/query failure.
# @PRE: Primary endpoint fails with SupersetAPIError and fallback fails with AuthenticationError.
# @POST: Raised exception remains primary SupersetAPIError (non-auth) to preserve root cause.
def test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error():
client = _RecordingNetworkClient(
scripted_responses=[
@@ -116,14 +119,14 @@ def test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error(
assert not isinstance(exc_info.value, AuthenticationError)
# #endregion test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error
# [/DEF:test_get_users_page_preserves_primary_schema_error_over_fallback_auth_error:Function]
# #region test_get_users_page_uses_fallback_endpoint_when_primary_fails [TYPE Function]
# @BRIEF Verifies adapter retries second users endpoint and succeeds when fallback is healthy.
# @PRE Primary endpoint fails; fallback returns valid users payload.
# @POST Result status is success and both endpoints were attempted in order.
# @RELATION BINDS_TO -> [TestSupersetProfileLookup]
# [DEF:test_get_users_page_uses_fallback_endpoint_when_primary_fails:Function]
# @RELATION: BINDS_TO -> TestSupersetProfileLookup
# @PURPOSE: Verifies adapter retries second users endpoint and succeeds when fallback is healthy.
# @PRE: Primary endpoint fails; fallback returns valid users payload.
# @POST: Result status is success and both endpoints were attempted in order.
def test_get_users_page_uses_fallback_endpoint_when_primary_fails():
client = _RecordingNetworkClient(
scripted_responses=[
@@ -144,7 +147,7 @@ def test_get_users_page_uses_fallback_endpoint_when_primary_fails():
]
# #endregion test_get_users_page_uses_fallback_endpoint_when_primary_fails
# [/DEF:test_get_users_page_uses_fallback_endpoint_when_primary_fails:Function]
# #endregion TestSupersetProfileLookup
# [/DEF:TestSupersetProfileLookup:Module]

57
backend/src/core/__tests__/test_throttled_scheduler.py
View File

@@ -2,14 +2,15 @@ import pytest
from datetime import time, date, datetime, timedelta
from src.core.scheduler import ThrottledSchedulerConfigurator
# #region test_throttled_scheduler [C:3] [TYPE Module]
# @BRIEF Unit tests for ThrottledSchedulerConfigurator distribution logic.
# @RELATION BELONGS_TO -> [SrcRoot]
# [DEF:test_throttled_scheduler:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic.
# #region test_calculate_schedule_even_distribution [TYPE Function]
# @BRIEF Validate even spacing across a two-hour scheduling window for three tasks.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_even_distribution:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate even spacing across a two-hour scheduling window for three tasks.
def test_calculate_schedule_even_distribution():
"""
@TEST_SCENARIO: 3 tasks in a 2-hour window should be spaced 1 hour apart.
@@ -29,12 +30,12 @@ def test_calculate_schedule_even_distribution():
assert schedule[2] == datetime(2024, 1, 1, 3, 0)
# #endregion test_calculate_schedule_even_distribution
# [/DEF:test_calculate_schedule_even_distribution:Function]
# #region test_calculate_schedule_midnight_crossing [TYPE Function]
# @BRIEF Validate scheduler correctly rolls timestamps into the next day across midnight.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_midnight_crossing:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate scheduler correctly rolls timestamps into the next day across midnight.
def test_calculate_schedule_midnight_crossing():
"""
@TEST_SCENARIO: Window from 23:00 to 01:00 (next day).
@@ -54,12 +55,12 @@ def test_calculate_schedule_midnight_crossing():
assert schedule[2] == datetime(2024, 1, 2, 1, 0)
# #endregion test_calculate_schedule_midnight_crossing
# [/DEF:test_calculate_schedule_midnight_crossing:Function]
# #region test_calculate_schedule_single_task [TYPE Function]
# @BRIEF Validate single-task schedule returns only the window start timestamp.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_single_task:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate single-task schedule returns only the window start timestamp.
def test_calculate_schedule_single_task():
"""
@TEST_SCENARIO: Single task should be scheduled at start time.
@@ -77,12 +78,12 @@ def test_calculate_schedule_single_task():
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
# #endregion test_calculate_schedule_single_task
# [/DEF:test_calculate_schedule_single_task:Function]
# #region test_calculate_schedule_empty_list [TYPE Function]
# @BRIEF Validate empty dashboard list produces an empty schedule.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_empty_list:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate empty dashboard list produces an empty schedule.
def test_calculate_schedule_empty_list():
"""
@TEST_SCENARIO: Empty dashboard list returns empty schedule.
@@ -99,12 +100,12 @@ def test_calculate_schedule_empty_list():
assert schedule == []
# #endregion test_calculate_schedule_empty_list
# [/DEF:test_calculate_schedule_empty_list:Function]
# #region test_calculate_schedule_zero_window [TYPE Function]
# @BRIEF Validate zero-length window schedules all tasks at identical start timestamp.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_zero_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate zero-length window schedules all tasks at identical start timestamp.
def test_calculate_schedule_zero_window():
"""
@TEST_SCENARIO: Window start == end. All tasks at start time.
@@ -123,12 +124,12 @@ def test_calculate_schedule_zero_window():
assert schedule[1] == datetime(2024, 1, 1, 1, 0)
# #endregion test_calculate_schedule_zero_window
# [/DEF:test_calculate_schedule_zero_window:Function]
# #region test_calculate_schedule_very_small_window [TYPE Function]
# @BRIEF Validate sub-second interpolation when task count exceeds near-zero window granularity.
# @RELATION BINDS_TO -> [test_throttled_scheduler]
# [DEF:test_calculate_schedule_very_small_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate sub-second interpolation when task count exceeds near-zero window granularity.
def test_calculate_schedule_very_small_window():
"""
@TEST_SCENARIO: Window smaller than number of tasks (in seconds).
@@ -148,5 +149,5 @@ def test_calculate_schedule_very_small_window():
assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1)
# #endregion test_calculate_schedule_very_small_window
# #endregion test_throttled_scheduler
# [/DEF:test_calculate_schedule_very_small_window:Function]
# [/DEF:test_throttled_scheduler:Module]

151
backend/src/core/async_superset_client.py
View File

@@ -1,9 +1,8 @@
# #region AsyncSupersetClientModule [C:3] [TYPE Module] [SEMANTICS superset, async, client, httpx, dashboards, datasets]
# @BRIEF Parse a Superset dashboard URL and extract native filter state asynchronously.
# @LAYER Core
# @LAYER: Core
# [SECTION: IMPORTS]
import asyncio
import json
import re
@@ -13,7 +12,6 @@ from .config_models import Environment
from .logger import logger as app_logger, belief_scope
from .superset_client import SupersetClient
from .utils.async_network import AsyncAPIClient
# [/SECTION]
# #region AsyncSupersetClient [C:3] [TYPE Class]
@@ -22,12 +20,13 @@ from .utils.async_network import AsyncAPIClient
# @RELATION DEPENDS_ON -> [AsyncAPIClient]
# @RELATION CALLS -> [AsyncAPIClient.request]
class AsyncSupersetClient(SupersetClient):
# #region AsyncSupersetClientInit [C:3] [TYPE Function]
# @BRIEF Initialize async Superset client with AsyncAPIClient transport.
# @PRE env is valid Environment instance.
# @POST Client uses async network transport and inherited projection helpers.
# @DATA_CONTRACT Input[Environment] -> self.network[AsyncAPIClient]
# @RELATION DEPENDS_ON -> [AsyncAPIClient]
# [DEF:AsyncSupersetClientInit:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize async Superset client with AsyncAPIClient transport.
# @PRE: env is valid Environment instance.
# @POST: Client uses async network transport and inherited projection helpers.
# @DATA_CONTRACT: Input[Environment] -> self.network[AsyncAPIClient]
# @RELATION: [DEPENDS_ON] ->[AsyncAPIClient]
def __init__(self, env: Environment):
self.env = env
auth_payload = {
@@ -43,23 +42,25 @@ class AsyncSupersetClient(SupersetClient):
)
self.delete_before_reimport = False
# #endregion AsyncSupersetClientInit
# [/DEF:AsyncSupersetClientInit:Function]
# #region AsyncSupersetClientClose [C:3] [TYPE Function]
# @BRIEF Close async transport resources.
# @POST Underlying AsyncAPIClient is closed.
# @SIDE_EFFECT Closes network sockets.
# @RELATION CALLS -> [AsyncAPIClient.aclose]
# [DEF:AsyncSupersetClientClose:Function]
# @COMPLEXITY: 3
# @PURPOSE: Close async transport resources.
# @POST: Underlying AsyncAPIClient is closed.
# @SIDE_EFFECT: Closes network sockets.
# @RELATION: [CALLS] ->[AsyncAPIClient.aclose]
async def aclose(self) -> None:
await self.network.aclose()
# #endregion AsyncSupersetClientClose
# [/DEF:AsyncSupersetClientClose:Function]
# #region get_dashboards_page_async [C:3] [TYPE Function]
# @BRIEF Fetch one dashboards page asynchronously.
# @POST Returns total count and page result list.
# @DATA_CONTRACT Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION CALLS -> [AsyncAPIClient.request]
# [DEF:get_dashboards_page_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch one dashboards page asynchronously.
# @POST: Returns total count and page result list.
# @DATA_CONTRACT: Input[query: Optional[Dict]] -> Output[Tuple[int, List[Dict]]]
# @RELATION: [CALLS] -> [AsyncAPIClient.request]
async def get_dashboards_page_async(
self, query: Optional[Dict] = None
) -> Tuple[int, List[Dict]]:
@@ -91,13 +92,14 @@ class AsyncSupersetClient(SupersetClient):
total_count = response_json.get("count", len(result))
return total_count, result
# #endregion get_dashboards_page_async
# [/DEF:get_dashboards_page_async:Function]
# #region get_dashboard_async [C:3] [TYPE Function]
# @BRIEF Fetch one dashboard payload asynchronously.
# @POST Returns raw dashboard payload from Superset API.
# @DATA_CONTRACT Input[dashboard_id: int] -> Output[Dict]
# @RELATION CALLS -> [AsyncAPIClient.request]
# [DEF:get_dashboard_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch one dashboard payload asynchronously.
# @POST: Returns raw dashboard payload from Superset API.
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[AsyncAPIClient.request]
async def get_dashboard_async(self, dashboard_id: int) -> Dict:
with belief_scope(
"AsyncSupersetClient.get_dashboard_async", f"id={dashboard_id}"
@@ -107,13 +109,14 @@ class AsyncSupersetClient(SupersetClient):
)
return cast(Dict, response)
# #endregion get_dashboard_async
# [/DEF:get_dashboard_async:Function]
# #region get_chart_async [C:3] [TYPE Function]
# @BRIEF Fetch one chart payload asynchronously.
# @POST Returns raw chart payload from Superset API.
# @DATA_CONTRACT Input[chart_id: int] -> Output[Dict]
# @RELATION CALLS -> [AsyncAPIClient.request]
# [DEF:get_chart_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch one chart payload asynchronously.
# @POST: Returns raw chart payload from Superset API.
# @DATA_CONTRACT: Input[chart_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[AsyncAPIClient.request]
async def get_chart_async(self, chart_id: int) -> Dict:
with belief_scope("AsyncSupersetClient.get_chart_async", f"id={chart_id}"):
response = await self.network.request(
@@ -121,14 +124,15 @@ class AsyncSupersetClient(SupersetClient):
)
return cast(Dict, response)
# #endregion get_chart_async
# [/DEF:get_chart_async:Function]
# #region get_dashboard_detail_async [C:3] [TYPE Function]
# @BRIEF Fetch dashboard detail asynchronously with concurrent charts/datasets requests.
# @POST Returns dashboard detail payload for overview page.
# @DATA_CONTRACT Input[dashboard_id: int] -> Output[Dict]
# @RELATION CALLS -> [get_dashboard_async]
# @RELATION CALLS -> [get_chart_async]
# [DEF:get_dashboard_detail_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch dashboard detail asynchronously with concurrent charts/datasets requests.
# @POST: Returns dashboard detail payload for overview page.
# @DATA_CONTRACT: Input[dashboard_id: int] -> Output[Dict]
# @RELATION: [CALLS] ->[get_dashboard_async]
# @RELATION: [CALLS] ->[get_chart_async]
async def get_dashboard_detail_async(self, dashboard_id: int) -> Dict:
with belief_scope(
"AsyncSupersetClient.get_dashboard_detail_async", f"id={dashboard_id}"
@@ -418,12 +422,13 @@ class AsyncSupersetClient(SupersetClient):
"dataset_count": len(datasets),
}
# #endregion get_dashboard_detail_async
# [/DEF:get_dashboard_detail_async:Function]
# #region get_dashboard_permalink_state_async [C:2] [TYPE Function]
# @BRIEF Fetch stored dashboard permalink state asynchronously.
# @POST Returns dashboard permalink state payload from Superset API.
# @DATA_CONTRACT Input[permalink_key: str] -> Output[Dict]
# [DEF:get_dashboard_permalink_state_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetch stored dashboard permalink state asynchronously.
# @POST: Returns dashboard permalink state payload from Superset API.
# @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict]
async def get_dashboard_permalink_state_async(self, permalink_key: str) -> Dict:
with belief_scope(
"AsyncSupersetClient.get_dashboard_permalink_state_async",
@@ -434,12 +439,13 @@ class AsyncSupersetClient(SupersetClient):
)
return cast(Dict, response)
# #endregion get_dashboard_permalink_state_async
# [/DEF:get_dashboard_permalink_state_async:Function]
# #region get_native_filter_state_async [C:2] [TYPE Function]
# @BRIEF Fetch stored native filter state asynchronously.
# @POST Returns native filter state payload from Superset API.
# @DATA_CONTRACT Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
# [DEF:get_native_filter_state_async:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetch stored native filter state asynchronously.
# @POST: Returns native filter state payload from Superset API.
# @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
async def get_native_filter_state_async(
self, dashboard_id: int, filter_state_key: str
) -> Dict:
@@ -453,13 +459,14 @@ class AsyncSupersetClient(SupersetClient):
)
return cast(Dict, response)
# #endregion get_native_filter_state_async
# [/DEF:get_native_filter_state_async:Function]
# #region extract_native_filters_from_permalink_async [C:3] [TYPE Function]
# @BRIEF Extract native filters dataMask from a permalink key asynchronously.
# @POST Returns extracted dataMask with filter states.
# @DATA_CONTRACT Input[permalink_key: str] -> Output[Dict]
# @RELATION CALLS -> [get_dashboard_permalink_state_async]
# [DEF:extract_native_filters_from_permalink_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters dataMask from a permalink key asynchronously.
# @POST: Returns extracted dataMask with filter states.
# @DATA_CONTRACT: Input[permalink_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[get_dashboard_permalink_state_async]
async def extract_native_filters_from_permalink_async(
self, permalink_key: str
) -> Dict:
@@ -493,13 +500,14 @@ class AsyncSupersetClient(SupersetClient):
"permalink_key": permalink_key,
}
# #endregion extract_native_filters_from_permalink_async
# [/DEF:extract_native_filters_from_permalink_async:Function]
# #region extract_native_filters_from_key_async [C:3] [TYPE Function]
# @BRIEF Extract native filters from a native_filters_key URL parameter asynchronously.
# @POST Returns extracted filter state with extraFormData.
# @DATA_CONTRACT Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
# @RELATION CALLS -> [get_native_filter_state_async]
# [DEF:extract_native_filters_from_key_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters from a native_filters_key URL parameter asynchronously.
# @POST: Returns extracted filter state with extraFormData.
# @DATA_CONTRACT: Input[dashboard_id: Union[int, str], filter_state_key: str] -> Output[Dict]
# @RELATION: [CALLS] ->[get_native_filter_state_async]
async def extract_native_filters_from_key_async(
self, dashboard_id: int, filter_state_key: str
) -> Dict:
@@ -553,14 +561,15 @@ class AsyncSupersetClient(SupersetClient):
"filter_state_key": filter_state_key,
}
# #endregion extract_native_filters_from_key_async
# [/DEF:extract_native_filters_from_key_async:Function]
# #region parse_dashboard_url_for_filters_async [C:3] [TYPE Function]
# @BRIEF Parse a Superset dashboard URL and extract native filter state asynchronously.
# @POST Returns extracted filter state or empty dict if no filters found.
# @DATA_CONTRACT Input[url: str] -> Output[Dict]
# @RELATION CALLS -> [extract_native_filters_from_permalink_async]
# @RELATION CALLS -> [extract_native_filters_from_key_async]
# [DEF:parse_dashboard_url_for_filters_async:Function]
# @COMPLEXITY: 3
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state asynchronously.
# @POST: Returns extracted filter state or empty dict if no filters found.
# @DATA_CONTRACT: Input[url: str] -> Output[Dict]
# @RELATION: [CALLS] ->[extract_native_filters_from_permalink_async]
# @RELATION: [CALLS] ->[extract_native_filters_from_key_async]
async def parse_dashboard_url_for_filters_async(self, url: str) -> Dict:
with belief_scope(
"AsyncSupersetClient.parse_dashboard_url_for_filters_async", f"url={url}"
@@ -662,7 +671,9 @@ class AsyncSupersetClient(SupersetClient):
return result
# #endregion parse_dashboard_url_for_filters_async
# [/DEF:parse_dashboard_url_for_filters_async:Function]
# #endregion AsyncSupersetClient
# #endregion AsyncSupersetClientModule

93
backend/src/core/auth/__tests__/test_auth.py
View File

@@ -1,7 +1,8 @@
# #region test_auth [C:3] [TYPE Module]
# @BRIEF Unit tests for authentication module
# @LAYER Domain
# @RELATION VERIFIES -> [AuthPackage]
# [DEF:test_auth:Module]
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for authentication module
# @LAYER: Domain
# @RELATION: VERIFIES -> AuthPackage
import sys
from pathlib import Path
@@ -57,9 +58,9 @@ def auth_repo(db_session):
return AuthRepository(db_session)
# #region test_create_user [TYPE Function]
# @BRIEF Verifies that a persisted user can be retrieved with intact credential hash.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_create_user:Function]
# @PURPOSE: Verifies that a persisted user can be retrieved with intact credential hash.
# @RELATION: BINDS_TO -> test_auth
def test_create_user(auth_repo):
"""Test user creation"""
user = User(
@@ -79,12 +80,12 @@ def test_create_user(auth_repo):
assert verify_password("testpassword123", retrieved_user.password_hash)
# #endregion test_create_user
# [/DEF:test_create_user:Function]
# #region test_authenticate_user [TYPE Function]
# @BRIEF Validates authentication outcomes for valid, wrong-password, and unknown-user cases.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_authenticate_user:Function]
# @PURPOSE: Validates authentication outcomes for valid, wrong-password, and unknown-user cases.
# @RELATION: BINDS_TO -> test_auth
def test_authenticate_user(auth_service, auth_repo):
"""Test user authentication with valid and invalid credentials"""
user = User(
@@ -111,12 +112,12 @@ def test_authenticate_user(auth_service, auth_repo):
assert invalid_user is None
# #endregion test_authenticate_user
# [/DEF:test_authenticate_user:Function]
# #region test_create_session [TYPE Function]
# @BRIEF Ensures session creation returns bearer token payload fields.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_create_session:Function]
# @PURPOSE: Ensures session creation returns bearer token payload fields.
# @RELATION: BINDS_TO -> test_auth
def test_create_session(auth_service, auth_repo):
"""Test session token creation"""
user = User(
@@ -136,12 +137,12 @@ def test_create_session(auth_service, auth_repo):
assert len(session["access_token"]) > 0
# #endregion test_create_session
# [/DEF:test_create_session:Function]
# #region test_role_permission_association [TYPE Function]
# @BRIEF Confirms role-permission many-to-many assignments persist and reload correctly.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_role_permission_association:Function]
# @PURPOSE: Confirms role-permission many-to-many assignments persist and reload correctly.
# @RELATION: BINDS_TO -> test_auth
def test_role_permission_association(auth_repo):
"""Test role and permission association"""
role = Role(name="Admin", description="System administrator")
@@ -162,12 +163,12 @@ def test_role_permission_association(auth_repo):
assert "admin:users:WRITE" in permissions
# #endregion test_role_permission_association
# [/DEF:test_role_permission_association:Function]
# #region test_user_role_association [TYPE Function]
# @BRIEF Confirms user-role assignment persists and is queryable from repository reads.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_user_role_association:Function]
# @PURPOSE: Confirms user-role assignment persists and is queryable from repository reads.
# @RELATION: BINDS_TO -> test_auth
def test_user_role_association(auth_repo):
"""Test user and role association"""
role = Role(name="Admin", description="System administrator")
@@ -190,12 +191,12 @@ def test_user_role_association(auth_repo):
assert retrieved_user.roles[0].name == "Admin"
# #endregion test_user_role_association
# [/DEF:test_user_role_association:Function]
# #region test_ad_group_mapping [TYPE Function]
# @BRIEF Verifies AD group mapping rows persist and reference the expected role.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_ad_group_mapping:Function]
# @PURPOSE: Verifies AD group mapping rows persist and reference the expected role.
# @RELATION: BINDS_TO -> test_auth
def test_ad_group_mapping(auth_repo):
"""Test AD group mapping"""
role = Role(name="ADFS_Admin", description="ADFS administrators")
@@ -217,12 +218,12 @@ def test_ad_group_mapping(auth_repo):
assert retrieved_mapping.role_id == role.id
# #endregion test_ad_group_mapping
# [/DEF:test_ad_group_mapping:Function]
# #region test_authenticate_user_updates_last_login [TYPE Function]
# @BRIEF Verifies successful authentication updates last_login audit field.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_authenticate_user_updates_last_login:Function]
# @PURPOSE: Verifies successful authentication updates last_login audit field.
# @RELATION: BINDS_TO -> test_auth
def test_authenticate_user_updates_last_login(auth_service, auth_repo):
"""@SIDE_EFFECT: authenticate_user updates last_login timestamp on success."""
user = User(
@@ -241,12 +242,12 @@ def test_authenticate_user_updates_last_login(auth_service, auth_repo):
assert authenticated.last_login is not None
# #endregion test_authenticate_user_updates_last_login
# [/DEF:test_authenticate_user_updates_last_login:Function]
# #region test_authenticate_inactive_user [TYPE Function]
# @BRIEF Verifies inactive accounts are rejected during password authentication.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_authenticate_inactive_user:Function]
# @PURPOSE: Verifies inactive accounts are rejected during password authentication.
# @RELATION: BINDS_TO -> test_auth
def test_authenticate_inactive_user(auth_service, auth_repo):
"""@PRE: User with is_active=False should not authenticate."""
user = User(
@@ -263,24 +264,24 @@ def test_authenticate_inactive_user(auth_service, auth_repo):
assert result is None
# #endregion test_authenticate_inactive_user
# [/DEF:test_authenticate_inactive_user:Function]
# #region test_verify_password_empty_hash [TYPE Function]
# @BRIEF Verifies password verification safely rejects empty or null password hashes.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_verify_password_empty_hash:Function]
# @PURPOSE: Verifies password verification safely rejects empty or null password hashes.
# @RELATION: BINDS_TO -> test_auth
def test_verify_password_empty_hash():
"""@PRE: verify_password with empty/None hash returns False."""
assert verify_password("anypassword", "") is False
assert verify_password("anypassword", None) is False
# #endregion test_verify_password_empty_hash
# [/DEF:test_verify_password_empty_hash:Function]
# #region test_provision_adfs_user_new [TYPE Function]
# @BRIEF Verifies JIT provisioning creates a new ADFS user and maps group-derived roles.
# @RELATION BINDS_TO -> [test_auth]
# [DEF:test_provision_adfs_user_new:Function]
# @PURPOSE: Verifies JIT provisioning creates a new ADFS user and maps group-derived roles.
# @RELATION: BINDS_TO -> test_auth
def test_provision_adfs_user_new(auth_service, auth_repo):
"""@POST: provision_adfs_user creates a new ADFS user with correct roles."""
# Set up a role and AD group mapping
@@ -307,7 +308,7 @@ def test_provision_adfs_user_new(auth_service, auth_repo):
assert user.roles[0].name == "ADFS_Viewer"
# #endregion test_provision_adfs_user_new
# [/DEF:test_provision_adfs_user_new:Function]
# [DEF:test_provision_adfs_user_existing:Function]
@@ -337,5 +338,5 @@ def test_provision_adfs_user_existing(auth_service, auth_repo):
assert len(user.roles) == 0 # No matching group mappings
# #endregion test_auth
# #endregion test_provision_adfs_user_existing
# [/DEF:test_auth:Module]
# [/DEF:test_provision_adfs_user_existing:Function]

18
backend/src/core/auth/config.py
View File

@@ -1,21 +1,19 @@
# #region AuthConfigModule [C:2] [TYPE Module] [SEMANTICS auth, config, settings, jwt, adfs]
#
# @BRIEF Centralized configuration for authentication and authorization.
# @LAYER Core
# @INVARIANT All sensitive configuration must have defaults or be loaded from environment.
# @RELATION DEPENDS_ON -> [pydantic]
#
# @LAYER: Core
# @RELATION DEPENDS_ON -> pydantic
#
# @INVARIANT: All sensitive configuration must have defaults or be loaded from environment.
# [SECTION: IMPORTS]
from pydantic import Field
from pydantic_settings import BaseSettings
# [/SECTION]
# #region AuthConfig [TYPE Class]
# @BRIEF Holds authentication-related settings.
# @PRE Environment variables may be provided via .env file.
# @POST Returns a configuration object with validated settings.
# @RELATION INHERITS -> [pydantic_settings.BaseSettings]
# @PRE: Environment variables may be provided via .env file.
# @POST: Returns a configuration object with validated settings.
# @RELATION INHERITS -> pydantic_settings.BaseSettings
class AuthConfig(BaseSettings):
# JWT Settings
SECRET_KEY: str = Field(default="super-secret-key-change-in-production", env="AUTH_SECRET_KEY")
@@ -41,7 +39,7 @@ class AuthConfig(BaseSettings):
# #region auth_config [TYPE Variable]
# @BRIEF Singleton instance of AuthConfig.
# @RELATION DEPENDS_ON -> [AuthConfig]
# @RELATION DEPENDS_ON -> AuthConfig
auth_config = AuthConfig()
# #endregion auth_config

22
backend/src/core/auth/jwt.py
View File

@@ -1,30 +1,25 @@
# #region AuthJwtModule [C:3] [TYPE Module] [SEMANTICS jwt, token, session, auth]
#
# @BRIEF JWT token generation and validation logic.
# @LAYER Core
# @INVARIANT Tokens must include expiration time and user identifier.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [auth_config]
# @RELATION USES -> [auth_config]
#
#
# @INVARIANT: Tokens must include expiration time and user identifier.
# [SECTION: IMPORTS]
from datetime import datetime, timedelta
from typing import Optional
from jose import jwt
from .config import auth_config
from ..logger import belief_scope
# [/SECTION]
# #region create_access_token [TYPE Function]
# @BRIEF Generates a new JWT access token.
# @PRE data dict contains 'sub' (user_id) and optional 'scopes' (roles).
# @POST Returns a signed JWT string.
# @PRE: data dict contains 'sub' (user_id) and optional 'scopes' (roles).
# @POST: Returns a signed JWT string.
# @RELATION DEPENDS_ON -> [auth_config]
#
# @PARAM: data (dict) - Payload data for the token.
# @PARAM: expires_delta (Optional[timedelta]) - Custom expiration time.
# @RETURN: str - The encoded JWT.
def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
with belief_scope("create_access_token"):
to_encode = data.copy()
@@ -47,13 +42,10 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -
# #region decode_token [TYPE Function]
# @BRIEF Decodes and validates a JWT token.
# @PRE token is a signed JWT string.
# @POST Returns the decoded payload if valid.
# @PRE: token is a signed JWT string.
# @POST: Returns the decoded payload if valid.
# @RELATION DEPENDS_ON -> [auth_config]
#
# @PARAM: token (str) - The JWT to decode.
# @RETURN: dict - The decoded payload.
# @THROW: jose.JWTError - If token is invalid or expired.
def decode_token(token: str) -> dict:
with belief_scope("decode_token"):
payload = jwt.decode(

21
backend/src/core/auth/logger.py
View File

@@ -1,24 +1,19 @@
# #region AuthLoggerModule [C:3] [TYPE Module] [SEMANTICS auth, logger, audit, security]
#
# @BRIEF Audit logging for security-related events.
# @LAYER Core
# @INVARIANT Must not log sensitive data like passwords or full tokens.
# @RELATION USES -> [belief_scope]
#
# @LAYER: Core
# @RELATION USES -> belief_scope
#
# @INVARIANT: Must not log sensitive data like passwords or full tokens.
# [SECTION: IMPORTS]
from ..logger import logger, belief_scope
from datetime import datetime
# [/SECTION]
# #region log_security_event [TYPE Function]
# @BRIEF Logs a security-related event for audit trails.
# @PRE event_type and username are strings.
# @POST Security event is written to the application log.
# @RELATION USES -> [logger]
# @PARAM: event_type (str) - Type of event (e.g., LOGIN_SUCCESS, PERMISSION_DENIED).
# @PARAM: username (str) - The user involved in the event.
# @PARAM: details (dict) - Additional non-sensitive metadata.
# @PRE: event_type and username are strings.
# @POST: Security event is written to the application log.
# @RELATION USES -> logger
def log_security_event(event_type: str, username: str, details: dict = None):
with belief_scope("log_security_event", f"{event_type}:{username}"):
timestamp = datetime.utcnow().isoformat()
@@ -28,4 +23,4 @@ def log_security_event(event_type: str, username: str, details: dict = None):
logger.info(msg)
# #endregion log_security_event
# #endregion AuthLoggerModule
# #endregion AuthLoggerModule

31
backend/src/core/auth/oauth.py
View File

@@ -1,29 +1,27 @@
# #region AuthOauthModule [C:2] [TYPE Module] [SEMANTICS auth, oauth, oidc, adfs]
#
# @BRIEF ADFS OIDC configuration and client using Authlib.
# @LAYER Core
# @INVARIANT Must use secure OIDC flows.
# @RELATION DEPENDS_ON -> [authlib]
# @RELATION USES -> [auth_config]
#
# @LAYER: Core
# @RELATION DEPENDS_ON -> authlib
# @RELATION USES -> auth_config
#
# @INVARIANT: Must use secure OIDC flows.
# [SECTION: IMPORTS]
from authlib.integrations.starlette_client import OAuth
from .config import auth_config
# [/SECTION]
# #region oauth [TYPE Variable]
# @BRIEF Global Authlib OAuth registry.
# @RELATION DEPENDS_ON -> [OAuth]
# @RELATION DEPENDS_ON -> OAuth
oauth = OAuth()
# #endregion oauth
# #region register_adfs [TYPE Function]
# @BRIEF Registers the ADFS OIDC client.
# @PRE ADFS configuration is provided in auth_config.
# @POST ADFS client is registered in oauth registry.
# @RELATION USES -> [oauth]
# @RELATION USES -> [auth_config]
# @PRE: ADFS configuration is provided in auth_config.
# @POST: ADFS client is registered in oauth registry.
# @RELATION USES -> oauth
# @RELATION USES -> auth_config
def register_adfs():
if auth_config.ADFS_CLIENT_ID:
oauth.register(
@@ -39,10 +37,9 @@ def register_adfs():
# #region is_adfs_configured [TYPE Function]
# @BRIEF Checks if ADFS is properly configured.
# @PRE None.
# @POST Returns True if ADFS client is registered, False otherwise.
# @RETURN bool - Configuration status.
# @RELATION USES -> [oauth]
# @PRE: None.
# @POST: Returns True if ADFS client is registered, False otherwise.
# @RELATION USES -> oauth
def is_adfs_configured() -> bool:
"""Check if ADFS OAuth client is registered."""
return 'adfs' in oauth._registry
@@ -51,4 +48,4 @@ def is_adfs_configured() -> bool:
# Initial registration
register_adfs()
# #endregion AuthOauthModule
# #endregion AuthOauthModule

108
backend/src/core/auth/repository.py
View File

@@ -1,39 +1,37 @@
# #region AuthRepositoryModule [C:5] [TYPE Module] [SEMANTICS auth, repository, database, user, role, permission]
# @BRIEF Data access layer for authentication and user preference entities.
# @LAYER Domain
# @PRE Database connection is active.
# @POST Provides valid access to identity data.
# @SIDE_EFFECT Executes database read queries through the injected SQLAlchemy session boundary.
# @DATA_CONTRACT Input[sqlalchemy.orm.Session] -> Output[User|Role|Permission|UserDashboardPreference access]
# @INVARIANT All database read/write operations must execute via the injected SQLAlchemy session boundary.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [AuthModels]
# @RELATION DEPENDS_ON -> [ProfileModels]
# @RELATION DEPENDS_ON -> [belief_scope]
# @INVARIANT: All database read/write operations must execute via the injected SQLAlchemy session boundary.
# @DATA_CONTRACT: Input[sqlalchemy.orm.Session] -> Output[User|Role|Permission|UserDashboardPreference access]
# @PRE: Database connection is active.
# @POST: Provides valid access to identity data.
# @SIDE_EFFECT: Executes database read queries through the injected SQLAlchemy session boundary.
# [SECTION: IMPORTS]
from typing import List, Optional
from sqlalchemy.orm import Session, selectinload
from ...models.auth import Permission, Role, User, ADGroupMapping
from ...models.profile import UserDashboardPreference
from ..logger import belief_scope, logger
# [/SECTION]
# #region AuthRepository [TYPE Class]
# @BRIEF Provides low-level CRUD operations for identity and authorization records.
# @PRE Database session is bound.
# @POST Entity instances returned safely.
# @SIDE_EFFECT Performs database reads.
# @PRE: Database session is bound.
# @POST: Entity instances returned safely.
# @SIDE_EFFECT: Performs database reads.
# @RELATION DEPENDS_ON -> [AuthModels]
class AuthRepository:
def __init__(self, db: Session):
self.db = db
# #region get_user_by_id [TYPE Function]
# @BRIEF Retrieve user by UUID.
# @PRE user_id is a valid UUID string.
# @POST Returns User object if found, else None.
# @RELATION DEPENDS_ON -> [User]
# [DEF:get_user_by_id:Function]
# @PURPOSE: Retrieve user by UUID.
# @PRE: user_id is a valid UUID string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_id(self, user_id: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_id"):
logger.reason(f"Fetching user by id: {user_id}")
@@ -41,13 +39,13 @@ class AuthRepository:
logger.reflect(f"User found: {result is not None}")
return result
# #endregion get_user_by_id
# [/DEF:get_user_by_id:Function]
# #region get_user_by_username [TYPE Function]
# @BRIEF Retrieve user by username.
# @PRE username is a non-empty string.
# @POST Returns User object if found, else None.
# @RELATION DEPENDS_ON -> [User]
# [DEF:get_user_by_username:Function]
# @PURPOSE: Retrieve user by username.
# @PRE: username is a non-empty string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_username(self, username: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_username"):
logger.reason(f"Fetching user by username: {username}")
@@ -55,12 +53,12 @@ class AuthRepository:
logger.reflect(f"User found: {result is not None}")
return result
# #endregion get_user_by_username
# [/DEF:get_user_by_username:Function]
# #region get_role_by_id [TYPE Function]
# @BRIEF Retrieve role by UUID with permissions preloaded.
# @RELATION DEPENDS_ON -> [Role]
# @RELATION DEPENDS_ON -> [Permission]
# [DEF:get_role_by_id:Function]
# @PURPOSE: Retrieve role by UUID with permissions preloaded.
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
def get_role_by_id(self, role_id: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_id"):
return (
@@ -70,31 +68,31 @@ class AuthRepository:
.first()
)
# #endregion get_role_by_id
# [/DEF:get_role_by_id:Function]
# #region get_role_by_name [TYPE Function]
# @BRIEF Retrieve role by unique name.
# @RELATION DEPENDS_ON -> [Role]
# [DEF:get_role_by_name:Function]
# @PURPOSE: Retrieve role by unique name.
# @RELATION: DEPENDS_ON -> [Role]
def get_role_by_name(self, name: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_name"):
return self.db.query(Role).filter(Role.name == name).first()
# #endregion get_role_by_name
# [/DEF:get_role_by_name:Function]
# #region get_permission_by_id [TYPE Function]
# @BRIEF Retrieve permission by UUID.
# @RELATION DEPENDS_ON -> [Permission]
# [DEF:get_permission_by_id:Function]
# @PURPOSE: Retrieve permission by UUID.
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_id(self, permission_id: str) -> Optional[Permission]:
with belief_scope("AuthRepository.get_permission_by_id"):
return (
self.db.query(Permission).filter(Permission.id == permission_id).first()
)
# #endregion get_permission_by_id
# [/DEF:get_permission_by_id:Function]
# #region get_permission_by_resource_action [TYPE Function]
# @BRIEF Retrieve permission by resource and action tuple.
# @RELATION DEPENDS_ON -> [Permission]
# [DEF:get_permission_by_resource_action:Function]
# @PURPOSE: Retrieve permission by resource and action tuple.
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_resource_action(
self, resource: str, action: str
) -> Optional[Permission]:
@@ -105,20 +103,20 @@ class AuthRepository:
.first()
)
# #endregion get_permission_by_resource_action
# [/DEF:get_permission_by_resource_action:Function]
# #region list_permissions [TYPE Function]
# @BRIEF List all system permissions.
# @RELATION DEPENDS_ON -> [Permission]
# [DEF:list_permissions:Function]
# @PURPOSE: List all system permissions.
# @RELATION: DEPENDS_ON -> [Permission]
def list_permissions(self) -> List[Permission]:
with belief_scope("AuthRepository.list_permissions"):
return self.db.query(Permission).all()
# #endregion list_permissions
# [/DEF:list_permissions:Function]
# #region get_user_dashboard_preference [TYPE Function]
# @BRIEF Retrieve dashboard filters/preferences for a user.
# @RELATION DEPENDS_ON -> [UserDashboardPreference]
# [DEF:get_user_dashboard_preference:Function]
# @PURPOSE: Retrieve dashboard filters/preferences for a user.
# @RELATION: DEPENDS_ON -> [UserDashboardPreference]
def get_user_dashboard_preference(
self, user_id: str
) -> Optional[UserDashboardPreference]:
@@ -129,14 +127,14 @@ class AuthRepository:
.first()
)
# #endregion get_user_dashboard_preference
# [/DEF:get_user_dashboard_preference:Function]
# #region get_roles_by_ad_groups [TYPE Function]
# @BRIEF Retrieve roles that match a list of AD group names.
# @PRE groups is a list of strings representing AD group identifiers.
# @POST Returns a list of Role objects mapped to the provided AD groups.
# @RELATION DEPENDS_ON -> [Role]
# @RELATION DEPENDS_ON -> [ADGroupMapping]
# [DEF:get_roles_by_ad_groups:Function]
# @PURPOSE: Retrieve roles that match a list of AD group names.
# @PRE: groups is a list of strings representing AD group identifiers.
# @POST: Returns a list of Role objects mapped to the provided AD groups.
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [ADGroupMapping]
def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]:
with belief_scope("AuthRepository.get_roles_by_ad_groups"):
logger.reason(f"Fetching roles for AD groups: {groups}")
@@ -149,7 +147,7 @@ class AuthRepository:
.all()
)
# #endregion get_roles_by_ad_groups
# [/DEF:get_roles_by_ad_groups:Function]
# #endregion AuthRepository

27
backend/src/core/auth/security.py
View File

@@ -1,24 +1,19 @@
# #region AuthSecurityModule [C:2] [TYPE Module] [SEMANTICS security, password, hashing, bcrypt]
#
# @BRIEF Utility for password hashing and verification using Passlib.
# @LAYER Core
# @INVARIANT Uses bcrypt for hashing with standard work factor.
# @RELATION DEPENDS_ON -> [bcrypt]
#
# @LAYER: Core
# @RELATION DEPENDS_ON -> bcrypt
#
# @INVARIANT: Uses bcrypt for hashing with standard work factor.
# [SECTION: IMPORTS]
import bcrypt
# [/SECTION]
# #region verify_password [TYPE Function]
# @BRIEF Verifies a plain password against a hashed password.
# @PRE plain_password is a string, hashed_password is a bcrypt hash.
# @POST Returns True if password matches, False otherwise.
# @RELATION DEPENDS_ON -> [bcrypt]
# @PRE: plain_password is a string, hashed_password is a bcrypt hash.
# @POST: Returns True if password matches, False otherwise.
# @RELATION DEPENDS_ON -> bcrypt
#
# @PARAM: plain_password (str) - The unhashed password.
# @PARAM: hashed_password (str) - The stored hash.
# @RETURN: bool - Verification result.
def verify_password(plain_password: str, hashed_password: str) -> bool:
if not hashed_password:
return False
@@ -33,12 +28,10 @@ def verify_password(plain_password: str, hashed_password: str) -> bool:
# #region get_password_hash [TYPE Function]
# @BRIEF Generates a bcrypt hash for a plain password.
# @PRE password is a string.
# @POST Returns a secure bcrypt hash string.
# @RELATION DEPENDS_ON -> [bcrypt]
# @PRE: password is a string.
# @POST: Returns a secure bcrypt hash string.
# @RELATION DEPENDS_ON -> bcrypt
#
# @PARAM: password (str) - The password to hash.
# @RETURN: str - The generated hash.
def get_password_hash(password: str) -> str:
return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8")
# #endregion get_password_hash

330
backend/src/core/config_manager.py
View File

@@ -1,18 +1,18 @@
# #region ConfigManager [C:5] [TYPE Module] [SEMANTICS config, manager, persistence, migration, postgresql]
#
# @BRIEF Manages application configuration persistence in DB with one-time migration from legacy JSON.
# @LAYER Domain
# @PRE Database schema for AppConfigRecord must be initialized.
# @POST Configuration is loaded into memory and logger is configured.
# @SIDE_EFFECT Performs DB I/O and may update global logging level.
# @DATA_CONTRACT Input[json, record] -> Model[AppConfig]
# @INVARIANT Configuration must always be representable by AppConfig and persisted under global record id.
# @LAYER: Domain
# @PRE: Database schema for AppConfigRecord must be initialized.
# @POST: Configuration is loaded into memory and logger is configured.
# @SIDE_EFFECT: Performs DB I/O and may update global logging level.
# @DATA_CONTRACT: Input[json, record] -> Model[AppConfig]
# @INVARIANT: Configuration must always be representable by AppConfig and persisted under global record id.
# @RELATION DEPENDS_ON -> [AppConfig]
# @RELATION DEPENDS_ON -> [SessionLocal]
# @RELATION DEPENDS_ON -> [AppConfigRecord]
# @RELATION CALLS -> [logger]
# @RELATION CALLS -> [configure_logger]
#
#
import json
import os
from pathlib import Path
@@ -24,30 +24,30 @@ from .config_models import AppConfig, Environment, GlobalSettings
from .database import SessionLocal
from ..models.config import AppConfigRecord
from ..models.mapping import Environment as EnvironmentRecord
from .logger import configure_logger, belief_scope
from .cot_logger import MarkerLogger
from .logger import logger, configure_logger, belief_scope
log = MarkerLogger("ConfigManager")
# #region ConfigManager [C:5] [TYPE Class]
# @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle.
# @PRE Database is accessible and AppConfigRecord schema is loaded.
# @POST Configuration state is synchronized between memory and database.
# @SIDE_EFFECT Performs DB I/O, OS path validation, and logger reconfiguration.
# @PRE: Database is accessible and AppConfigRecord schema is loaded.
# @POST: Configuration state is synchronized between memory and database.
# @SIDE_EFFECT: Performs DB I/O, OS path validation, and logger reconfiguration.
class ConfigManager:
# #region __init__ [TYPE Function]
# @BRIEF Initialize manager state from persisted or migrated configuration.
# @PRE config_path is a non-empty string path.
# @POST self.config is initialized as AppConfig and logger is configured.
# @SIDE_EFFECT Reads config sources and updates logging configuration.
# @DATA_CONTRACT Input(str config_path) -> Output(None; self.config: AppConfig)
# [DEF:__init__:Function]
# @PURPOSE: Initialize manager state from persisted or migrated configuration.
# @PRE: config_path is a non-empty string path.
# @POST: self.config is initialized as AppConfig and logger is configured.
# @SIDE_EFFECT: Reads config sources and updates logging configuration.
# @DATA_CONTRACT: Input(str config_path) -> Output(None; self.config: AppConfig)
def __init__(self, config_path: str = "config.json"):
with belief_scope("ConfigManager.__init__"):
if not isinstance(config_path, str) or not config_path:
log.explore("Invalid config_path provided", error="config_path must be a non-empty string", payload={"path": config_path})
logger.explore(
"Invalid config_path provided", extra={"path": config_path}
)
raise ValueError("config_path must be a non-empty string")
log.reason(f"Initializing ConfigManager with legacy path: {config_path}")
logger.reason(f"Initializing ConfigManager with legacy path: {config_path}")
self.config_path = Path(config_path)
self.raw_payload: dict[str, Any] = {}
@@ -56,17 +56,20 @@ class ConfigManager:
configure_logger(self.config.settings.logging)
if not isinstance(self.config, AppConfig):
log.explore("Config loading resulted in invalid type", error="Loaded config is not an AppConfig instance", payload={"type": type(self.config)})
logger.explore(
"Config loading resulted in invalid type",
extra={"type": type(self.config)},
)
raise TypeError("self.config must be an instance of AppConfig")
log.reflect("ConfigManager initialization complete")
logger.reflect("ConfigManager initialization complete")
# #endregion __init__
# [/DEF:__init__:Function]
# #region _apply_features_from_env [TYPE Function]
# @BRIEF Read FEATURES__* env vars and apply them to a GlobalSettings features config.
# @SIDE_EFFECT Reads os.environ; mutates settings.features in-place.
# @RATIONALE Env vars seed the initial defaults. After first bootstrap, DB is source of truth.
# [DEF:_apply_features_from_env:Function]
# @PURPOSE: Read FEATURES__* env vars and apply them to a GlobalSettings features config.
# @SIDE_EFFECT: Reads os.environ; mutates settings.features in-place.
# @RATIONALE: Env vars seed the initial defaults. After first bootstrap, DB is source of truth.
@staticmethod
def _apply_features_from_env(settings: GlobalSettings) -> None:
with belief_scope("ConfigManager._apply_features_from_env"):
@@ -74,35 +77,35 @@ class ConfigManager:
if dataset_review_env is not None:
parsed = dataset_review_env.strip().lower() in ("true", "1", "yes")
settings.features.dataset_review = parsed
log.reason(
logger.reason(
"Applied FEATURES__DATASET_REVIEW from env",
payload={"value": parsed, "raw": dataset_review_env},
extra={"value": parsed, "raw": dataset_review_env},
)
health_monitor_env = os.getenv("FEATURES__HEALTH_MONITOR")
if health_monitor_env is not None:
parsed = health_monitor_env.strip().lower() in ("true", "1", "yes")
settings.features.health_monitor = parsed
log.reason(
logger.reason(
"Applied FEATURES__HEALTH_MONITOR from env",
payload={"value": parsed, "raw": health_monitor_env},
extra={"value": parsed, "raw": health_monitor_env},
)
# #endregion _apply_features_from_env
# [/DEF:_apply_features_from_env:Function]
# #region _default_config [TYPE Function]
# @BRIEF Build default application configuration fallback.
# [DEF:_default_config:Function]
# @PURPOSE: Build default application configuration fallback.
def _default_config(self) -> AppConfig:
with belief_scope("ConfigManager._default_config"):
log.reason("Building default AppConfig fallback")
logger.reason("Building default AppConfig fallback")
config = AppConfig(environments=[], settings=GlobalSettings())
self._apply_features_from_env(config.settings)
return config
# #endregion _default_config
# [/DEF:_default_config:Function]
# #region _sync_raw_payload_from_config [TYPE Function]
# @BRIEF Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.
# [DEF:_sync_raw_payload_from_config:Function]
# @PURPOSE: Merge typed AppConfig state into raw payload while preserving unsupported legacy sections.
def _sync_raw_payload_from_config(self) -> dict[str, Any]:
with belief_scope("ConfigManager._sync_raw_payload_from_config"):
typed_payload = self.config.model_dump()
@@ -110,9 +113,9 @@ class ConfigManager:
merged_payload["environments"] = typed_payload.get("environments", [])
merged_payload["settings"] = typed_payload.get("settings", {})
self.raw_payload = merged_payload
log.reason(
logger.reason(
"Synchronized raw payload from typed config",
payload={
extra={
"environments_count": len(
merged_payload.get("environments", []) or []
),
@@ -126,42 +129,45 @@ class ConfigManager:
)
return merged_payload
# #endregion _sync_raw_payload_from_config
# [/DEF:_sync_raw_payload_from_config:Function]
# #region _load_from_legacy_file [TYPE Function]
# @BRIEF Load legacy JSON configuration for migration fallback path.
# [DEF:_load_from_legacy_file:Function]
# @PURPOSE: Load legacy JSON configuration for migration fallback path.
def _load_from_legacy_file(self) -> dict[str, Any]:
with belief_scope("ConfigManager._load_from_legacy_file"):
if not self.config_path.exists():
log.reason(
logger.reason(
"Legacy config file not found; using default payload",
payload={"path": str(self.config_path)},
extra={"path": str(self.config_path)},
)
return {}
log.reason(
"Loading legacy config file", payload={"path": str(self.config_path)}
logger.reason(
"Loading legacy config file", extra={"path": str(self.config_path)}
)
with self.config_path.open("r", encoding="utf-8") as fh:
payload = json.load(fh)
if not isinstance(payload, dict):
log.explore("Legacy config payload is not a JSON object", error=f"Expected dict, got {type(payload).__name__}", payload={
"path": str(self.config_path),
"type": type(payload).__name__,
})
logger.explore(
"Legacy config payload is not a JSON object",
extra={
"path": str(self.config_path),
"type": type(payload).__name__,
},
)
raise ValueError("Legacy config payload must be a JSON object")
log.reason(
logger.reason(
"Legacy config file loaded successfully",
payload={"path": str(self.config_path), "keys": sorted(payload.keys())},
extra={"path": str(self.config_path), "keys": sorted(payload.keys())},
)
return payload
# #endregion _load_from_legacy_file
# [/DEF:_load_from_legacy_file:Function]
# #region _get_record [TYPE Function]
# @BRIEF Resolve global configuration record from DB.
# [DEF:_get_record:Function]
# @PURPOSE: Resolve global configuration record from DB.
def _get_record(self, session: Session) -> Optional[AppConfigRecord]:
with belief_scope("ConfigManager._get_record"):
record = (
@@ -169,24 +175,24 @@ class ConfigManager:
.filter(AppConfigRecord.id == "global")
.first()
)
log.reason(
"Resolved app config record", payload={"exists": record is not None}
logger.reason(
"Resolved app config record", extra={"exists": record is not None}
)
return record
# #endregion _get_record
# [/DEF:_get_record:Function]
# #region _load_config [TYPE Function]
# @BRIEF Load configuration from DB or perform one-time migration from legacy JSON.
# [DEF:_load_config:Function]
# @PURPOSE: Load configuration from DB or perform one-time migration from legacy JSON.
def _load_config(self) -> AppConfig:
with belief_scope("ConfigManager._load_config"):
session = SessionLocal()
try:
record = self._get_record(session)
if record and isinstance(record.payload, dict):
log.reason(
logger.reason(
"Loading configuration from database",
payload={"record_id": record.id},
extra={"record_id": record.id},
)
self.raw_payload = dict(record.payload)
config = AppConfig.model_validate(
@@ -197,18 +203,18 @@ class ConfigManager:
)
self._sync_environment_records(session, config)
session.commit()
log.reason(
logger.reason(
"Database configuration validated successfully",
payload={
extra={
"environments_count": len(config.environments),
"payload_keys": sorted(self.raw_payload.keys()),
},
)
return config
log.reason(
logger.reason(
"Database configuration record missing; attempting legacy file migration",
payload={"legacy_path": str(self.config_path)},
extra={"legacy_path": str(self.config_path)},
)
legacy_payload = self._load_from_legacy_file()
@@ -221,9 +227,9 @@ class ConfigManager:
}
)
self._apply_features_from_env(config.settings)
log.reason(
logger.reason(
"Legacy payload validated; persisting migrated configuration to database",
payload={
extra={
"environments_count": len(config.environments),
"payload_keys": sorted(self.raw_payload.keys()),
},
@@ -231,7 +237,7 @@ class ConfigManager:
self._save_config_to_db(config, session=session)
return config
log.reason(
logger.reason(
"No persisted config found; falling back to default configuration"
)
config = self._default_config()
@@ -239,20 +245,26 @@ class ConfigManager:
self._save_config_to_db(config, session=session)
return config
except (json.JSONDecodeError, TypeError, ValueError) as exc:
log.explore("Recoverable config load failure; falling back to default configuration", error=str(exc), payload={"legacy_path": str(self.config_path)})
logger.explore(
"Recoverable config load failure; falling back to default configuration",
extra={"error": str(exc), "legacy_path": str(self.config_path)},
)
config = self._default_config()
self.raw_payload = config.model_dump()
return config
except Exception as exc:
log.explore("Critical config load failure; re-raising persistence or validation error", error=str(exc))
logger.explore(
"Critical config load failure; re-raising persistence or validation error",
extra={"error": str(exc)},
)
raise
finally:
session.close()
# #endregion _load_config
# [/DEF:_load_config:Function]
# #region _sync_environment_records [TYPE Function]
# @BRIEF Mirror configured environments into the relational environments table used by FK-backed domain models.
# [DEF:_sync_environment_records:Function]
# @PURPOSE: Mirror configured environments into the relational environments table used by FK-backed domain models.
def _sync_environment_records(self, session: Session, config: AppConfig) -> None:
with belief_scope("ConfigManager._sync_environment_records"):
configured_envs = list(config.environments or [])
@@ -276,9 +288,9 @@ class ConfigManager:
record = persisted_by_id.get(normalized_id)
if record is None:
log.reason(
logger.reason(
"Creating relational environment record from typed config",
payload={
extra={
"environment_id": normalized_id,
"environment_name": display_name,
},
@@ -297,10 +309,37 @@ class ConfigManager:
record.url = normalized_url
record.credentials_id = credentials_id
# #endregion _sync_environment_records
# [/DEF:_sync_environment_records:Function]
# #region _save_config_to_db [TYPE Function]
# @BRIEF Persist provided AppConfig into the global DB configuration record.
# [DEF:_delete_stale_environment_records:Function]
# @PURPOSE: Remove persisted environment records that are no longer in the configured environments.
# @PRE: _sync_environment_records must already have run so the query returns a current view.
# @POST: Stale Environment rows are deleted via the session (caller must commit).
# @SIDE_EFFECT: Only safe to call during explicit save (_save_config_to_db), NOT during _load_config,
# because FK-dependent rows (task_records, database_mappings, etc.) may reference deleted rows.
def _delete_stale_environment_records(
self, session: Session, config: AppConfig
) -> None:
with belief_scope("ConfigManager._delete_stale_environment_records"):
persisted_records = session.query(EnvironmentRecord).all()
configured_ids = {
str(env.id or "").strip()
for env in (config.environments or [])
if str(env.id or "").strip()
}
for record in persisted_records:
record_id = str(record.id or "").strip()
if record_id and record_id not in configured_ids:
logger.reason(
"Deleting stale environment record",
extra={"environment_id": record_id},
)
session.delete(record)
# [/DEF:_delete_stale_environment_records:Function]
# [DEF:_save_config_to_db:Function]
# @PURPOSE: Persist provided AppConfig into the global DB configuration record.
def _save_config_to_db(
self, config: AppConfig, session: Optional[Session] = None
) -> None:
@@ -312,22 +351,23 @@ class ConfigManager:
payload = self._sync_raw_payload_from_config()
record = self._get_record(db)
if record is None:
log.reason("Creating new global app config record")
logger.reason("Creating new global app config record")
record = AppConfigRecord(id="global", payload=payload)
db.add(record)
else:
log.reason(
logger.reason(
"Updating existing global app config record",
payload={"record_id": record.id},
extra={"record_id": record.id},
)
record.payload = payload
self._sync_environment_records(db, config)
self._delete_stale_environment_records(db, config)
db.commit()
log.reason(
logger.reason(
"Configuration persisted to database",
payload={
extra={
"environments_count": len(
payload.get("environments", []) or []
),
@@ -336,54 +376,54 @@ class ConfigManager:
)
except Exception:
db.rollback()
log.explore("Database save failed; transaction rolled back", error="Database transaction rolled back")
logger.explore("Database save failed; transaction rolled back")
raise
finally:
if owns_session:
db.close()
# #endregion _save_config_to_db
# [/DEF:_save_config_to_db:Function]
# #region save [TYPE Function]
# @BRIEF Persist current in-memory configuration state.
# [DEF:save:Function]
# @PURPOSE: Persist current in-memory configuration state.
def save(self) -> None:
with belief_scope("ConfigManager.save"):
log.reason("Persisting current in-memory configuration")
logger.reason("Persisting current in-memory configuration")
self._save_config_to_db(self.config)
# #endregion save
# [/DEF:save:Function]
# #region get_config [TYPE Function]
# @BRIEF Return current in-memory configuration snapshot.
# [DEF:get_config:Function]
# @PURPOSE: Return current in-memory configuration snapshot.
def get_config(self) -> AppConfig:
with belief_scope("ConfigManager.get_config"):
return self.config
# #endregion get_config
# [/DEF:get_config:Function]
# #region get_payload [TYPE Function]
# @BRIEF Return full persisted payload including sections outside typed AppConfig schema.
# [DEF:get_payload:Function]
# @PURPOSE: Return full persisted payload including sections outside typed AppConfig schema.
def get_payload(self) -> dict[str, Any]:
with belief_scope("ConfigManager.get_payload"):
return self._sync_raw_payload_from_config()
# #endregion get_payload
# [/DEF:get_payload:Function]
# #region save_config [TYPE Function]
# @BRIEF Persist configuration provided either as typed AppConfig or raw payload dict.
# [DEF:save_config:Function]
# @PURPOSE: Persist configuration provided either as typed AppConfig or raw payload dict.
def save_config(self, config: Any) -> AppConfig:
with belief_scope("ConfigManager.save_config"):
if isinstance(config, AppConfig):
log.reason("Saving typed AppConfig payload")
logger.reason("Saving typed AppConfig payload")
self.config = config
self.raw_payload = config.model_dump()
self._save_config_to_db(config)
return self.config
if isinstance(config, dict):
log.reason(
logger.reason(
"Saving raw config payload",
payload={"keys": sorted(config.keys())},
extra={"keys": sorted(config.keys())},
)
self.raw_payload = dict(config)
typed_config = AppConfig.model_validate(
@@ -396,24 +436,27 @@ class ConfigManager:
self._save_config_to_db(typed_config)
return self.config
log.explore("Unsupported config type supplied to save_config", error=f"Expected AppConfig or dict, got {type(config).__name__}", payload={"type": type(config).__name__})
logger.explore(
"Unsupported config type supplied to save_config",
extra={"type": type(config).__name__},
)
raise TypeError("config must be AppConfig or dict")
# #endregion save_config
# [/DEF:save_config:Function]
# #region update_global_settings [TYPE Function]
# @BRIEF Replace global settings and persist the resulting configuration.
# [DEF:update_global_settings:Function]
# @PURPOSE: Replace global settings and persist the resulting configuration.
def update_global_settings(self, settings: GlobalSettings) -> AppConfig:
with belief_scope("ConfigManager.update_global_settings"):
log.reason("Updating global settings")
logger.reason("Updating global settings")
self.config.settings = settings
self.save()
return self.config
# #endregion update_global_settings
# [/DEF:update_global_settings:Function]
# #region validate_path [TYPE Function]
# @BRIEF Validate that path exists and is writable, creating it when absent.
# [DEF:validate_path:Function]
# @PURPOSE: Validate that path exists and is writable, creating it when absent.
def validate_path(self, path: str) -> tuple[bool, str]:
with belief_scope("ConfigManager.validate_path", f"path={path}"):
try:
@@ -431,32 +474,34 @@ class ConfigManager:
fh.write("ok")
test_file.unlink(missing_ok=True)
log.reason("Path validation succeeded", payload={"path": str(target)})
logger.reason("Path validation succeeded", extra={"path": str(target)})
return True, "OK"
except Exception as exc:
log.explore("Path validation failed", error=str(exc), payload={"path": path})
logger.explore(
"Path validation failed", extra={"path": path, "error": str(exc)}
)
return False, str(exc)
# #endregion validate_path
# [/DEF:validate_path:Function]
# #region get_environments [TYPE Function]
# @BRIEF Return all configured environments.
# [DEF:get_environments:Function]
# @PURPOSE: Return all configured environments.
def get_environments(self) -> List[Environment]:
with belief_scope("ConfigManager.get_environments"):
return list(self.config.environments)
# #endregion get_environments
# [/DEF:get_environments:Function]
# #region has_environments [TYPE Function]
# @BRIEF Check whether at least one environment exists in configuration.
# [DEF:has_environments:Function]
# @PURPOSE: Check whether at least one environment exists in configuration.
def has_environments(self) -> bool:
with belief_scope("ConfigManager.has_environments"):
return len(self.config.environments) > 0
# #endregion has_environments
# [/DEF:has_environments:Function]
# #region get_environment [TYPE Function]
# @BRIEF Resolve a configured environment by identifier.
# [DEF:get_environment:Function]
# @PURPOSE: Resolve a configured environment by identifier.
def get_environment(self, env_id: str) -> Optional[Environment]:
with belief_scope("ConfigManager.get_environment", f"env_id={env_id}"):
normalized = str(env_id or "").strip()
@@ -468,10 +513,10 @@ class ConfigManager:
return env
return None
# #endregion get_environment
# [/DEF:get_environment:Function]
# #region add_environment [TYPE Function]
# @BRIEF Upsert environment by id into configuration and persist.
# [DEF:add_environment:Function]
# @PURPOSE: Upsert environment by id into configuration and persist.
def add_environment(self, env: Environment) -> AppConfig:
with belief_scope("ConfigManager.add_environment", f"env_id={env.id}"):
existing_index = next(
@@ -487,12 +532,12 @@ class ConfigManager:
item.is_default = False
if existing_index is None:
log.reason("Appending new environment", payload={"env_id": env.id})
logger.reason("Appending new environment", extra={"env_id": env.id})
self.config.environments.append(env)
else:
log.reason(
logger.reason(
"Replacing existing environment during add",
payload={"env_id": env.id},
extra={"env_id": env.id},
)
self.config.environments[existing_index] = env
@@ -504,10 +549,10 @@ class ConfigManager:
self.save()
return self.config
# #endregion add_environment
# [/DEF:add_environment:Function]
# #region update_environment [TYPE Function]
# @BRIEF Update existing environment by id and preserve masked password placeholder behavior.
# [DEF:update_environment:Function]
# @PURPOSE: Update existing environment by id and preserve masked password placeholder behavior.
def update_environment(self, env_id: str, env: Environment) -> bool:
with belief_scope("ConfigManager.update_environment", f"env_id={env_id}"):
for index, existing in enumerate(self.config.environments):
@@ -527,17 +572,19 @@ class ConfigManager:
updated.is_default = True
self.config.environments[index] = updated
log.reason("Environment updated", payload={"env_id": env_id})
logger.reason("Environment updated", extra={"env_id": env_id})
self.save()
return True
log.explore("Environment update skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id})
logger.explore(
"Environment update skipped; env not found", extra={"env_id": env_id}
)
return False
# #endregion update_environment
# [/DEF:update_environment:Function]
# #region delete_environment [TYPE Function]
# @BRIEF Delete environment by id and persist when deletion occurs.
# [DEF:delete_environment:Function]
# @PURPOSE: Delete environment by id and persist when deletion occurs.
def delete_environment(self, env_id: str) -> bool:
with belief_scope("ConfigManager.delete_environment", f"env_id={env_id}"):
before = len(self.config.environments)
@@ -547,7 +594,10 @@ class ConfigManager:
]
if len(self.config.environments) == before:
log.explore("Environment delete skipped; env not found", error=f"No matching environment found for id: {env_id}", payload={"env_id": env_id})
logger.explore(
"Environment delete skipped; env not found",
extra={"env_id": env_id},
)
return False
if removed and removed[0].is_default and self.config.environments:
@@ -559,14 +609,14 @@ class ConfigManager:
)
self.config.settings.default_environment_id = replacement
log.reason(
logger.reason(
"Environment deleted",
payload={"env_id": env_id, "remaining": len(self.config.environments)},
extra={"env_id": env_id, "remaining": len(self.config.environments)},
)
self.save()
return True
# #endregion delete_environment
# [/DEF:delete_environment:Function]
# #endregion ConfigManager

4
backend/src/core/config_models.py
View File

@@ -1,6 +1,6 @@
# #region ConfigModels [C:3] [TYPE Module] [SEMANTICS config, models, pydantic]
# @BRIEF Defines the data models for application configuration using Pydantic.
# @LAYER Core
# @LAYER: Core
# @RELATION IMPLEMENTS -> [CoreContracts]
# @RELATION IMPLEMENTS -> [ConnectionContracts]
@@ -71,7 +71,7 @@ class CleanReleaseConfig(BaseModel):
# #region FeaturesConfig [C:1] [TYPE DataClass]
# @BRIEF Top-level feature flags that toggle entire project features on/off.
# @RATIONALE Features are read from environment variables on bootstrap and persisted in DB.
# @RATIONALE: Features are read from environment variables on bootstrap and persisted in DB.
# DB is source of truth after initial bootstrap; env vars only seed defaults.
class FeaturesConfig(BaseModel):
dataset_review: bool = True

4
backend/src/core/cot_logger.py
View File

@@ -3,8 +3,8 @@
# Uses ContextVar for trace_id and span_id propagation across async contexts.
# Provides log(), MarkerLogger, seed_trace_id(), push_span(), pop_span().
# @LAYER: Core
# @RELATION: [CALLED_BY] -> [TraceContextMiddleware]
# @RELATION: [CALLED_BY] -> [All C4+ service and route modules]
# @RELATION CALLED_BY -> [TraceContextMiddleware]
# @RELATION CALLED_BY -> [All C4+ service and route modules]
# @PRE: Python 3.7+ (ContextVar available).
# @POST: JSON log records written to the 'cot' logger at appropriate levels.
# @SIDE_EFFECT: Writes structured JSON to the 'cot' Python logger.

122
backend/src/core/database.py
View File

@@ -1,14 +1,13 @@
# #region DatabaseModule [C:3] [TYPE Module] [SEMANTICS database, postgresql, sqlalchemy, session, persistence]
#
# @BRIEF Configures database connection and session management (PostgreSQL-first).
# @LAYER Core
# @INVARIANT A single engine instance is used for the entire application.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [auth_config]
# @RELATION DEPENDS_ON -> [ConnectionConfig]
#
#
# @INVARIANT: A single engine instance is used for the entire application.
# [SECTION: IMPORTS]
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.orm import sessionmaker
from ..models.mapping import Base
@@ -28,7 +27,6 @@ from .logger import belief_scope, logger
from .auth.config import auth_config
import os
from pathlib import Path
# [/SECTION]
# #region BASE_DIR [C:1] [TYPE Variable]
# @BRIEF Base directory for the backend.
@@ -58,7 +56,7 @@ AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL
# #region engine [C:1] [TYPE Variable]
# @BRIEF SQLAlchemy engine for mappings database.
# @SIDE_EFFECT Creates database engine and manages connection pool.
# @SIDE_EFFECT: Creates database engine and manages connection pool.
def _build_engine(db_url: str):
with belief_scope("_build_engine"):
if db_url.startswith("sqlite"):
@@ -81,27 +79,27 @@ auth_engine = _build_engine(AUTH_DATABASE_URL)
# #region SessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the main mappings database.
# @PRE engine is initialized.
# @PRE: engine is initialized.
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# #endregion SessionLocal
# #region TasksSessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the tasks execution database.
# @PRE tasks_engine is initialized.
# @PRE: tasks_engine is initialized.
TasksSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=tasks_engine)
# #endregion TasksSessionLocal
# #region AuthSessionLocal [C:1] [TYPE Class]
# @BRIEF A session factory for the authentication database.
# @PRE auth_engine is initialized.
# @PRE: auth_engine is initialized.
AuthSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=auth_engine)
# #endregion AuthSessionLocal
# #region _ensure_user_dashboard_preferences_columns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for user_dashboard_preferences table.
# @PRE bind_engine points to application database where profile table is stored.
# @POST Missing columns are added without data loss.
# @PRE: bind_engine points to application database where profile table is stored.
# @POST: Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [engine]
def _ensure_user_dashboard_preferences_columns(bind_engine):
with belief_scope("_ensure_user_dashboard_preferences_columns"):
@@ -257,8 +255,8 @@ def _ensure_llm_validation_results_columns(bind_engine):
# #region _ensure_git_server_configs_columns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for git_server_configs table.
# @PRE bind_engine points to application database.
# @POST Missing columns are added without data loss.
# @PRE: bind_engine points to application database.
# @POST: Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [engine]
def _ensure_git_server_configs_columns(bind_engine):
with belief_scope("_ensure_git_server_configs_columns"):
@@ -297,8 +295,8 @@ def _ensure_git_server_configs_columns(bind_engine):
# #region _ensure_auth_users_columns [C:3] [TYPE Function]
# @BRIEF Applies additive schema upgrades for auth users table.
# @PRE bind_engine points to authentication database.
# @POST Missing columns are added without data loss.
# @PRE: bind_engine points to authentication database.
# @POST: Missing columns are added without data loss.
# @RELATION DEPENDS_ON -> [auth_engine]
def _ensure_auth_users_columns(bind_engine):
with belief_scope("_ensure_auth_users_columns"):
@@ -359,8 +357,8 @@ def _ensure_auth_users_columns(bind_engine):
# #region ensure_connection_configs_table [C:3] [TYPE Function]
# @BRIEF Ensures the external connection registry table exists in the main database.
# @PRE bind_engine points to the application database.
# @POST connection_configs table exists without dropping existing data.
# @PRE: bind_engine points to the application database.
# @POST: connection_configs table exists without dropping existing data.
# @RELATION DEPENDS_ON -> [ConnectionConfig]
def ensure_connection_configs_table(bind_engine):
with belief_scope("ensure_connection_configs_table"):
@@ -379,8 +377,8 @@ def ensure_connection_configs_table(bind_engine):
# #region _ensure_filter_source_enum_values [C:3] [TYPE Function]
# @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type.
# @PRE bind_engine points to application database with imported_filters table.
# @POST New enum values are available without data loss.
# @PRE: bind_engine points to application database with imported_filters table.
# @POST: New enum values are available without data loss.
# @RELATION DEPENDS_ON -> [engine]
def _ensure_filter_source_enum_values(bind_engine):
with belief_scope("_ensure_filter_source_enum_values"):
@@ -450,9 +448,9 @@ def _ensure_filter_source_enum_values(bind_engine):
# #region _ensure_dataset_review_session_columns [C:4] [TYPE Function]
# @BRIEF Apply additive schema upgrades for dataset review persistence required by optimistic-lock and recovery metadata semantics.
# @PRE bind_engine points to the application database where dataset review tables are stored.
# @POST Missing additive columns across legacy dataset review tables are created without removing existing data.
# @SIDE_EFFECT Executes ALTER TABLE statements against dataset review tables in the application database.
# @PRE: bind_engine points to the application database where dataset review tables are stored.
# @POST: Missing additive columns across legacy dataset review tables are created without removing existing data.
# @SIDE_EFFECT: Executes ALTER TABLE statements against dataset review tables in the application database.
# @RELATION DEPENDS_ON -> [DatasetReviewSession]
# @RELATION DEPENDS_ON -> [ImportedFilter]
def _ensure_translation_jobs_columns(bind_engine):
@@ -486,62 +484,6 @@ def _ensure_translation_jobs_columns(bind_engine):
)
raise
if "target_database_id" not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(
text(
"ALTER TABLE translation_jobs "
"ADD COLUMN target_database_id VARCHAR"
)
)
logger.reflect(
"Added target_database_id column to translation_jobs",
)
except Exception as migration_error:
logger.explore(
"Failed to add target_database_id to translation_jobs",
extra={"error": str(migration_error)},
)
raise
# #region _ensure_translation_records_columns [TYPE Function]
# @BRIEF Add source_data JSON column to translation_records and translation_preview_records.
# @PRE bind_engine points to the application database.
# @POST source_data column exists on both tables.
# @SIDE_EFFECT Executes ALTER TABLE statements.
def _ensure_translation_records_columns(bind_engine):
with belief_scope("_ensure_translation_records_columns"):
migrations = [
("translation_records", "source_data",
"ALTER TABLE translation_records ADD COLUMN source_data JSON"),
("translation_preview_records", "source_data",
"ALTER TABLE translation_preview_records ADD COLUMN source_data JSON"),
]
for table_name, column_name, alter_sql in migrations:
inspector = inspect(bind_engine)
if table_name not in inspector.get_table_names():
continue
existing_columns = {
str(c.get("name") or "").strip()
for c in inspector.get_columns(table_name)
}
if column_name not in existing_columns:
try:
with bind_engine.begin() as connection:
connection.execute(text(alter_sql))
logger.reflect(
f"Added {column_name} column to {table_name}",
)
except Exception as migration_error:
logger.explore(
f"Failed to add {column_name} to {table_name}",
extra={"error": str(migration_error)},
)
raise
# #endregion _ensure_translation_records_columns
def _ensure_dataset_review_session_columns(bind_engine):
with belief_scope("_ensure_dataset_review_session_columns"):
@@ -621,9 +563,9 @@ def _ensure_dataset_review_session_columns(bind_engine):
# #region init_db [C:3] [TYPE Function]
# @BRIEF Initializes the database by creating all tables.
# @PRE engine, tasks_engine and auth_engine are initialized.
# @POST Database tables created in all databases.
# @SIDE_EFFECT Creates physical database files if they don't exist.
# @PRE: engine, tasks_engine and auth_engine are initialized.
# @POST: Database tables created in all databases.
# @SIDE_EFFECT: Creates physical database files if they don't exist.
# @RELATION CALLS -> [ensure_connection_configs_table]
# @RELATION CALLS -> [_ensure_filter_source_enum_values]
# @RELATION CALLS -> [_ensure_dataset_review_session_columns]
@@ -641,7 +583,6 @@ def init_db():
_ensure_filter_source_enum_values(engine)
_ensure_dataset_review_session_columns(engine)
_ensure_translation_jobs_columns(engine)
_ensure_translation_records_columns(engine)
# #endregion init_db
@@ -649,9 +590,8 @@ def init_db():
# #region get_db [C:3] [TYPE Function]
# @BRIEF Dependency for getting a database session.
# @PRE SessionLocal is initialized.
# @POST Session is closed after use.
# @RETURN Generator[Session, None, None]
# @PRE: SessionLocal is initialized.
# @POST: Session is closed after use.
# @RELATION DEPENDS_ON -> [SessionLocal]
def get_db():
with belief_scope("get_db"):
@@ -667,9 +607,8 @@ def get_db():
# #region get_tasks_db [C:3] [TYPE Function]
# @BRIEF Dependency for getting a tasks database session.
# @PRE TasksSessionLocal is initialized.
# @POST Session is closed after use.
# @RETURN Generator[Session, None, None]
# @PRE: TasksSessionLocal is initialized.
# @POST: Session is closed after use.
# @RELATION DEPENDS_ON -> [TasksSessionLocal]
def get_tasks_db():
with belief_scope("get_tasks_db"):
@@ -685,10 +624,9 @@ def get_tasks_db():
# #region get_auth_db [C:3] [TYPE Function]
# @BRIEF Dependency for getting an authentication database session.
# @PRE AuthSessionLocal is initialized.
# @POST Session is closed after use.
# @DATA_CONTRACT None -> Output[sqlalchemy.orm.Session]
# @RETURN Generator[Session, None, None]
# @PRE: AuthSessionLocal is initialized.
# @POST: Session is closed after use.
# @DATA_CONTRACT: None -> Output[sqlalchemy.orm.Session]
# @RELATION DEPENDS_ON -> [AuthSessionLocal]
def get_auth_db():
with belief_scope("get_auth_db"):

18
backend/src/core/encryption_key.py
View File

@@ -1,12 +1,12 @@
# #region EncryptionKeyModule [C:5] [TYPE Module] [SEMANTICS encryption, key, bootstrap, environment, startup]
# @BRIEF Resolve and persist the Fernet encryption key required by runtime services.
# @LAYER Infra
# @PRE Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT Input[env_file_path] -> Output[encryption_key]
# @INVARIANT Runtime key resolution never falls back to an ephemeral secret.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Runtime key resolution never falls back to an ephemeral secret.
# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key]
from __future__ import annotations
@@ -22,9 +22,9 @@ DEFAULT_ENV_FILE_PATH = Path(__file__).resolve().parents[2] / ".env"
# #region ensure_encryption_key [TYPE Function]
# @BRIEF Ensure backend runtime has a persistent valid Fernet key.
# @PRE env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment.
# @POST Returns a valid Fernet key and guarantees it is present in process environment.
# @SIDE_EFFECT May create or append backend/.env when key is missing.
# @PRE: env_file_path points to a writable backend .env file or ENCRYPTION_KEY exists in process environment.
# @POST: Returns a valid Fernet key and guarantees it is present in process environment.
# @SIDE_EFFECT: May create or append backend/.env when key is missing.
def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str:
with belief_scope("ensure_encryption_key", f"env_file_path={env_file_path}"):
existing_key = os.getenv("ENCRYPTION_KEY", "").strip()

418
backend/src/core/logger.py
View File

@@ -1,26 +1,16 @@
# #region LoggerModule [C:5] [TYPE Module] [SEMANTICS logging,websocket,streaming,handler,cot,json]
# @BRIEF Application logging system with CotJsonFormatter producing molecular CoT JSON output.
# @LAYER Core
# @RELATION USED_BY -> [All application modules]
# @RELATION DEPENDS_ON -> [CotLoggerModule]
# @RELATION DEPENDS_ON -> [WebSocketLogHandler]
# @PRE Python 3.7+ with cot_logger ContextVars available.
# @POST All log output is single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.
# @SIDE_EFFECT Configures global logger handlers and formatters; seeds global _enable_belief_state and _task_log_level; writes JSON to console/files.
# @RATIONALE Replaced BeliefFormatter text-based [Entry]/[Exit]/[Action] logging with CotJsonFormatter JSON output. Old format was not machine-parseable and mixed presentation with intent. CotJsonFormatter produces structured JSON consistent with cot_logger.py MarkerLogger protocol.
# @REJECTED Keeping both formatters side-by-side was rejected (two inconsistent output formats would confuse log consumers).
# @DATA_CONTRACT All log records conform to molecular CoT JSON schema: {ts, level, trace_id, src, marker, intent, span_id?, payload?, error?}
# @INVARIANT CotJsonFormatter.format() always returns valid single-line JSON string.
import json
# #region LoggerModule [TYPE Module] [SEMANTICS logging, websocket, streaming, handler]
# @BRIEF Configures the application's logging system, including a custom handler for buffering logs and streaming them over WebSockets.
# @LAYER: Core
# @RELATION Used by the main application and other modules to log events. The WebSocketLogHandler is used by the WebSocket endpoint in app.py.
import logging
import threading
import types
from datetime import datetime, timezone
from datetime import datetime
from typing import Dict, Any, List, Optional
from collections import deque
from contextlib import contextmanager
from logging.handlers import RotatingFileHandler
from .cot_logger import _trace_id, _span_id
from .ws_log_handler import WebSocketLogHandler, LogEntry # noqa: F401
from pydantic import BaseModel, Field
# Thread-local storage for belief state
_belief_state = threading.local()
@@ -31,122 +21,73 @@ _enable_belief_state = True
# Global task log level filter
_task_log_level = "INFO"
# #region CotJsonFormatter [C:3] [TYPE Class] [SEMANTICS logging,formatter,json,cot,protocol]
# @BRIEF JSON formatter implementing the molecular CoT logging protocol. Outputs single-line JSON with ts, level, trace_id, src, marker, intent, payload, error, span_id.
# @RELATION IMPLEMENTS -> [CotJsonFormat]
class CotJsonFormatter(logging.Formatter):
"""JSON formatter implementing the molecular CoT logging protocol.
Reads structured data from the LogRecord's extra attributes (marker, intent, payload,
error, src) set via the ``extra`` kwarg. Falls back to plain message wrapping when no
structured extra is present.
Output format (single-line JSON)::
{
"ts": "2026-05-12T10:30:00.123",
"level": "INFO",
"trace_id": "uuid-or-no-trace",
"src": "module.name",
"marker": "REASON|REFLECT|EXPLORE",
"intent": "human-readable intent",
"span_id": "optional-span",
"payload": { ... }, # optional
"error": "..." # optional
}
"""
# #region CotJsonFormatter.format [C:2] [TYPE Function] [SEMANTICS format,json,record]
# @BRIEF Format a LogRecord into a single-line CoT JSON string with molecular CoT protocol fields.
# #region BeliefFormatter [TYPE Class]
# @BRIEF Custom logging formatter that adds belief state prefixes to log messages.
class BeliefFormatter(logging.Formatter):
# [DEF:format:Function]
# @PURPOSE: Formats the log record, adding belief state context if available.
# @PRE: record is a logging.LogRecord.
# @POST: Returns formatted string.
# @PARAM: record (logging.LogRecord) - The log record to format.
# @RETURN: str - The formatted log message.
# @SEMANTICS: logging, formatter, context
def format(self, record):
# Try to extract structured data from extra kwargs on the record
marker = getattr(record, 'marker', None)
intent = getattr(record, 'intent', None)
payload = getattr(record, 'payload', None)
error = getattr(record, 'error', None)
src = getattr(record, 'src', None) or record.name
anchor_id = getattr(_belief_state, 'anchor_id', None)
if anchor_id:
msg = str(record.msg)
# Supported molecular topology markers
markers = ("[EXPLORE]", "[REASON]", "[REFLECT]", "[COHERENCE:", "[Action]", "[Entry]", "[Exit]")
# Avoid duplicating anchor or overriding explicit markers
if msg.startswith(f"[{anchor_id}]"):
pass
elif any(msg.startswith(m) for m in markers):
record.msg = f"[{anchor_id}]{msg}"
else:
# Default covalent bond
record.msg = f"[{anchor_id}][Action] {msg}"
return super().format(record)
# [/DEF:format:Function]
# #endregion BeliefFormatter
# For records that already come as JSON strings (e.g. from cot_logger.log()),
# detect and pass through the message directly as intent.
if not marker:
marker = "REASON" # default for plain messages
if not intent:
intent = record.getMessage()
# Re-using LogEntry from task_manager for consistency
# #region LogEntry [TYPE Class] [SEMANTICS log, entry, record, pydantic]
# @BRIEF A Pydantic model representing a single, structured log entry. This is a re-definition for consistency, as it's also defined in task_manager.py.
class LogEntry(BaseModel):
timestamp: datetime = Field(default_factory=datetime.utcnow)
level: str
message: str
context: Optional[Dict[str, Any]] = None
log_obj = {
"ts": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"level": record.levelname,
"trace_id": _trace_id.get() or "no-trace",
"src": src,
"marker": marker,
"intent": intent,
}
# #endregion LogEntry
span_id = _span_id.get()
if span_id:
log_obj["span_id"] = span_id
if payload:
log_obj["payload"] = payload
if error:
log_obj["error"] = error
return json.dumps(log_obj, ensure_ascii=False, default=str)
# #endregion CotJsonFormatter.format
# #endregion CotJsonFormatter
# #region belief_scope [C:4] [TYPE Function] [SEMANTICS logging,context,belief_state,cot,marker]
# @BRIEF Context manager for structured Belief State logging with CoT markers (REASON on entry, REFLECT on success, EXPLORE on failure).
# @RELATION CALLED_BY -> [All C3+ modules using belief_scope]
# @RELATION BINDS_TO -> [CotJsonFormatter]
# @PRE anchor_id must be provided.
# @POST Thread-local belief state is updated. Entry logged as REASON marker, success coherence as REFLECT marker, failure as EXPLORE marker.
# @SIDE_EFFECT Writes debug-level log records with structured extra data; mutates _belief_state.anchor_id.
# #region belief_scope [TYPE Function] [SEMANTICS logging, context, belief_state]
# @BRIEF Context manager for structured Belief State logging.
# @PRE: anchor_id must be provided.
# @POST: Thread-local belief state is updated and entry/exit logs are generated.
@contextmanager
def belief_scope(anchor_id: str, message: str = ""):
"""Context manager for structured Belief State logging with CoT markers.
# Log Entry if enabled (DEBUG level to reduce noise)
if _enable_belief_state:
entry_msg = f"[{anchor_id}][Entry]"
if message:
entry_msg += f" {message}"
logger.debug(entry_msg)
Emits structured markers via the ``extra`` dict so that CotJsonFormatter
produces proper JSON output.
Usage::
with belief_scope("MyFunction", "Processing request"):
logger.info("Doing work")
"""
# Set thread-local anchor_id
old_anchor = getattr(_belief_state, 'anchor_id', None)
_belief_state.anchor_id = anchor_id
# Log Entry (REASON marker) — only when belief state logging is enabled
if _enable_belief_state:
entry_msg = message or f"Enter {anchor_id}"
logger.debug(entry_msg, extra={
"marker": "REASON",
"intent": entry_msg,
"src": anchor_id,
})
try:
yield
# Coherence OK / scope exit — always logged for internal tracking
logger.debug("Coherence OK", extra={
"marker": "REFLECT",
"intent": "Coherence OK",
"src": anchor_id,
"payload": {"contract_id": anchor_id},
})
# Log Coherence OK and Exit (DEBUG level to reduce noise)
logger.debug("[COHERENCE:OK]")
if _enable_belief_state:
logger.debug("[Exit]")
except Exception as e:
# Coherence FAILED — always logged for error tracking
logger.debug(f"Coherence FAILED: {e}", extra={
"marker": "EXPLORE",
"intent": "Coherence FAILED",
"error": str(e),
"src": anchor_id,
"payload": {"contract_id": anchor_id},
})
# Log Coherence Failed (DEBUG level to reduce noise)
logger.debug(f"[COHERENCE:FAILED] {str(e)}")
raise
finally:
# Restore old anchor
@@ -154,14 +95,10 @@ def belief_scope(anchor_id: str, message: str = ""):
# #endregion belief_scope
# #region configure_logger [C:4] [TYPE Function] [SEMANTICS logging,configuration,initialization,cot]
# @BRIEF Configures the main logger and the CoT logger with CotJsonFormatter, file handlers, and global flags.
# @RELATION CALLS -> [CotJsonFormatter]
# @RELATION CALLS -> [CotLoggerModule]
# @PRE config is a valid LoggingConfig instance.
# @POST Logger levels, handlers, formatters, belief state flag, and task log level are updated. The 'cot' logger has CotJsonFormatter with propagation disabled.
# @SIDE_EFFECT Mutates global _enable_belief_state, _task_log_level; adds/removes handlers on both logger and cot_logger; creates file directories.
# #region configure_logger [TYPE Function] [SEMANTICS logging, configuration, initialization]
# @BRIEF Configures the logger with the provided logging settings.
# @PRE: config is a valid LoggingConfig instance.
# @POST: Logger level, handlers, belief state flag, and task log level are updated.
def configure_logger(config):
global _enable_belief_state, _task_log_level
_enable_belief_state = config.enable_belief_state
@@ -182,162 +119,183 @@ def configure_logger(config):
from pathlib import Path
log_file = Path(config.file_path)
log_file.parent.mkdir(parents=True, exist_ok=True)
file_handler = RotatingFileHandler(
config.file_path,
maxBytes=config.max_bytes,
backupCount=config.backup_count
)
file_handler.setFormatter(CotJsonFormatter())
file_handler.setFormatter(BeliefFormatter(
'[%(asctime)s][%(levelname)s][%(name)s] %(message)s'
))
logger.addHandler(file_handler)
# Update existing handlers' formatters to CotJsonFormatter
# Update existing handlers' formatters to BeliefFormatter
for handler in logger.handlers:
if not isinstance(handler, RotatingFileHandler):
handler.setFormatter(CotJsonFormatter())
# Also configure the 'cot' logger for consistent JSON output
from .cot_logger import cot_logger
cot_logger.setLevel(level)
# Remove all existing handlers from the cot logger
for h in list(cot_logger.handlers):
cot_logger.removeHandler(h)
h.close()
# Add a console handler with CotJsonFormatter (if file path set, add file handler too)
cot_console = logging.StreamHandler()
cot_console.setFormatter(CotJsonFormatter())
cot_logger.addHandler(cot_console)
if config.file_path:
from pathlib import Path
log_file = Path(config.file_path)
log_file.parent.mkdir(parents=True, exist_ok=True)
cot_file = RotatingFileHandler(
config.file_path,
maxBytes=config.max_bytes,
backupCount=config.backup_count
)
cot_file.setFormatter(CotJsonFormatter())
cot_logger.addHandler(cot_file)
# Disable propagation so cot messages don't double-emit via root loggers
cot_logger.propagate = False
handler.setFormatter(BeliefFormatter(
'[%(asctime)s][%(levelname)s][%(name)s] %(message)s'
))
# #endregion configure_logger
# #region get_task_log_level [C:1] [TYPE Function] [SEMANTICS logging,configuration,getter]
# #region get_task_log_level [TYPE Function] [SEMANTICS logging, configuration, getter]
# @BRIEF Returns the current task log level filter.
# @PRE: None.
# @POST: Returns the task log level string.
def get_task_log_level() -> str:
"""Returns the current task log level filter."""
return _task_log_level
# #endregion get_task_log_level
# #region should_log_task_level [C:2] [TYPE Function] [SEMANTICS logging,filter,level]
# @BRIEF Checks if a log level should be recorded based on the current task log level threshold.
# #region should_log_task_level [TYPE Function] [SEMANTICS logging, filter, level]
# @BRIEF Checks if a log level should be recorded based on task_log_level setting.
# @PRE: level is a valid log level string.
# @POST: Returns True if level meets or exceeds task_log_level threshold.
def should_log_task_level(level: str) -> bool:
"""Checks if a log level should be recorded based on task_log_level setting."""
level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3}
current_level = _task_log_level.upper()
check_level = level.upper()
current_order = level_order.get(current_level, 1) # Default to INFO
check_order = level_order.get(check_level, 1)
return check_order >= current_order
# #endregion should_log_task_level
# #region WebSocketLogHandler [TYPE Class] [SEMANTICS logging, handler, websocket, buffer]
# @BRIEF A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
class WebSocketLogHandler(logging.Handler):
"""
A logging handler that stores log records and can be extended to send them
over WebSockets.
"""
# [DEF:__init__:Function]
# @PURPOSE: Initializes the handler with a fixed-capacity buffer.
# @PRE: capacity is an integer.
# @POST: Instance initialized with empty deque.
# @PARAM: capacity (int) - Maximum number of logs to keep in memory.
# @SEMANTICS: logging, initialization, buffer
def __init__(self, capacity: int = 1000):
super().__init__()
self.log_buffer: deque[LogEntry] = deque(maxlen=capacity)
# In a real implementation, you'd have a way to manage active WebSocket connections
# e.g., self.active_connections: Set[WebSocket] = set()
# [/DEF:__init__:Function]
# #region Logger [C:2] [TYPE Global] [SEMANTICS logger,global,instance]
# @BRIEF The global application logger instance configured with CotJsonFormatter, console handler, and WebSocketLogHandler.
# [DEF:emit:Function]
# @PURPOSE: Captures a log record, formats it, and stores it in the buffer.
# @PRE: record is a logging.LogRecord.
# @POST: Log is added to the log_buffer.
# @PARAM: record (logging.LogRecord) - The log record to emit.
# @SEMANTICS: logging, handler, buffer
def emit(self, record: logging.LogRecord):
try:
log_entry = LogEntry(
level=record.levelname,
message=self.format(record),
context={
"name": record.name,
"pathname": record.pathname,
"lineno": record.lineno,
"funcName": record.funcName,
"process": record.process,
"thread": record.thread,
}
)
self.log_buffer.append(log_entry)
# Here you would typically send the log_entry to all active WebSocket connections
# for real-time streaming to the frontend.
# Example: for ws in self.active_connections: await ws.send_json(log_entry.dict())
except Exception:
self.handleError(record)
# [/DEF:emit:Function]
# [DEF:get_recent_logs:Function]
# @PURPOSE: Returns a list of recent log entries from the buffer.
# @PRE: None.
# @POST: Returns list of LogEntry objects.
# @RETURN: List[LogEntry] - List of buffered log entries.
# @SEMANTICS: logging, buffer, retrieval
def get_recent_logs(self) -> List[LogEntry]:
"""
Returns a list of recent log entries from the buffer.
"""
return list(self.log_buffer)
# [/DEF:get_recent_logs:Function]
# #endregion WebSocketLogHandler
# #region Logger [TYPE Global] [SEMANTICS logger, global, instance]
# @BRIEF The global logger instance for the application, configured with both a console handler and the custom WebSocket handler.
logger = logging.getLogger("superset_tools_app")
# #region believed [TYPE Function]
# @BRIEF A decorator that wraps a function in a belief scope.
# @PRE: anchor_id must be a string.
# @POST: Returns a decorator function.
def believed(anchor_id: str):
# [DEF:decorator:Function]
# @PURPOSE: Internal decorator for belief scope.
# @PRE: func must be a callable.
# @POST: Returns the wrapped function.
def decorator(func):
# [DEF:wrapper:Function]
# @PURPOSE: Internal wrapper that enters belief scope.
# @PRE: None.
# @POST: Executes the function within a belief scope.
def wrapper(*args, **kwargs):
with belief_scope(anchor_id):
return func(*args, **kwargs)
# [/DEF:wrapper:Function]
return wrapper
# [/DEF:decorator:Function]
return decorator
# #endregion believed
logger.setLevel(logging.INFO)
# Create CotJsonFormatter
_formatter = CotJsonFormatter()
# Create a formatter
formatter = BeliefFormatter(
'[%(asctime)s][%(levelname)s][%(name)s] %(message)s'
)
# Add console handler
console_handler = logging.StreamHandler()
console_handler.setFormatter(_formatter)
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
# Add WebSocket log handler
websocket_log_handler = WebSocketLogHandler()
websocket_log_handler.setFormatter(_formatter)
websocket_log_handler.setFormatter(formatter)
logger.addHandler(websocket_log_handler)
# Example usage:
# logger.info("Application started", extra={"context_key": "context_value"})
# logger.error("An error occurred", exc_info=True)
import types
# #region explore [C:2] [TYPE Function] [SEMANTICS log,explore,marker,warning]
# @BRIEF Logs an EXPLORE marker (WARNING level) with structured extra data for CotJsonFormatter.
# #region explore [TYPE Function] [SEMANTICS log, explore, molecule]
# @BRIEF Logs an EXPLORE message (Van der Waals force) for searching, alternatives, and hypotheses.
def explore(self, msg, *args, **kwargs):
"""Log an EXPLORE marker — searching, alternatives, violated assumptions.
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
extra = {'marker': 'EXPLORE', 'intent': msg}
extra.update(user_extra)
self.warning(msg, *args, extra=extra, **kwargs)
self.warning(f"[EXPLORE] {msg}", *args, **kwargs)
# #endregion explore
# #region reason [C:2] [TYPE Function] [SEMANTICS log,reason,marker,info]
# @BRIEF Logs a REASON marker (INFO level) with structured extra data for CotJsonFormatter.
# #region reason [TYPE Function] [SEMANTICS log, reason, molecule]
# @BRIEF Logs a REASON message (Covalent bond) for strict deduction and core logic.
def reason(self, msg, *args, **kwargs):
"""Log a REASON marker — strict deduction, core logic.
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
extra = {'marker': 'REASON', 'intent': msg}
extra.update(user_extra)
self.info(msg, *args, extra=extra, **kwargs)
self.info(f"[REASON] {msg}", *args, **kwargs)
# #endregion reason
# #region reflect [C:2] [TYPE Function] [SEMANTICS log,reflect,marker,debug]
# @BRIEF Logs a REFLECT marker (DEBUG level) with structured extra data for CotJsonFormatter.
# #region reflect [TYPE Function] [SEMANTICS log, reflect, molecule]
# @BRIEF Logs a REFLECT message (Hydrogen bond) for self-check and structural validation.
def reflect(self, msg, *args, **kwargs):
"""Log a REFLECT marker — self-check, structural validation.
Passes structured ``extra`` data so CotJsonFormatter produces proper JSON.
"""
user_extra = kwargs.pop('extra', {})
extra = {'marker': 'REFLECT', 'intent': msg}
extra.update(user_extra)
self.debug(msg, *args, extra=extra, **kwargs)
self.debug(f"[REFLECT] {msg}", *args, **kwargs)
# #endregion reflect
logger.explore = types.MethodType(explore, logger)
logger.reason = types.MethodType(reason, logger)
logger.reflect = types.MethodType(reflect, logger)
# #endregion Logger
# #region believed [C:2] [TYPE Function] [SEMANTICS decorator,belief_scope,wrapper]
# @BRIEF Decorator that wraps a function in a belief_scope context manager.
def believed(anchor_id: str):
# #region believed.decorator [C:1] [TYPE Function] [SEMANTICS decorator,inner]
def decorator(func):
# #region believed.decorator.wrapper [C:1] [TYPE Function] [SEMANTICS wrapper,belief_scope]
def wrapper(*args, **kwargs):
with belief_scope(anchor_id):
return func(*args, **kwargs)
# #endregion believed.decorator.wrapper
return wrapper
# #endregion believed.decorator
return decorator
# #endregion believed
# #endregion LoggerModule
# #endregion LoggerModule

336
backend/src/core/logger/__tests__/test_logger.py
View File

@@ -1,7 +1,8 @@
# #region test_logger [C:3] [TYPE Module]
# @BRIEF Unit tests for logger module — CoT JSON format markers.
# @LAYER Infra
# @RELATION VERIFIES -> [src.core.logger]
# [DEF:test_logger:Module]
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for logger module
# @LAYER: Infra
# @RELATION: VERIFIES -> src.core.logger
import sys
from pathlib import Path
@@ -11,14 +12,12 @@ sys.path.append(str(Path(__file__).parent.parent.parent.parent / "src"))
import pytest
import logging
import json
from src.core.logger import (
belief_scope,
logger,
configure_logger,
get_task_log_level,
should_log_task_level,
CotJsonFormatter,
should_log_task_level
)
from src.core.config_models import LoggingConfig
@@ -44,13 +43,13 @@ def reset_logger_state():
configure_logger(config)
# #region test_belief_scope_logs_reason_reflect_at_debug [TYPE Function]
# @BRIEF Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level.
# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST Logs are verified to contain REASON (entry) and REFLECT (coherence) markers.
# @RELATION BINDS_TO -> [test_logger]
def test_belief_scope_logs_reason_reflect_at_debug(caplog):
"""Test that belief_scope generates REASON and REFLECT CoT markers at DEBUG level."""
# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.
def test_belief_scope_logs_entry_action_exit_at_debug(caplog):
"""Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level."""
# Configure logger to DEBUG level
config = LoggingConfig(
level="DEBUG",
@@ -58,43 +57,32 @@ def test_belief_scope_logs_reason_reflect_at_debug(caplog):
enable_belief_state=True
)
configure_logger(config)
caplog.set_level("DEBUG")
with belief_scope("TestFunction"):
logger.info("Doing something important")
# Check that the records contain the expected markers
reason_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'TestFunction'
]
reflect_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'TestFunction'
]
info_records = [
r for r in caplog.records
if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()
]
assert len(reason_records) >= 1, "REASON marker not found for TestFunction"
assert len(reflect_records) >= 1, "REFLECT marker not found for TestFunction"
assert len(info_records) >= 1, "INFO log 'Doing something important' not found"
# Check that the logs contain the expected patterns
log_messages = [record.message for record in caplog.records]
assert any("[TestFunction][Entry]" in msg for msg in log_messages), "Entry log not found"
assert any("[TestFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found"
assert any("[TestFunction][Exit]" in msg for msg in log_messages), "Exit log not found"
# Reset to INFO
config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
configure_logger(config)
# #endregion test_belief_scope_logs_reason_reflect_at_debug
# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]
# #region test_belief_scope_error_handling [TYPE Function]
# @BRIEF Test that belief_scope logs EXPLORE marker on exception.
# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST Logs are verified to contain EXPLORE marker with error info.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_belief_scope_error_handling:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception.
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST: Logs are verified to contain Coherence:Failed tag.
def test_belief_scope_error_handling(caplog):
"""Test that belief_scope logs EXPLORE marker on exception."""
"""Test that belief_scope logs Coherence:Failed on exception."""
# Configure logger to DEBUG level
config = LoggingConfig(
level="DEBUG",
@@ -102,37 +90,32 @@ def test_belief_scope_error_handling(caplog):
enable_belief_state=True
)
configure_logger(config)
caplog.set_level("DEBUG")
with pytest.raises(ValueError):
with belief_scope("FailingFunction"):
raise ValueError("Something went wrong")
# Check that an EXPLORE marker was emitted
explore_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'EXPLORE'
and getattr(r, 'src', None) == 'FailingFunction'
]
assert len(explore_records) >= 1, "EXPLORE marker not found for FailingFunction"
assert 'Something went wrong' in explore_records[0].getMessage() or \
'Something went wrong' in str(getattr(explore_records[0], 'error', ''))
log_messages = [record.message for record in caplog.records]
assert any("[FailingFunction][Entry]" in msg for msg in log_messages), "Entry log not found"
assert any("[FailingFunction][COHERENCE:FAILED]" in msg for msg in log_messages), "Failed coherence log not found"
# Exit should not be logged on failure
# Reset to INFO
config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
configure_logger(config)
# #endregion test_belief_scope_error_handling
# [/DEF:test_belief_scope_error_handling:Function]
# #region test_belief_scope_success_coherence [TYPE Function]
# @BRIEF Test that belief_scope logs REFLECT marker on success.
# @PRE belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST Logs are verified to contain REFLECT marker.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_belief_scope_success_coherence:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that belief_scope logs Coherence:OK on success.
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
# @POST: Logs are verified to contain Coherence:OK tag.
def test_belief_scope_success_coherence(caplog):
"""Test that belief_scope logs REFLECT marker on success."""
"""Test that belief_scope logs Coherence:OK on success."""
# Configure logger to DEBUG level
config = LoggingConfig(
level="DEBUG",
@@ -140,77 +123,60 @@ def test_belief_scope_success_coherence(caplog):
enable_belief_state=True
)
configure_logger(config)
caplog.set_level("DEBUG")
with belief_scope("SuccessFunction"):
pass
reflect_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REFLECT'
and getattr(r, 'src', None) == 'SuccessFunction'
]
log_messages = [record.message for record in caplog.records]
assert len(reflect_records) >= 1, "REFLECT marker not found for SuccessFunction"
assert 'Coherence OK' in reflect_records[0].getMessage() or \
getattr(reflect_records[0], 'intent', '') == 'Coherence OK'
assert any("[SuccessFunction][COHERENCE:OK]" in msg for msg in log_messages), "Success coherence log not found"
# [/DEF:test_belief_scope_success_coherence:Function]
# #endregion test_belief_scope_success_coherence
# #region test_belief_scope_reason_not_visible_at_info [TYPE Function]
# @BRIEF Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level.
# @PRE belief_scope is available. caplog fixture is used.
# @POST REASON/REFLECT markers are not captured at INFO level.
# @RELATION BINDS_TO -> [test_logger]
def test_belief_scope_reason_not_visible_at_info(caplog):
"""Test that belief_scope REASON/REFLECT markers are NOT visible at INFO level."""
# [DEF:test_belief_scope_not_visible_at_info:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.
# @PRE: belief_scope is available. caplog fixture is used.
# @POST: Entry/Exit/Coherence logs are not captured at INFO level.
def test_belief_scope_not_visible_at_info(caplog):
"""Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level."""
caplog.set_level("INFO")
with belief_scope("InfoLevelFunction"):
logger.info("Doing something important")
# The REASON and REFLECT markers are debug-level, so they should NOT appear at INFO
reason_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'InfoLevelFunction'
]
reflect_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'InfoLevelFunction'
]
log_messages = [record.message for record in caplog.records]
assert len(reason_records) == 0, "REASON marker should not be visible at INFO"
assert len(reflect_records) == 0, "REFLECT marker should not be visible at INFO"
# But the INFO-level message should be visible
info_records = [
r for r in caplog.records
if r.levelname == 'INFO' and 'Doing something important' in r.getMessage()
]
assert len(info_records) >= 1, "INFO log 'Doing something important' should be visible"
# #endregion test_belief_scope_reason_not_visible_at_info
# Action log should be visible
assert any("[InfoLevelFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found"
# Entry/Exit/Coherence should NOT be visible at INFO level
assert not any("[InfoLevelFunction][Entry]" in msg for msg in log_messages), "Entry log should not be visible at INFO"
assert not any("[InfoLevelFunction][Exit]" in msg for msg in log_messages), "Exit log should not be visible at INFO"
assert not any("[InfoLevelFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence log should not be visible at INFO"
# [/DEF:test_belief_scope_not_visible_at_info:Function]
# #region test_task_log_level_default [TYPE Function]
# @BRIEF Test that default task log level is INFO.
# @PRE None.
# @POST Default level is INFO.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_task_log_level_default:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that default task log level is INFO.
# @PRE: None.
# @POST: Default level is INFO.
def test_task_log_level_default():
"""Test that default task log level is INFO (after reset fixture)."""
level = get_task_log_level()
assert level == "INFO"
# #endregion test_task_log_level_default
# [/DEF:test_task_log_level_default:Function]
# #region test_should_log_task_level [TYPE Function]
# @BRIEF Test that should_log_task_level correctly filters log levels.
# @PRE None.
# @POST Filtering works correctly for all level combinations.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_should_log_task_level:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that should_log_task_level correctly filters log levels.
# @PRE: None.
# @POST: Filtering works correctly for all level combinations.
def test_should_log_task_level():
"""Test that should_log_task_level correctly filters log levels."""
# Default level is INFO
@@ -218,14 +184,14 @@ def test_should_log_task_level():
assert should_log_task_level("WARNING") is True, "WARNING should be logged at INFO threshold"
assert should_log_task_level("INFO") is True, "INFO should be logged at INFO threshold"
assert should_log_task_level("DEBUG") is False, "DEBUG should NOT be logged at INFO threshold"
# #endregion test_should_log_task_level
# [/DEF:test_should_log_task_level:Function]
# #region test_configure_logger_task_log_level [TYPE Function]
# @BRIEF Test that configure_logger updates task_log_level.
# @PRE LoggingConfig is available.
# @POST task_log_level is updated correctly.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_configure_logger_task_log_level:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that configure_logger updates task_log_level.
# @PRE: LoggingConfig is available.
# @POST: task_log_level is updated correctly.
def test_configure_logger_task_log_level():
"""Test that configure_logger updates task_log_level."""
config = LoggingConfig(
@@ -234,19 +200,19 @@ def test_configure_logger_task_log_level():
enable_belief_state=True
)
configure_logger(config)
assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG"
assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold"
# #endregion test_configure_logger_task_log_level
# [/DEF:test_configure_logger_task_log_level:Function]
# #region test_enable_belief_state_flag [TYPE Function]
# @BRIEF Test that enable_belief_state flag controls belief_scope entry logging.
# @PRE LoggingConfig is available. caplog fixture is used.
# @POST REASON entry marker is suppressed when disabled; REFLECT coherence still logged.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_enable_belief_state_flag:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging.
# @PRE: LoggingConfig is available. caplog fixture is used.
# @POST: belief_scope logs are controlled by the flag.
def test_enable_belief_state_flag(caplog):
"""Test that enable_belief_state flag controls belief_scope REASON entry logging."""
"""Test that enable_belief_state flag controls belief_scope logging."""
# Disable belief state
config = LoggingConfig(
level="DEBUG",
@@ -254,31 +220,25 @@ def test_enable_belief_state_flag(caplog):
enable_belief_state=False
)
configure_logger(config)
caplog.set_level("DEBUG")
with belief_scope("DisabledFunction"):
logger.info("Doing something")
# REASON entry marker should NOT be logged when disabled
reason_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REASON' and getattr(r, 'src', None) == 'DisabledFunction'
]
assert len(reason_records) == 0, "REASON entry should not be logged when disabled"
# REFLECT coherence marker should still be logged (internal tracking)
reflect_records = [
r for r in caplog.records
if getattr(r, 'marker', None) == 'REFLECT' and getattr(r, 'src', None) == 'DisabledFunction'
]
assert len(reflect_records) >= 1, "REFLECT coherence should still be logged"
# #endregion test_enable_belief_state_flag
log_messages = [record.message for record in caplog.records]
# Entry and Exit should NOT be logged when disabled
assert not any("[DisabledFunction][Entry]" in msg for msg in log_messages), "Entry should not be logged when disabled"
assert not any("[DisabledFunction][Exit]" in msg for msg in log_messages), "Exit should not be logged when disabled"
# Coherence:OK should still be logged (internal tracking)
assert any("[DisabledFunction][COHERENCE:OK]" in msg for msg in log_messages), "Coherence should still be logged"
# [/DEF:test_enable_belief_state_flag:Function]
# #region test_belief_scope_missing_anchor [TYPE Function]
# @BRIEF Test @PRE condition: anchor_id must be provided
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_belief_scope_missing_anchor:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test @PRE condition: anchor_id must be provided
def test_belief_scope_missing_anchor():
"""Test that belief_scope enforces anchor_id to be provided."""
import pytest
@@ -287,17 +247,17 @@ def test_belief_scope_missing_anchor():
# Missing required positional argument 'anchor_id'
with belief_scope():
pass
# #endregion test_belief_scope_missing_anchor
# [/DEF:test_belief_scope_missing_anchor:Function]
# #region test_configure_logger_post_conditions [TYPE Function]
# @BRIEF Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.
# @RELATION BINDS_TO -> [test_logger]
# [DEF:test_configure_logger_post_conditions:Function]
# @RELATION: BINDS_TO -> test_logger
# @PURPOSE: Test @POST condition: Logger level, handlers, belief state flag, and task log level are updated.
def test_configure_logger_post_conditions(tmp_path):
"""Test that configure_logger satisfies all @POST conditions."""
import logging
from logging.handlers import RotatingFileHandler
from src.core.config_models import LoggingConfig
from src.core.logger import configure_logger, logger, CotJsonFormatter, get_task_log_level
from src.core.logger import configure_logger, logger, BeliefFormatter, get_task_log_level
import src.core.logger as logger_module
log_file = tmp_path / "test.log"
@@ -307,93 +267,25 @@ def test_configure_logger_post_conditions(tmp_path):
enable_belief_state=False,
file_path=str(log_file)
)
configure_logger(config)
# 1. Logger level is updated
assert logger.level == logging.WARNING
# 2. Handlers are updated (file handler removed old ones, added new one)
file_handlers = [h for h in logger.handlers if isinstance(h, RotatingFileHandler)]
assert len(file_handlers) == 1
import pathlib
assert pathlib.Path(file_handlers[0].baseFilename) == log_file.resolve()
# 3. Formatter is set to CotJsonFormatter
# 3. Formatter is set to BeliefFormatter
for handler in logger.handlers:
assert isinstance(handler.formatter, CotJsonFormatter)
assert isinstance(handler.formatter, BeliefFormatter)
# 4. Global states
assert getattr(logger_module, '_enable_belief_state') is False
assert get_task_log_level() == "DEBUG"
# #endregion test_configure_logger_post_conditions
# [/DEF:test_configure_logger_post_conditions:Function]
# #region test_cot_json_formatter_output [TYPE Function]
# @BRIEF Test that CotJsonFormatter produces valid JSON with expected fields.
# @RELATION BINDS_TO -> [test_logger]
def test_cot_json_formatter_output():
"""Test that CotJsonFormatter produces valid JSON with expected fields."""
import json
from datetime import datetime, timezone
formatter = CotJsonFormatter()
# Create a LogRecord with structured extra data
record = logging.LogRecord(
name="test.module",
level=logging.INFO,
pathname="/fake/path.py",
lineno=42,
msg="Test action",
args=(),
exc_info=None,
)
record.marker = "REASON"
record.intent = "Test action"
record.src = "TestModule"
record.payload = {"key": "value"}
output = formatter.format(record)
parsed = json.loads(output)
assert parsed["level"] == "INFO"
assert parsed["marker"] == "REASON"
assert parsed["intent"] == "Test action"
assert parsed["src"] == "TestModule"
assert parsed["payload"] == {"key": "value"}
assert "ts" in parsed
assert "trace_id" in parsed
# #endregion test_cot_json_formatter_output
# #region test_cot_json_formatter_plain_message [TYPE Function]
# @BRIEF Test that CotJsonFormatter wraps plain messages (no extra) with default marker.
# @RELATION BINDS_TO -> [test_logger]
def test_cot_json_formatter_plain_message():
"""Test that CotJsonFormatter wraps plain messages with default marker."""
import json
formatter = CotJsonFormatter()
# Create a LogRecord WITHOUT extra data (plain message)
record = logging.LogRecord(
name="test.module",
level=logging.INFO,
pathname="/fake/path.py",
lineno=42,
msg="Plain info message",
args=(),
exc_info=None,
)
output = formatter.format(record)
parsed = json.loads(output)
assert parsed["level"] == "INFO"
assert parsed["marker"] == "REASON" # default for plain messages
assert parsed["intent"] == "Plain info message"
assert parsed["src"] == "test.module"
assert "ts" in parsed
assert "trace_id" in parsed
# #endregion test_cot_json_formatter_plain_message
# #endregion test_logger
# [/DEF:test_logger:Module]

97
backend/src/core/mapping_service.py
View File

@@ -1,40 +1,35 @@
# #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS mapping, ids, synchronization, environments, cross-filters]
#
# @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
# @LAYER Core
# @PRE Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @LAYER: Core
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [LoggerModule]
#
# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
#
# @INVARIANT: sync_environment must handle remote API failures gracefully.
# [SECTION: IMPORTS]
from typing import Dict, List, Optional
from datetime import datetime, timezone
from sqlalchemy.orm import Session
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from src.models.mapping import ResourceMapping, ResourceType
from src.core.cot_logger import MarkerLogger
from src.core.logger import logger, belief_scope
log = MarkerLogger("IdMapping")
# [/SECTION]
# #region IdMappingService [C:5] [TYPE Class]
# @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs.
# @PRE db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @SIDE_EFFECT Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService]
# @INVARIANT self.db remains the authoritative session for all mapping operations.
# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [LoggerModule]
# @INVARIANT: self.db remains the authoritative session for all mapping operations.
# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]
#
# @TEST_CONTRACT: IdMappingServiceModel ->
# {
@@ -51,17 +46,17 @@ log = MarkerLogger("IdMapping")
# @TEST_EDGE: get_batch_empty_list -> returns empty dict
# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]
class IdMappingService:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the mapping service.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapping service.
def __init__(self, db_session: Session):
self.db = db_session
self.scheduler = BackgroundScheduler()
self._sync_job = None
# #endregion __init__
# [/DEF:__init__:Function]
# #region start_scheduler [TYPE Function]
# @BRIEF Starts the background scheduler with a given cron string.
# [DEF:start_scheduler:Function]
# @PURPOSE: Starts the background scheduler with a given cron string.
# @PARAM: cron_string (str) - Cron expression for the sync interval.
# @PARAM: environments (List[str]) - List of environment IDs to sync.
# @PARAM: superset_client_factory - Function to get a client for an environment.
@@ -71,7 +66,9 @@ class IdMappingService:
with belief_scope("IdMappingService.start_scheduler"):
if self._sync_job:
self.scheduler.remove_job(self._sync_job.id)
log.reflect("Removed existing sync job.")
logger.info(
"[IdMappingService.start_scheduler][Reflect] Removed existing sync job."
)
def sync_all():
for env_id in environments:
@@ -88,14 +85,18 @@ class IdMappingService:
if not self.scheduler.running:
self.scheduler.start()
log.reason(f"Started background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}"
)
else:
log.reason(f"Updated background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}"
)
# #endregion start_scheduler
# [/DEF:start_scheduler:Function]
# #region sync_environment [TYPE Function]
# @BRIEF Fully synchronizes mapping for a specific environment.
# [DEF:sync_environment:Function]
# @PURPOSE: Fully synchronizes mapping for a specific environment.
# @PARAM: environment_id (str) - Target environment ID.
# @PARAM: superset_client - Instance capable of hitting the Superset API.
# @PRE: environment_id exists in the database.
@@ -108,7 +109,9 @@ class IdMappingService:
If incremental=True, only fetches items changed since the max last_synced_at date.
"""
with belief_scope("IdMappingService.sync_environment"):
log.reason(f"Starting sync for environment {environment_id} (incremental={incremental})")
logger.info(
f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})"
)
# Implementation Note: In a real scenario, superset_client needs to be an instance
# capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/
@@ -128,7 +131,9 @@ class IdMappingService:
total_deleted = 0
try:
for res_enum, endpoint, name_field in types_to_poll:
log.reason(f"Polling {endpoint} endpoint")
logger.debug(
f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint"
)
# Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... )
# This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] }
@@ -153,8 +158,8 @@ class IdMappingService:
from datetime import timedelta
since_dttm = max_date - timedelta(minutes=5)
log.reason(
f"Incremental sync since {since_dttm}"
logger.debug(
f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}"
)
resources = superset_client.get_all_resources(
@@ -218,24 +223,32 @@ class IdMappingService:
deleted = stale_query.delete(synchronize_session="fetch")
if deleted:
total_deleted += deleted
log.reason(f"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}")
logger.info(
f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}"
)
except Exception as loop_e:
log.explore(f"Error polling {endpoint}", error=str(loop_e))
logger.error(
f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}"
)
# Continue to next resource type instead of blowing up the whole sync
self.db.commit()
log.reflect(f"Successfully synced {total_synced} items and deleted {total_deleted} stale items.")
logger.info(
f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items."
)
except Exception as e:
self.db.rollback()
log.explore("Critical sync failure", error=str(e))
logger.error(
f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}"
)
raise
# #endregion sync_environment
# [/DEF:sync_environment:Function]
# #region get_remote_id [TYPE Function]
# @BRIEF Retrieves the remote integer ID for a given universal UUID.
# [DEF:get_remote_id:Function]
# @PURPOSE: Retrieves the remote integer ID for a given universal UUID.
# @PARAM: environment_id (str)
# @PARAM: resource_type (ResourceType)
# @PARAM: uuid (str)
@@ -258,10 +271,10 @@ class IdMappingService:
return None
return None
# #endregion get_remote_id
# [/DEF:get_remote_id:Function]
# #region get_remote_ids_batch [TYPE Function]
# @BRIEF Retrieves remote integer IDs for a list of universal UUIDs efficiently.
# [DEF:get_remote_ids_batch:Function]
# @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.
# @PARAM: environment_id (str)
# @PARAM: resource_type (ResourceType)
# @PARAM: uuids (List[str])
@@ -291,7 +304,7 @@ class IdMappingService:
return result
# #endregion get_remote_ids_batch
# [/DEF:get_remote_ids_batch:Function]
# #endregion IdMappingService

5
backend/src/core/middleware/trace.py
View File

@@ -2,8 +2,8 @@
# @BRIEF FastAPI/Starlette middleware that seeds a trace_id for every incoming HTTP request.
# Optionally extracts X-Trace-ID header for cross-service trace propagation.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] -> [CotLoggerModule]
# @RELATION: [CALLED_BY] -> [AppModule]
# @RELATION DEPENDS_ON -> [CotLoggerModule]
# @RELATION CALLED_BY -> [AppModule]
# @PRE: FastAPI app instance with Starlette middleware support.
# @POST: Every request gets a trace_id via seed_trace_id(). Existing X-Trace-ID header is
# preserved and used as the trace_id when present.
@@ -68,3 +68,4 @@ class TraceContextMiddleware(BaseHTTPMiddleware):
# #endregion TraceContextMiddleware
# #endregion TraceContextMiddlewareModule

40
backend/src/core/migration/archive_parser.py
View File

@@ -1,8 +1,8 @@
# #region MigrationArchiveParserModule [C:3] [TYPE Module] [SEMANTICS migration, zip, parser, yaml, metadata]
# @BRIEF Parse Superset export ZIP archives into normalized object catalogs for diffing.
# @LAYER Core
# @INVARIANT Parsing is read-only and never mutates archive files.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Parsing is read-only and never mutates archive files.
import json
import tempfile
@@ -21,12 +21,12 @@ from ..logger import logger, belief_scope
# @RELATION CONTAINS -> [_collect_yaml_objects]
# @RELATION CONTAINS -> [_normalize_object_payload]
class MigrationArchiveParser:
# #region extract_objects_from_zip [TYPE Function]
# @BRIEF Extract object catalogs from Superset archive.
# @PRE zip_path points to a valid readable ZIP.
# @POST Returns object lists grouped by resource type.
# @RETURN Dict[str, List[Dict[str, Any]]]
# @RELATION DEPENDS_ON -> [_collect_yaml_objects]
# [DEF:extract_objects_from_zip:Function]
# @PURPOSE: Extract object catalogs from Superset archive.
# @RELATION: DEPENDS_ON -> [_collect_yaml_objects]
# @PRE: zip_path points to a valid readable ZIP.
# @POST: Returns object lists grouped by resource type.
# @RETURN: Dict[str, List[Dict[str, Any]]]
def extract_objects_from_zip(
self, zip_path: str
) -> Dict[str, List[Dict[str, Any]]]:
@@ -49,13 +49,13 @@ class MigrationArchiveParser:
return result
# #endregion extract_objects_from_zip
# [/DEF:extract_objects_from_zip:Function]
# #region _collect_yaml_objects [TYPE Function]
# @BRIEF Read and normalize YAML manifests for one object type.
# @PRE object_type is one of dashboards/charts/datasets.
# @POST Returns only valid normalized objects.
# @RELATION DEPENDS_ON -> [_normalize_object_payload]
# [DEF:_collect_yaml_objects:Function]
# @PURPOSE: Read and normalize YAML manifests for one object type.
# @RELATION: DEPENDS_ON -> [_normalize_object_payload]
# @PRE: object_type is one of dashboards/charts/datasets.
# @POST: Returns only valid normalized objects.
def _collect_yaml_objects(
self, root_dir: Path, object_type: str
) -> List[Dict[str, Any]]:
@@ -79,12 +79,12 @@ class MigrationArchiveParser:
)
return objects
# #endregion _collect_yaml_objects
# [/DEF:_collect_yaml_objects:Function]
# #region _normalize_object_payload [TYPE Function]
# @BRIEF Convert raw YAML payload to stable diff signature shape.
# @PRE payload is parsed YAML mapping.
# @POST Returns normalized descriptor with `uuid`, `title`, and `signature`.
# [DEF:_normalize_object_payload:Function]
# @PURPOSE: Convert raw YAML payload to stable diff signature shape.
# @PRE: payload is parsed YAML mapping.
# @POST: Returns normalized descriptor with `uuid`, `title`, and `signature`.
def _normalize_object_payload(
self, payload: Dict[str, Any], object_type: str
) -> Optional[Dict[str, Any]]:
@@ -149,7 +149,7 @@ class MigrationArchiveParser:
return None
# #endregion _normalize_object_payload
# [/DEF:_normalize_object_payload:Function]
# #endregion MigrationArchiveParser

74
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -1,11 +1,11 @@
# #region MigrationDryRunOrchestratorModule [C:3] [TYPE Module] [SEMANTICS migration, dry_run, diff, risk, superset]
# @BRIEF Compute pre-flight migration diff and risk scoring without apply.
# @LAYER Core
# @INVARIANT Dry run is informative only and must not mutate target environment.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [MigrationEngine]
# @RELATION DEPENDS_ON -> [MigrationArchiveParser]
# @RELATION DEPENDS_ON -> [RiskAssessorModule]
# @INVARIANT: Dry run is informative only and must not mutate target environment.
from datetime import datetime, timezone
import json
@@ -34,25 +34,25 @@ from ..utils.fileio import create_temp_file
# @RELATION CONTAINS -> [_build_target_signatures]
# @RELATION CONTAINS -> [_build_risks]
class MigrationDryRunService:
# #region __init__ [TYPE Function]
# @BRIEF Wire parser dependency for archive object extraction.
# @PRE parser can be omitted to use default implementation.
# @POST Service is ready to calculate dry-run payload.
# [DEF:__init__:Function]
# @PURPOSE: Wire parser dependency for archive object extraction.
# @PRE: parser can be omitted to use default implementation.
# @POST: Service is ready to calculate dry-run payload.
def __init__(self, parser: MigrationArchiveParser | None = None):
self.parser = parser or MigrationArchiveParser()
# #endregion __init__
# [/DEF:__init__:Function]
# #region run [TYPE Function]
# @BRIEF Execute full dry-run computation for selected dashboards.
# @PRE source/target clients are authenticated and selection validated by caller.
# @POST Returns JSON-serializable pre-flight payload with summary, diff and risk.
# @SIDE_EFFECT Reads source export archives and target metadata via network.
# @RELATION DEPENDS_ON -> [_load_db_mapping]
# @RELATION DEPENDS_ON -> [_accumulate_objects]
# @RELATION DEPENDS_ON -> [_build_target_signatures]
# @RELATION DEPENDS_ON -> [_build_object_diff]
# @RELATION DEPENDS_ON -> [_build_risks]
# [DEF:run:Function]
# @PURPOSE: Execute full dry-run computation for selected dashboards.
# @RELATION: DEPENDS_ON -> [_load_db_mapping]
# @RELATION: DEPENDS_ON -> [_accumulate_objects]
# @RELATION: DEPENDS_ON -> [_build_target_signatures]
# @RELATION: DEPENDS_ON -> [_build_object_diff]
# @RELATION: DEPENDS_ON -> [_build_risks]
# @PRE: source/target clients are authenticated and selection validated by caller.
# @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk.
# @SIDE_EFFECT: Reads source export archives and target metadata via network.
def run(
self,
selection: DashboardSelection,
@@ -154,10 +154,10 @@ class MigrationDryRunService:
"risk": score_risks(risk),
}
# #endregion run
# [/DEF:run:Function]
# #region _load_db_mapping [TYPE Function]
# @BRIEF Resolve UUID mapping for optional DB config replacement.
# [DEF:_load_db_mapping:Function]
# @PURPOSE: Resolve UUID mapping for optional DB config replacement.
def _load_db_mapping(
self, db: Session, selection: DashboardSelection
) -> Dict[str, str]:
@@ -171,10 +171,10 @@ class MigrationDryRunService:
)
return {row.source_db_uuid: row.target_db_uuid for row in rows}
# #endregion _load_db_mapping
# [/DEF:_load_db_mapping:Function]
# #region _accumulate_objects [TYPE Function]
# @BRIEF Merge extracted resources by UUID to avoid duplicates.
# [DEF:_accumulate_objects:Function]
# @PURPOSE: Merge extracted resources by UUID to avoid duplicates.
def _accumulate_objects(
self,
target: Dict[str, Dict[str, Dict[str, Any]]],
@@ -186,10 +186,10 @@ class MigrationDryRunService:
if uuid:
target[object_type][str(uuid)] = item
# #endregion _accumulate_objects
# [/DEF:_accumulate_objects:Function]
# #region _index_by_uuid [TYPE Function]
# @BRIEF Build UUID-index map for normalized resources.
# [DEF:_index_by_uuid:Function]
# @PURPOSE: Build UUID-index map for normalized resources.
def _index_by_uuid(
self, objects: List[Dict[str, Any]]
) -> Dict[str, Dict[str, Any]]:
@@ -200,11 +200,11 @@ class MigrationDryRunService:
indexed[str(uuid)] = obj
return indexed
# #endregion _index_by_uuid
# [/DEF:_index_by_uuid:Function]
# #region _build_object_diff [TYPE Function]
# @BRIEF Compute create/update/delete buckets by UUID+signature.
# @RELATION DEPENDS_ON -> [_index_by_uuid]
# [DEF:_build_object_diff:Function]
# @PURPOSE: Compute create/update/delete buckets by UUID+signature.
# @RELATION: DEPENDS_ON -> [_index_by_uuid]
def _build_object_diff(
self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]]
) -> Dict[str, List[Dict[str, Any]]]:
@@ -228,10 +228,10 @@ class MigrationDryRunService:
)
return {"create": created, "update": updated, "delete": deleted}
# #endregion _build_object_diff
# [/DEF:_build_object_diff:Function]
# #region _build_target_signatures [TYPE Function]
# @BRIEF Pull target metadata and normalize it into comparable signatures.
# [DEF:_build_target_signatures:Function]
# @PURPOSE: Pull target metadata and normalize it into comparable signatures.
def _build_target_signatures(
self, client: SupersetClient
) -> Dict[str, List[Dict[str, Any]]]:
@@ -341,10 +341,10 @@ class MigrationDryRunService:
],
}
# #endregion _build_target_signatures
# [/DEF:_build_target_signatures:Function]
# #region _build_risks [TYPE Function]
# @BRIEF Build risk items for missing datasource, broken refs, overwrite, owner mismatch.
# [DEF:_build_risks:Function]
# @PURPOSE: Build risk items for missing datasource, broken refs, overwrite, owner mismatch.
def _build_risks(
self,
source_objects: Dict[str, List[Dict[str, Any]]],
@@ -354,7 +354,7 @@ class MigrationDryRunService:
) -> List[Dict[str, Any]]:
return build_risks(source_objects, target_objects, diff, target_client)
# #endregion _build_risks
# [/DEF:_build_risks:Function]
# #endregion MigrationDryRunService

81
backend/src/core/migration/risk_assessor.py
View File

@@ -1,16 +1,16 @@
# #region RiskAssessorModule [C:5] [TYPE Module] [SEMANTICS migration, dry_run, risk, scoring, preflight]
# @BRIEF Compute deterministic migration risk items and aggregate score for dry-run reporting.
# @LAYER Domain
# @PRE Risk assessor functions receive normalized migration object collections from dry-run orchestration.
# @POST Risk scoring output preserves item list and provides bounded score with derived level.
# @SIDE_EFFECT Emits diagnostic logs and performs read-only metadata requests via Superset client.
# @DATA_CONTRACT Module[build_risks, score_risks]
# @INVARIANT Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION CONTAINS -> [index_by_uuid]
# @RELATION CONTAINS -> [extract_owner_identifiers]
# @RELATION CONTAINS -> [build_risks]
# @RELATION CONTAINS -> [score_risks]
# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.
# @POST: Risk scoring output preserves item list and provides bounded score with derived level.
# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.
# @DATA_CONTRACT: Module[build_risks, score_risks]
# @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
# @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]
# @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
@@ -26,28 +26,25 @@
from typing import Any, Dict, List
from ..logger import belief_scope
from ..cot_logger import MarkerLogger
from ..logger import logger, belief_scope
from ..superset_client import SupersetClient
log = MarkerLogger("RiskAssessor")
# #region index_by_uuid [TYPE Function]
# @BRIEF Build UUID-index from normalized objects.
# @PRE Input list items are dict-like payloads potentially containing "uuid".
# @POST Returns mapping keyed by string uuid; only truthy uuid values are included.
# @SIDE_EFFECT Emits reasoning/reflective logs only.
# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]
# @PRE: Input list items are dict-like payloads potentially containing "uuid".
# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.
# @SIDE_EFFECT: Emits reasoning/reflective logs only.
# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]
def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
with belief_scope("risk_assessor.index_by_uuid"):
log.reason("Building UUID index", payload={"objects_count": len(objects)})
logger.reason("Building UUID index", extra={"objects_count": len(objects)})
indexed: Dict[str, Dict[str, Any]] = {}
for obj in objects:
uuid = obj.get("uuid")
if uuid:
indexed[str(uuid)] = obj
log.reflect("UUID index built", payload={"indexed_count": len(indexed)})
logger.reflect("UUID index built", extra={"indexed_count": len(indexed)})
return indexed
@@ -56,15 +53,15 @@ def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
# #region extract_owner_identifiers [TYPE Function]
# @BRIEF Normalize owner payloads for stable comparison.
# @PRE Owners may be list payload, scalar values, or None.
# @POST Returns sorted unique owner identifiers as strings.
# @SIDE_EFFECT Emits reasoning/reflective logs only.
# @DATA_CONTRACT Any -> List[str]
# @PRE: Owners may be list payload, scalar values, or None.
# @POST: Returns sorted unique owner identifiers as strings.
# @SIDE_EFFECT: Emits reasoning/reflective logs only.
# @DATA_CONTRACT: Any -> List[str]
def extract_owner_identifiers(owners: Any) -> List[str]:
with belief_scope("risk_assessor.extract_owner_identifiers"):
log.reason("Normalizing owner identifiers")
logger.reason("Normalizing owner identifiers")
if not isinstance(owners, list):
log.reflect("Owners payload is not list; returning empty identifiers")
logger.reflect("Owners payload is not list; returning empty identifiers")
return []
ids: List[str] = []
for owner in owners:
@@ -76,8 +73,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]:
elif owner is not None:
ids.append(str(owner))
normalized_ids = sorted(set(ids))
log.reflect(
"Owner identifiers normalized", payload={"owner_count": len(normalized_ids)}
logger.reflect(
"Owner identifiers normalized", extra={"owner_count": len(normalized_ids)}
)
return normalized_ids
@@ -87,18 +84,18 @@ def extract_owner_identifiers(owners: Any) -> List[str]:
# #region build_risks [TYPE Function]
# @BRIEF Build risk list from computed diffs and target catalog state.
# @PRE source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
# @PRE target_client is authenticated/usable for database list retrieval.
# @POST Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
# @SIDE_EFFECT Calls target Superset API for databases metadata and emits logs.
# @DATA_CONTRACT (
# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]],
# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]],
# @DATA_CONTRACT Dict[str, Dict[str, List[Dict[str, Any]]]],
# @DATA_CONTRACT SupersetClient
# @DATA_CONTRACT ) -> List[Dict[str, Any]]
# @RELATION DEPENDS_ON -> [index_by_uuid]
# @RELATION DEPENDS_ON -> [extract_owner_identifiers]
# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
# @PRE: target_client is authenticated/usable for database list retrieval.
# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.
# @DATA_CONTRACT: (
# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],
# @DATA_CONTRACT: Dict[str, List[Dict[str, Any]]],
# @DATA_CONTRACT: Dict[str, Dict[str, List[Dict[str, Any]]]],
# @DATA_CONTRACT: SupersetClient
# @DATA_CONTRACT: ) -> List[Dict[str, Any]]
def build_risks(
source_objects: Dict[str, List[Dict[str, Any]]],
target_objects: Dict[str, List[Dict[str, Any]]],
@@ -106,7 +103,7 @@ def build_risks(
target_client: SupersetClient,
) -> List[Dict[str, Any]]:
with belief_scope("risk_assessor.build_risks"):
log.reason("Building migration risks from diff payload")
logger.reason("Building migration risks from diff payload")
risks: List[Dict[str, Any]] = []
for object_type in ("dashboards", "charts", "datasets"):
for item in diff[object_type]["update"]:
@@ -171,7 +168,7 @@ def build_risks(
"message": f"Owner mismatch for dashboard {item.get('title') or item['uuid']}",
}
)
log.reflect("Risk list assembled", payload={"risk_count": len(risks)})
logger.reflect("Risk list assembled", extra={"risk_count": len(risks)})
return risks
@@ -180,20 +177,20 @@ def build_risks(
# #region score_risks [TYPE Function]
# @BRIEF Aggregate risk list into score and level.
# @PRE risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.
# @POST Returns dict with score in [0,100], derived level, and original items.
# @SIDE_EFFECT Emits reasoning/reflective logs only.
# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Any]
# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.
# @POST: Returns dict with score in [0,100], derived level, and original items.
# @SIDE_EFFECT: Emits reasoning/reflective logs only.
# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]
def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:
with belief_scope("risk_assessor.score_risks"):
log.reason("Scoring risk items", payload={"risk_items_count": len(risk_items)})
logger.reason("Scoring risk items", extra={"risk_items_count": len(risk_items)})
weights = {"high": 25, "medium": 10, "low": 5}
score = min(
100, sum(weights.get(item.get("severity", "low"), 5) for item in risk_items)
)
level = "low" if score < 25 else "medium" if score < 60 else "high"
result = {"score": score, "level": level, "items": risk_items}
log.reflect("Risk score computed", payload={"score": score, "level": level})
logger.reflect("Risk score computed", extra={"score": score, "level": level})
return result

74
backend/src/core/migration_engine.py
View File

@@ -1,18 +1,17 @@
# #region MigrationEngineModule [C:5] [TYPE Module] [SEMANTICS migration, engine, zip, yaml, transformation, cross-filter, id-mapping]
#
# @BRIEF Transforms Superset export ZIP archives while preserving archive integrity and patching mapped identifiers.
# @LAYER Domain
# @PRE Input archives are readable Superset exports and optional mapping collaborators expose remote id lookup APIs.
# @POST Migration engine contracts preserve ZIP integrity while exposing transformation entrypoints for import pipelines.
# @SIDE_EFFECT Reads and writes temporary archive contents during transformation workflows and emits structured belief-state logs.
# @DATA_CONTRACT Input[zip_path, output_path, db_mapping, target_env_id?, fix_cross_filters?] -> Output[Transformed Superset archive]
# @INVARIANT ZIP structure and non-targeted metadata must remain valid after transformation.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [LoggerModule]
# @RELATION DEPENDS_ON -> [IdMappingService]
# @RELATION DEPENDS_ON -> [ResourceType]
# @RELATION DEPENDS_ON -> [yaml]
#
# @PRE: Input archives are readable Superset exports and optional mapping collaborators expose remote id lookup APIs.
# @POST: Migration engine contracts preserve ZIP integrity while exposing transformation entrypoints for import pipelines.
# @SIDE_EFFECT: Reads and writes temporary archive contents during transformation workflows and emits structured belief-state logs.
# @DATA_CONTRACT: Input[zip_path, output_path, db_mapping, target_env_id?, fix_cross_filters?] -> Output[Transformed Superset archive]
# @INVARIANT: ZIP structure and non-targeted metadata must remain valid after transformation.
# [SECTION: IMPORTS]
import zipfile
import yaml
import os
@@ -24,19 +23,18 @@ from typing import Dict, Optional, List
from .logger import logger, belief_scope
from src.core.mapping_service import IdMappingService
from src.models.mapping import ResourceType
# [/SECTION]
# #region MigrationEngine [TYPE Class]
# @BRIEF Engine for transforming Superset export ZIPs.
# @RELATION CONTAINS -> [__init__, transform_zip, _transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata]
class MigrationEngine:
# #region __init__ [TYPE Function]
# @BRIEF Initializes migration orchestration dependencies for ZIP/YAML metadata transformations.
# @PRE mapping_service is None or implements batch remote ID lookup for ResourceType.CHART.
# @POST self.mapping_service is assigned and available for optional cross-filter patching flows.
# @SIDE_EFFECT Mutates in-memory engine state by storing dependency reference.
# @DATA_CONTRACT Input[Optional[IdMappingService]] -> Output[MigrationEngine]
# [DEF:__init__:Function]
# @PURPOSE: Initializes migration orchestration dependencies for ZIP/YAML metadata transformations.
# @PRE: mapping_service is None or implements batch remote ID lookup for ResourceType.CHART.
# @POST: self.mapping_service is assigned and available for optional cross-filter patching flows.
# @SIDE_EFFECT: Mutates in-memory engine state by storing dependency reference.
# @DATA_CONTRACT: Input[Optional[IdMappingService]] -> Output[MigrationEngine]
# @PARAM: mapping_service (Optional[IdMappingService]) - Used for resolving target environment integer IDs.
def __init__(self, mapping_service: Optional[IdMappingService] = None):
with belief_scope("MigrationEngine.__init__"):
@@ -44,11 +42,11 @@ class MigrationEngine:
self.mapping_service = mapping_service
logger.reflect("MigrationEngine initialized")
# #endregion __init__
# [/DEF:__init__:Function]
# #region transform_zip [TYPE Function]
# @BRIEF Extracts ZIP, replaces database UUIDs in YAMLs, patches cross-filters, and re-packages.
# @RELATION DEPENDS_ON -> [_transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata]
# [DEF:transform_zip:Function]
# @PURPOSE: Extracts ZIP, replaces database UUIDs in YAMLs, patches cross-filters, and re-packages.
# @RELATION: DEPENDS_ON -> [_transform_yaml, _extract_chart_uuids_from_archive, _patch_dashboard_metadata]
# @PARAM: zip_path (str) - Path to the source ZIP file.
# @PARAM: output_path (str) - Path where the transformed ZIP will be saved.
# @PARAM: db_mapping (Dict[str, str]) - Mapping of source UUID to target UUID.
@@ -144,10 +142,10 @@ class MigrationEngine:
logger.explore(f"Error transforming ZIP: {e}")
return False
# #endregion transform_zip
# [/DEF:transform_zip:Function]
# #region _transform_yaml [TYPE Function]
# @BRIEF Replaces database_uuid in a single YAML file.
# [DEF:_transform_yaml:Function]
# @PURPOSE: Replaces database_uuid in a single YAML file.
# @PARAM: file_path (Path) - Path to the YAML file.
# @PARAM: db_mapping (Dict[str, str]) - UUID mapping dictionary.
# @PRE: file_path exists, is readable YAML, and db_mapping contains source->target UUID pairs.
@@ -174,14 +172,14 @@ class MigrationEngine:
yaml.dump(data, f)
logger.reflect(f"Database UUID patched in {file_path.name}")
# #endregion _transform_yaml
# [/DEF:_transform_yaml:Function]
# #region _extract_chart_uuids_from_archive [TYPE Function]
# @BRIEF Scans extracted chart YAML files and builds a source chart ID to UUID lookup map.
# @PRE temp_dir exists and points to extracted archive root with optional chart YAML resources.
# @POST Returns a best-effort Dict[int, str] containing only parseable chart id/uuid pairs.
# @SIDE_EFFECT Reads chart YAML files from filesystem; suppresses per-file parsing failures.
# @DATA_CONTRACT Input[Path] -> Output[Dict[int,str]]
# [DEF:_extract_chart_uuids_from_archive:Function]
# @PURPOSE: Scans extracted chart YAML files and builds a source chart ID to UUID lookup map.
# @PRE: temp_dir exists and points to extracted archive root with optional chart YAML resources.
# @POST: Returns a best-effort Dict[int, str] containing only parseable chart id/uuid pairs.
# @SIDE_EFFECT: Reads chart YAML files from filesystem; suppresses per-file parsing failures.
# @DATA_CONTRACT: Input[Path] -> Output[Dict[int,str]]
# @PARAM: temp_dir (Path) - Root dir of unpacked archive.
# @RETURN: Dict[int, str] - Mapping of source Integer ID to UUID.
def _extract_chart_uuids_from_archive(self, temp_dir: Path) -> Dict[int, str]:
@@ -204,14 +202,14 @@ class MigrationEngine:
pass
return mapping
# #endregion _extract_chart_uuids_from_archive
# [/DEF:_extract_chart_uuids_from_archive:Function]
# #region _patch_dashboard_metadata [TYPE Function]
# @BRIEF Rewrites dashboard json_metadata chart/dataset integer identifiers using target environment mappings.
# @PRE file_path points to dashboard YAML with json_metadata; target_env_id is non-empty; source_map contains source id->uuid.
# @POST json_metadata is re-serialized with mapped integer IDs when remote mappings are available; otherwise file remains unchanged.
# @SIDE_EFFECT Reads/writes YAML file, performs mapping lookup via mapping_service, emits logs for recoverable/terminal failures.
# @DATA_CONTRACT Input[(Path file_path, str target_env_id, Dict[int,str] source_map)] -> Output[None]
# [DEF:_patch_dashboard_metadata:Function]
# @PURPOSE: Rewrites dashboard json_metadata chart/dataset integer identifiers using target environment mappings.
# @PRE: file_path points to dashboard YAML with json_metadata; target_env_id is non-empty; source_map contains source id->uuid.
# @POST: json_metadata is re-serialized with mapped integer IDs when remote mappings are available; otherwise file remains unchanged.
# @SIDE_EFFECT: Reads/writes YAML file, performs mapping lookup via mapping_service, emits logs for recoverable/terminal failures.
# @DATA_CONTRACT: Input[(Path file_path, str target_env_id, Dict[int,str] source_map)] -> Output[None]
# @PARAM: file_path (Path)
# @PARAM: target_env_id (str)
# @PARAM: source_map (Dict[int, str])
@@ -295,8 +293,8 @@ class MigrationEngine:
except Exception as e:
logger.explore(f"Metadata patch failed for {file_path.name}: {e}")
# #endregion _patch_dashboard_metadata
# [/DEF:_patch_dashboard_metadata:Function]
# #endregion MigrationEngine
# #endregion MigrationEngineModule
# #endregion MigrationEngineModule

98
backend/src/core/plugin_base.py
View File

@@ -6,9 +6,9 @@ from pydantic import BaseModel, Field
# #region PluginBase [TYPE Class] [SEMANTICS plugin, interface, base, abstract]
# @BRIEF Defines the abstract base class that all plugins must implement to be recognized by the system. It enforces a common structure for plugin metadata and execution.
# @LAYER Core
# @INVARIANT All plugins MUST inherit from this class.
# @LAYER: Core
# @RELATION Used by PluginLoader to identify valid plugins.
# @INVARIANT: All plugins MUST inherit from this class.
class PluginBase(ABC):
"""
Base class for all plugins.
@@ -17,74 +17,74 @@ class PluginBase(ABC):
@property
@abstractmethod
# #region id [TYPE Function]
# @BRIEF Returns the unique identifier for the plugin.
# @PRE Plugin instance exists.
# @POST Returns string ID.
# @RETURN str - Plugin ID.
# [DEF:id:Function]
# @PURPOSE: Returns the unique identifier for the plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string ID.
# @RETURN: str - Plugin ID.
def id(self) -> str:
"""A unique identifier for the plugin."""
with belief_scope("id"):
pass
# #endregion id
# [/DEF:id:Function]
@property
@abstractmethod
# #region name [TYPE Function]
# @BRIEF Returns the human-readable name of the plugin.
# @PRE Plugin instance exists.
# @POST Returns string name.
# @RETURN str - Plugin name.
# [DEF:name:Function]
# @PURPOSE: Returns the human-readable name of the plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string name.
# @RETURN: str - Plugin name.
def name(self) -> str:
"""A human-readable name for the plugin."""
with belief_scope("name"):
pass
# #endregion name
# [/DEF:name:Function]
@property
@abstractmethod
# #region description [TYPE Function]
# @BRIEF Returns a brief description of the plugin.
# @PRE Plugin instance exists.
# @POST Returns string description.
# @RETURN str - Plugin description.
# [DEF:description:Function]
# @PURPOSE: Returns a brief description of the plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string description.
# @RETURN: str - Plugin description.
def description(self) -> str:
"""A brief description of what the plugin does."""
with belief_scope("description"):
pass
# #endregion description
# [/DEF:description:Function]
@property
@abstractmethod
# #region version [TYPE Function]
# @BRIEF Returns the version of the plugin.
# @PRE Plugin instance exists.
# @POST Returns string version.
# @RETURN str - Plugin version.
# [DEF:version:Function]
# @PURPOSE: Returns the version of the plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string version.
# @RETURN: str - Plugin version.
def version(self) -> str:
"""The version of the plugin."""
with belief_scope("version"):
pass
# #endregion version
# [/DEF:version:Function]
@property
# #region required_permission [TYPE Function]
# @BRIEF Returns the required permission string to execute this plugin.
# @PRE Plugin instance exists.
# @POST Returns string permission.
# @RETURN str - Required permission (e.g., "plugin:backup:execute").
# [DEF:required_permission:Function]
# @PURPOSE: Returns the required permission string to execute this plugin.
# @PRE: Plugin instance exists.
# @POST: Returns string permission.
# @RETURN: str - Required permission (e.g., "plugin:backup:execute").
def required_permission(self) -> str:
"""The permission string required to execute this plugin."""
with belief_scope("required_permission"):
return f"plugin:{self.id}:execute"
# #endregion required_permission
# [/DEF:required_permission:Function]
@property
# #region ui_route [TYPE Function]
# @BRIEF Returns the frontend route for the plugin's UI, if applicable.
# @PRE Plugin instance exists.
# @POST Returns string route or None.
# @RETURN Optional[str] - Frontend route.
# [DEF:ui_route:Function]
# @PURPOSE: Returns the frontend route for the plugin's UI, if applicable.
# @PRE: Plugin instance exists.
# @POST: Returns string route or None.
# @RETURN: Optional[str] - Frontend route.
def ui_route(self) -> Optional[str]:
"""
The frontend route for the plugin's UI.
@@ -92,14 +92,14 @@ class PluginBase(ABC):
"""
with belief_scope("ui_route"):
return None
# #endregion ui_route
# [/DEF:ui_route:Function]
@abstractmethod
# #region get_schema [TYPE Function]
# @BRIEF Returns the JSON schema for the plugin's input parameters.
# @PRE Plugin instance exists.
# @POST Returns dict schema.
# @RETURN Dict[str, Any] - JSON schema.
# [DEF:get_schema:Function]
# @PURPOSE: Returns the JSON schema for the plugin's input parameters.
# @PRE: Plugin instance exists.
# @POST: Returns dict schema.
# @RETURN: Dict[str, Any] - JSON schema.
def get_schema(self) -> Dict[str, Any]:
"""
Returns the JSON schema for the plugin's input parameters.
@@ -107,11 +107,11 @@ class PluginBase(ABC):
"""
with belief_scope("get_schema"):
pass
# #endregion get_schema
# [/DEF:get_schema:Function]
@abstractmethod
# #region execute [TYPE Function]
# @BRIEF Executes the plugin's core logic.
# [DEF:execute:Function]
# @PURPOSE: Executes the plugin's core logic.
# @PARAM: params (Dict[str, Any]) - Validated input parameters.
# @PRE: params must be a dictionary.
# @POST: Plugin execution is completed.
@@ -123,12 +123,12 @@ class PluginBase(ABC):
The `params` argument will be validated against the schema returned by `get_schema()`.
"""
pass
# #endregion execute
# [/DEF:execute:Function]
# #endregion PluginBase
# #region PluginConfig [TYPE Class] [SEMANTICS plugin, config, schema, pydantic]
# @BRIEF A Pydantic model used to represent the validated configuration and metadata of a loaded plugin. This object is what gets exposed to the API layer.
# @LAYER Core
# @LAYER: Core
# @RELATION Instantiated by PluginLoader after validating a PluginBase instance.
class PluginConfig(BaseModel):
"""Pydantic model for plugin configuration."""
@@ -138,4 +138,4 @@ class PluginConfig(BaseModel):
version: str = Field(..., description="Version of the plugin")
ui_route: Optional[str] = Field(None, description="Frontend route for the plugin UI")
input_schema: Dict[str, Any] = Field(..., description="JSON schema for input parameters", alias="schema")
# #endregion PluginConfig
# #endregion PluginConfig

74
backend/src/core/plugin_loader.py
View File

@@ -7,7 +7,7 @@ from .logger import belief_scope
# #region PluginLoader [C:3] [TYPE Class] [SEMANTICS plugin, loader, dynamic, import]
# @BRIEF Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
# @LAYER Core
# @LAYER: Core
# @RELATION Depends on PluginBase. It is used by the main application to discover and manage available plugins.
class PluginLoader:
"""
@@ -15,10 +15,10 @@ class PluginLoader:
that inherit from PluginBase.
"""
# #region __init__ [TYPE Function]
# @BRIEF Initializes the PluginLoader with a directory to scan.
# @PRE plugin_dir is a valid directory path.
# @POST Plugins are loaded and registered.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the PluginLoader with a directory to scan.
# @PRE: plugin_dir is a valid directory path.
# @POST: Plugins are loaded and registered.
# @PARAM: plugin_dir (str) - The directory containing plugin modules.
def __init__(self, plugin_dir: str):
with belief_scope("__init__"):
@@ -26,12 +26,12 @@ class PluginLoader:
self._plugins: Dict[str, PluginBase] = {}
self._plugin_configs: Dict[str, PluginConfig] = {}
self._load_plugins()
# #endregion __init__
# [/DEF:__init__:Function]
# #region _load_plugins [TYPE Function]
# @BRIEF Scans the plugin directory and loads all valid plugins.
# @PRE plugin_dir exists or can be created.
# @POST _load_module is called for each .py file.
# [DEF:_load_plugins:Function]
# @PURPOSE: Scans the plugin directory and loads all valid plugins.
# @PRE: plugin_dir exists or can be created.
# @POST: _load_module is called for each .py file.
def _load_plugins(self):
with belief_scope("_load_plugins"):
"""
@@ -61,12 +61,12 @@ class PluginLoader:
if filename.endswith(".py") and filename != "__init__.py":
module_name = filename[:-3]
self._load_module(module_name, file_path)
# #endregion _load_plugins
# [/DEF:_load_plugins:Function]
# #region _load_module [TYPE Function]
# @BRIEF Loads a single Python module and discovers PluginBase implementations.
# @PRE module_name and file_path are valid.
# @POST Plugin classes are instantiated and registered.
# [DEF:_load_module:Function]
# @PURPOSE: Loads a single Python module and discovers PluginBase implementations.
# @PRE: module_name and file_path are valid.
# @POST: Plugin classes are instantiated and registered.
# @PARAM: module_name (str) - The name of the module.
# @PARAM: file_path (str) - The path to the module file.
def _load_module(self, module_name: str, file_path: str):
@@ -102,12 +102,12 @@ class PluginLoader:
self._register_plugin(plugin_instance)
except Exception as e:
print(f"Error instantiating plugin {attribute_name} in {module_name}: {e}") # Replace with proper logging
# #endregion _load_module
# [/DEF:_load_module:Function]
# #region _register_plugin [TYPE Function]
# @BRIEF Registers a PluginBase instance and its configuration.
# @PRE plugin_instance is a valid implementation of PluginBase.
# @POST Plugin is added to _plugins and _plugin_configs.
# [DEF:_register_plugin:Function]
# @PURPOSE: Registers a PluginBase instance and its configuration.
# @PRE: plugin_instance is a valid implementation of PluginBase.
# @POST: Plugin is added to _plugins and _plugin_configs.
# @PARAM: plugin_instance (PluginBase) - The plugin instance to register.
def _register_plugin(self, plugin_instance: PluginBase):
with belief_scope("_register_plugin"):
@@ -143,13 +143,13 @@ class PluginLoader:
except Exception as e:
from ..core.logger import logger
logger.error(f"Error validating plugin '{plugin_instance.name}' (ID: {plugin_id}): {e}")
# #endregion _register_plugin
# [/DEF:_register_plugin:Function]
# #region get_plugin [TYPE Function]
# @BRIEF Retrieves a loaded plugin instance by its ID.
# @PRE plugin_id is a string.
# @POST Returns plugin instance or None.
# [DEF:get_plugin:Function]
# @PURPOSE: Retrieves a loaded plugin instance by its ID.
# @PRE: plugin_id is a string.
# @POST: Returns plugin instance or None.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: Optional[PluginBase] - The plugin instance if found, otherwise None.
def get_plugin(self, plugin_id: str) -> Optional[PluginBase]:
@@ -158,25 +158,25 @@ class PluginLoader:
Returns a loaded plugin instance by its ID.
"""
return self._plugins.get(plugin_id)
# #endregion get_plugin
# [/DEF:get_plugin:Function]
# #region get_all_plugin_configs [TYPE Function]
# @BRIEF Returns a list of all registered plugin configurations.
# @PRE None.
# @POST Returns list of all PluginConfig objects.
# @RETURN List[PluginConfig] - A list of plugin configurations.
# [DEF:get_all_plugin_configs:Function]
# @PURPOSE: Returns a list of all registered plugin configurations.
# @PRE: None.
# @POST: Returns list of all PluginConfig objects.
# @RETURN: List[PluginConfig] - A list of plugin configurations.
def get_all_plugin_configs(self) -> List[PluginConfig]:
with belief_scope("get_all_plugin_configs"):
"""
Returns a list of all loaded plugin configurations.
"""
return list(self._plugin_configs.values())
# #endregion get_all_plugin_configs
# [/DEF:get_all_plugin_configs:Function]
# #region has_plugin [TYPE Function]
# @BRIEF Checks if a plugin with the given ID is registered.
# @PRE plugin_id is a string.
# @POST Returns True if plugin exists.
# [DEF:has_plugin:Function]
# @PURPOSE: Checks if a plugin with the given ID is registered.
# @PRE: plugin_id is a string.
# @POST: Returns True if plugin exists.
# @PARAM: plugin_id (str) - The unique identifier of the plugin.
# @RETURN: bool - True if the plugin is registered, False otherwise.
def has_plugin(self, plugin_id: str) -> bool:
@@ -185,6 +185,6 @@ class PluginLoader:
Checks if a plugin with the given ID is loaded.
"""
return plugin_id in self._plugins
# #endregion has_plugin
# [/DEF:has_plugin:Function]
# #endregion PluginLoader

132
backend/src/core/scheduler.py
View File

@@ -1,30 +1,24 @@
# #region SchedulerModule [C:3] [TYPE Module] [SEMANTICS scheduler, apscheduler, cron, backup]
# @BRIEF Manages scheduled tasks using APScheduler.
# @LAYER Core
# @RELATION DEPENDS_ON -> [TaskManager]
# @LAYER: Core
# @RELATION DEPENDS_ON -> TaskManager
# [SECTION: IMPORTS]
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from .logger import logger, belief_scope
from .config_manager import ConfigManager
from .cot_logger import MarkerLogger, get_trace_id, set_trace_id
log = MarkerLogger("SchedulerService")
log_tsc = MarkerLogger("ThrottledSchedulerConfigurator")
import asyncio
from datetime import datetime, time, timedelta, date
# [/SECTION]
# #region SchedulerService [C:3] [TYPE Class] [SEMANTICS scheduler, service, apscheduler]
# @BRIEF Provides a service to manage scheduled backup tasks.
# @RELATION DEPENDS_ON -> [ThrottledSchedulerConfigurator]
class SchedulerService:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the scheduler service with task and config managers.
# @PRE task_manager and config_manager must be provided.
# @POST Scheduler instance is created but not started.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the scheduler service with task and config managers.
# @PRE: task_manager and config_manager must be provided.
# @POST: Scheduler instance is created but not started.
def __init__(self, task_manager, config_manager: ConfigManager):
with belief_scope("SchedulerService.__init__"):
self.task_manager = task_manager
@@ -32,37 +26,37 @@ class SchedulerService:
self.scheduler = BackgroundScheduler()
self.loop = asyncio.get_event_loop()
# #endregion __init__
# [/DEF:__init__:Function]
# #region start [TYPE Function]
# @BRIEF Starts the background scheduler and loads initial schedules.
# @PRE Scheduler should be initialized.
# @POST Scheduler is running and schedules are loaded.
# [DEF:start:Function]
# @PURPOSE: Starts the background scheduler and loads initial schedules.
# @PRE: Scheduler should be initialized.
# @POST: Scheduler is running and schedules are loaded.
def start(self):
with belief_scope("SchedulerService.start"):
if not self.scheduler.running:
self.scheduler.start()
log.reflect("Scheduler started")
logger.info("Scheduler started.")
self.load_schedules()
# #endregion start
# [/DEF:start:Function]
# #region stop [TYPE Function]
# @BRIEF Stops the background scheduler.
# @PRE Scheduler should be running.
# @POST Scheduler is shut down.
# [DEF:stop:Function]
# @PURPOSE: Stops the background scheduler.
# @PRE: Scheduler should be running.
# @POST: Scheduler is shut down.
def stop(self):
with belief_scope("SchedulerService.stop"):
if self.scheduler.running:
self.scheduler.shutdown()
log.reflect("Scheduler stopped")
logger.info("Scheduler stopped.")
# #endregion stop
# [/DEF:stop:Function]
# #region load_schedules [TYPE Function]
# @BRIEF Loads backup schedules from configuration and registers them.
# @PRE config_manager must have valid configuration.
# @POST All enabled backup jobs are added to the scheduler.
# [DEF:load_schedules:Function]
# @PURPOSE: Loads backup schedules from configuration and registers them.
# @PRE: config_manager must have valid configuration.
# @POST: All enabled backup jobs are added to the scheduler.
def load_schedules(self):
with belief_scope("SchedulerService.load_schedules"):
# Clear existing jobs
@@ -73,12 +67,12 @@ class SchedulerService:
if env.backup_schedule and env.backup_schedule.enabled:
self.add_backup_job(env.id, env.backup_schedule.cron_expression)
# #endregion load_schedules
# [/DEF:load_schedules:Function]
# #region add_backup_job [TYPE Function]
# @BRIEF Adds a scheduled backup job for an environment.
# @PRE env_id and cron_expression must be valid strings.
# @POST A new job is added to the scheduler or replaced if it already exists.
# [DEF:add_backup_job:Function]
# @PURPOSE: Adds a scheduled backup job for an environment.
# @PRE: env_id and cron_expression must be valid strings.
# @POST: A new job is added to the scheduler or replaced if it already exists.
# @PARAM: env_id (str) - The ID of the environment.
# @PARAM: cron_expression (str) - The cron expression for the schedule.
def add_backup_job(self, env_id: str, cron_expression: str):
@@ -95,24 +89,22 @@ class SchedulerService:
args=[env_id],
replace_existing=True,
)
log.reflect(
"Scheduled backup job added",
payload={"env_id": env_id, "cron": cron_expression},
logger.info(
f"Scheduled backup job added for environment {env_id}: {cron_expression}"
)
except Exception as e:
log.explore("Failed to add backup job", error=str(e), payload={"env_id": env_id, "cron": cron_expression})
logger.error(f"Failed to add backup job for environment {env_id}: {e}")
# #endregion add_backup_job
# [/DEF:add_backup_job:Function]
# #region _trigger_backup [TYPE Function]
# @BRIEF Triggered by the scheduler to start a backup task.
# @PRE env_id must be a valid environment ID.
# @POST A new backup task is created in the task manager if not already running.
# [DEF:_trigger_backup:Function]
# @PURPOSE: Triggered by the scheduler to start a backup task.
# @PRE: env_id must be a valid environment ID.
# @POST: A new backup task is created in the task manager if not already running.
# @PARAM: env_id (str) - The ID of the environment.
def _trigger_backup(self, env_id: str):
with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"):
log.reason("Triggering scheduled backup", payload={"env_id": env_id})
logger.info(f"Triggering scheduled backup for environment {env_id}")
# Check if a backup is already running for this environment
active_tasks = self.task_manager.get_tasks(limit=100)
@@ -122,23 +114,21 @@ class SchedulerService:
and task.status in ["PENDING", "RUNNING"]
and task.params.get("environment_id") == env_id
):
log.explore("Backup already running, skipping scheduled run", error="Duplicate backup prevented", payload={"env_id": env_id})
logger.warning(
f"Backup already running for environment {env_id}. Skipping scheduled run."
)
return
# Run the backup task
# We need to run this in the event loop since create_task is async
trace_id = get_trace_id()
async def _backup_task():
if trace_id:
set_trace_id(trace_id)
await self.task_manager.create_task(
asyncio.run_coroutine_threadsafe(
self.task_manager.create_task(
"superset-backup", {"environment_id": env_id}
)
),
self.loop,
)
asyncio.run_coroutine_threadsafe(_backup_task(), self.loop)
# #endregion _trigger_backup
# [/DEF:_trigger_backup:Function]
# #endregion SchedulerService
@@ -146,18 +136,18 @@ class SchedulerService:
# #region ThrottledSchedulerConfigurator [C:5] [TYPE Class] [SEMANTICS scheduler, throttling, distribution]
# @BRIEF Distributes validation tasks evenly within an execution window.
# @PRE Validation policies provide a finite dashboard list and a valid execution window.
# @POST Produces deterministic per-dashboard run timestamps within the configured window.
# @SIDE_EFFECT Emits warning logs for degenerate or near-zero scheduling windows.
# @DATA_CONTRACT Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]
# @INVARIANT Returned schedule size always matches number of dashboard IDs.
# @RELATION DEPENDS_ON -> [SchedulerModule]
# @PRE: Validation policies provide a finite dashboard list and a valid execution window.
# @POST: Produces deterministic per-dashboard run timestamps within the configured window.
# @RELATION DEPENDS_ON -> SchedulerModule
# @INVARIANT: Returned schedule size always matches number of dashboard IDs.
# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.
# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]
class ThrottledSchedulerConfigurator:
# #region calculate_schedule [TYPE Function]
# @BRIEF Calculates execution times for N tasks within a window.
# @PRE window_start, window_end (time), dashboard_ids (List), current_date (date).
# @POST Returns List[datetime] of scheduled times.
# @INVARIANT Tasks are distributed with near-even spacing.
# [DEF:calculate_schedule:Function]
# @PURPOSE: Calculates execution times for N tasks within a window.
# @PRE: window_start, window_end (time), dashboard_ids (List), current_date (date).
# @POST: Returns List[datetime] of scheduled times.
# @INVARIANT: Tasks are distributed with near-even spacing.
@staticmethod
def calculate_schedule(
window_start: time, window_end: time, dashboard_ids: list, current_date: date
@@ -178,7 +168,9 @@ class ThrottledSchedulerConfigurator:
# Minimum interval of 1 second to avoid division by zero or negative
if total_seconds <= 0:
log_tsc.explore("Window size is zero or negative, falling back to start time", error=f"total_seconds={total_seconds}", payload={"n": n})
logger.warning(
f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks."
)
return [start_dt] * n
# If window is too small for even distribution (e.g. 10 tasks in 5 seconds),
@@ -193,7 +185,9 @@ class ThrottledSchedulerConfigurator:
# If interval is too small (e.g. < 1s), we might want a fallback,
# but the spec says "handle too-small windows with explicit fallback/warning".
if interval < 1:
log_tsc.explore("Window too small for task distribution", error=f"interval={interval:.2f}s", payload={"n": n})
logger.warning(
f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated."
)
scheduled_times = []
for i in range(n):
@@ -201,7 +195,7 @@ class ThrottledSchedulerConfigurator:
return scheduled_times
# #endregion calculate_schedule
# [/DEF:calculate_schedule:Function]
# #endregion ThrottledSchedulerConfigurator

4
backend/src/core/superset_client/__init__.py
View File

@@ -1,13 +1,13 @@
# #region SupersetClientModule [C:3] [TYPE Module] [SEMANTICS superset, api, client, rest, http, dashboard, dataset, import, export]
# @LAYER: Infra
# @BRIEF Предоставляет высокоуровневый клиент для взаимодействия с Superset REST API, инкапсулируя логику запросов, обработку ошибок и пагинацию.
# @LAYER Infra
# @INVARIANT All network operations must use the internal APIClient instance.
# @RELATION DEPENDS_ON -> [ConfigModels]
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [get_filename_from_headers]
# @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewFiltersMixin]
#
# @INVARIANT: All network operations must use the internal APIClient instance.
# @PUBLIC_API: SupersetClient
#
# @RATIONALE: Decomposed from monolithic superset_client.py (2145 lines) into

164
backend/src/core/superset_client/_base.py
View File

@@ -1,6 +1,6 @@
# #region SupersetClientBase [C:3] [TYPE Module] [SEMANTICS superset, api, client, base, pagination, auth, import, export]
# @LAYER: Infra
# @BRIEF Base class for SupersetClient providing initialization, authentication, pagination, and import/export helpers.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [ConfigModels]
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
@@ -12,13 +12,12 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, cast
from requests import Response
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
from ..utils.network import APIClient, SupersetAPIError
from ..utils.fileio import get_filename_from_headers
from ..config_models import Environment
log = MarkerLogger("SupersetClientBase")
app_logger = cast(Any, app_logger)
# #region SupersetClientBase [C:3] [TYPE Class]
@@ -27,15 +26,16 @@ log = MarkerLogger("SupersetClientBase")
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
class SupersetClientBase:
# #region SupersetClientInit [C:3] [TYPE Function]
# @BRIEF Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @RELATION DEPENDS_ON -> [Environment]
# @RELATION DEPENDS_ON -> [APIClient]
# [DEF:SupersetClientInit:Function]
# @COMPLEXITY: 3
# @PURPOSE: Инициализирует клиент, проверяет конфигурацию и создает сетевой клиент.
# @RELATION: DEPENDS_ON -> [Environment]
# @RELATION: DEPENDS_ON -> [APIClient]
def __init__(self, env: Environment):
with belief_scope("SupersetClientInit"):
log.reason(
app_logger.reason(
"Initializing Superset client for environment",
payload={"environment": getattr(env, "id", None), "env_name": env.name},
extra={"environment": getattr(env, "id", None), "env_name": env.name},
)
self.env = env
# Construct auth payload expected by Superset API
@@ -51,46 +51,49 @@ class SupersetClientBase:
timeout=env.timeout,
)
self.delete_before_reimport: bool = False
log.reflect(
app_logger.reflect(
"Superset client initialized",
payload={"environment": getattr(self.env, "id", None)},
extra={"environment": getattr(self.env, "id", None)},
)
# #endregion SupersetClientInit
# [/DEF:SupersetClientInit:Function]
# #region SupersetClientAuthenticate [C:3] [TYPE Function]
# @BRIEF Authenticates the client using the configured credentials.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientAuthenticate:Function]
# @COMPLEXITY: 3
# @PURPOSE: Authenticates the client using the configured credentials.
# @RELATION: CALLS -> [APIClient]
def authenticate(self) -> Dict[str, str]:
with belief_scope("SupersetClientAuthenticate"):
log.reason(
app_logger.reason(
"Authenticating Superset client",
payload={"environment": getattr(self.env, "id", None)},
extra={"environment": getattr(self.env, "id", None)},
)
tokens = self.network.authenticate()
log.reflect(
app_logger.reflect(
"Superset client authentication completed",
payload={
extra={
"environment": getattr(self.env, "id", None),
"token_keys": sorted(tokens.keys()),
},
)
return tokens
# #endregion SupersetClientAuthenticate
# [/DEF:SupersetClientAuthenticate:Function]
@property
# #region SupersetClientHeaders [C:1] [TYPE Function]
# @BRIEF Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
# [DEF:SupersetClientHeaders:Function]
# @COMPLEXITY: 1
# @PURPOSE: Возвращает базовые HTTP-заголовки, используемые сетевым клиентом.
def headers(self) -> dict:
with belief_scope("headers"):
return self.network.headers
# #endregion SupersetClientHeaders
# [/DEF:SupersetClientHeaders:Function]
# --- Pagination helpers ---
# #region SupersetClientValidateQueryParams [C:1] [TYPE Function]
# @BRIEF Ensures query parameters have default page and page_size.
# [DEF:SupersetClientValidateQueryParams:Function]
# @COMPLEXITY: 1
# @PURPOSE: Ensures query parameters have default page and page_size.
def _validate_query_params(self, query: Optional[Dict]) -> Dict:
with belief_scope("_validate_query_params"):
# Superset list endpoints commonly cap page_size at 100.
@@ -98,11 +101,12 @@ class SupersetClientBase:
base_query = {"page": 0, "page_size": 100}
return {**base_query, **(query or {})}
# #endregion SupersetClientValidateQueryParams
# [/DEF:SupersetClientValidateQueryParams:Function]
# #region SupersetClientFetchTotalObjectCount [C:1] [TYPE Function]
# @BRIEF Fetches the total number of items for a given endpoint.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientFetchTotalObjectCount:Function]
# @COMPLEXITY: 1
# @PURPOSE: Fetches the total number of items for a given endpoint.
# @RELATION: CALLS -> [APIClient]
def _fetch_total_object_count(self, endpoint: str) -> int:
with belief_scope("_fetch_total_object_count"):
return self.network.fetch_paginated_count(
@@ -111,30 +115,34 @@ class SupersetClientBase:
count_field="count",
)
# #endregion SupersetClientFetchTotalObjectCount
# [/DEF:SupersetClientFetchTotalObjectCount:Function]
# #region SupersetClientFetchAllPages [C:1] [TYPE Function]
# @BRIEF Iterates through all pages to collect all data items.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientFetchAllPages:Function]
# @COMPLEXITY: 1
# @PURPOSE: Iterates through all pages to collect all data items.
# @RELATION: CALLS -> [APIClient]
def _fetch_all_pages(self, endpoint: str, pagination_options: Dict) -> List[Dict]:
with belief_scope("_fetch_all_pages"):
return self.network.fetch_paginated_data(
endpoint=endpoint, pagination_options=pagination_options
)
# #endregion SupersetClientFetchAllPages
# [/DEF:SupersetClientFetchAllPages:Function]
# --- Import/Export helpers ---
# #region SupersetClientDoImport [C:1] [TYPE Function]
# @BRIEF Performs the actual multipart upload for import.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientDoImport:Function]
# @COMPLEXITY: 1
# @PURPOSE: Performs the actual multipart upload for import.
# @RELATION: CALLS -> [APIClient]
def _do_import(self, file_name: Union[str, Path]) -> Dict:
with belief_scope("_do_import"):
log.reason("Uploading file for import", payload={"file_name": str(file_name)})
app_logger.debug(f"[_do_import][State] Uploading file: {file_name}")
file_path = Path(file_name)
if not file_path.exists():
log.explore("Import file does not exist", error="FileNotFound", payload={"file_name": str(file_name)})
app_logger.error(
f"[_do_import][Failure] File does not exist: {file_name}"
)
raise FileNotFoundError(f"File does not exist: {file_name}")
return self.network.upload_file(
@@ -148,10 +156,11 @@ class SupersetClientBase:
timeout=self.env.timeout * 2,
)
# #endregion SupersetClientDoImport
# [/DEF:SupersetClientDoImport:Function]
# #region SupersetClientValidateExportResponse [C:1] [TYPE Function]
# @BRIEF Validates that the export response is a non-empty ZIP archive.
# [DEF:SupersetClientValidateExportResponse:Function]
# @COMPLEXITY: 1
# @PURPOSE: Validates that the export response is a non-empty ZIP archive.
def _validate_export_response(self, response: Response, dashboard_id: int) -> None:
with belief_scope("_validate_export_response"):
content_type = response.headers.get("Content-Type", "")
@@ -162,10 +171,11 @@ class SupersetClientBase:
if not response.content:
raise SupersetAPIError("Получены пустые данные при экспорте")
# #endregion SupersetClientValidateExportResponse
# [/DEF:SupersetClientValidateExportResponse:Function]
# #region SupersetClientResolveExportFilename [C:1] [TYPE Function]
# @BRIEF Determines the filename for an exported dashboard.
# [DEF:SupersetClientResolveExportFilename:Function]
# @COMPLEXITY: 1
# @PURPOSE: Determines the filename for an exported dashboard.
def _resolve_export_filename(self, response: Response, dashboard_id: int) -> str:
with belief_scope("_resolve_export_filename"):
filename = get_filename_from_headers(dict(response.headers))
@@ -174,16 +184,17 @@ class SupersetClientBase:
timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
filename = f"dashboard_export_{dashboard_id}_{timestamp}.zip"
log.reflect(
"Export filename not found in headers, using generated name",
payload={"dashboard_id": dashboard_id, "generated_filename": filename},
app_logger.warning(
"[_resolve_export_filename][Warning] Generated filename: %s",
filename,
)
return filename
# #endregion SupersetClientResolveExportFilename
# [/DEF:SupersetClientResolveExportFilename:Function]
# #region SupersetClientValidateImportFile [C:1] [TYPE Function]
# @BRIEF Validates that the file to be imported is a valid ZIP with metadata.yaml.
# [DEF:SupersetClientValidateImportFile:Function]
# @COMPLEXITY: 1
# @PURPOSE: Validates that the file to be imported is a valid ZIP with metadata.yaml.
def _validate_import_file(self, zip_path: Union[str, Path]) -> None:
with belief_scope("_validate_import_file"):
path = Path(zip_path)
@@ -197,11 +208,12 @@ class SupersetClientBase:
f"Архив {zip_path} не содержит 'metadata.yaml'"
)
# #endregion SupersetClientValidateImportFile
# [/DEF:SupersetClientValidateImportFile:Function]
# #region SupersetClientResolveTargetIdForDelete [C:1] [TYPE Function]
# @BRIEF Resolves a dashboard ID from either an ID or a slug.
# @RELATION CALLS -> [SupersetClientGetDashboards]
# [DEF:SupersetClientResolveTargetIdForDelete:Function]
# @COMPLEXITY: 1
# @PURPOSE: Resolves a dashboard ID from either an ID or a slug.
# @RELATION: CALLS -> [SupersetClientGetDashboards]
def _resolve_target_id_for_delete(
self, dash_id: Optional[int], dash_slug: Optional[str]
) -> Optional[int]:
@@ -209,7 +221,10 @@ class SupersetClientBase:
if dash_id is not None:
return dash_id
if dash_slug is not None:
log.reason("Resolving dashboard ID by slug", payload={"slug": dash_slug})
app_logger.debug(
"[_resolve_target_id_for_delete][State] Resolving ID by slug '%s'.",
dash_slug,
)
try:
_, candidates = self.get_dashboards(
query={
@@ -218,17 +233,25 @@ class SupersetClientBase:
)
if candidates:
target_id = candidates[0]["id"]
log.reflect("Resolved slug to dashboard ID", payload={"slug": dash_slug, "target_id": target_id})
app_logger.debug(
"[_resolve_target_id_for_delete][Success] Resolved slug to ID %s.",
target_id,
)
return target_id
except Exception as e:
log.explore("Could not resolve slug to dashboard ID", error=str(e), payload={"slug": dash_slug})
app_logger.warning(
"[_resolve_target_id_for_delete][Warning] Could not resolve slug '%s' to ID: %s",
dash_slug,
e,
)
return None
# #endregion SupersetClientResolveTargetIdForDelete
# [/DEF:SupersetClientResolveTargetIdForDelete:Function]
# #region SupersetClientGetAllResources [C:3] [TYPE Function]
# @BRIEF Fetches all resources of a given type with id, uuid, and name columns.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
# [DEF:SupersetClientGetAllResources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches all resources of a given type with id, uuid, and name columns.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_all_resources(
self, resource_type: str, since_dttm: Optional["datetime"] = None
) -> List[Dict]:
@@ -252,7 +275,10 @@ class SupersetClientBase:
}
config = column_map.get(resource_type)
if not config:
log.explore("Unknown resource type requested", error=f"Invalid resource type: {resource_type}", payload={"resource_type": resource_type})
app_logger.warning(
"[get_all_resources][Warning] Unknown resource type: %s",
resource_type,
)
return []
query = {"columns": config["columns"]}
@@ -272,13 +298,15 @@ class SupersetClientBase:
endpoint=config["endpoint"],
pagination_options={"base_query": validated, "results_field": "result"},
)
log.reflect(
"Resources fetched",
payload={"resource_type": resource_type, "count": len(data)},
app_logger.info(
"[get_all_resources][Exit] Fetched %d %s resources.",
len(data),
resource_type,
)
return data
# #endregion SupersetClientGetAllResources
# [/DEF:SupersetClientGetAllResources:Function]
# #endregion SupersetClientBase
# #endregion SupersetClientBase

28
backend/src/core/superset_client/_charts.py
View File

@@ -1,6 +1,6 @@
# #region SupersetChartsMixin [C:3] [TYPE Module] [SEMANTICS superset, charts, list, get, layout]
# @LAYER: Infra
# @BRIEF Chart domain mixin for SupersetClient — list, get, extract IDs from layout.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
import json
@@ -16,19 +16,21 @@ app_logger = cast(Any, app_logger)
# @BRIEF Mixin providing all chart-related Superset API operations.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
class SupersetChartsMixin:
# #region SupersetClientGetChart [C:3] [TYPE Function]
# @BRIEF Fetches a single chart by ID.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetChart:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single chart by ID.
# @RELATION: CALLS -> [APIClient]
def get_chart(self, chart_id: int) -> Dict:
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
return cast(Dict, response)
# #endregion SupersetClientGetChart
# [/DEF:SupersetClientGetChart:Function]
# #region SupersetClientGetCharts [C:3] [TYPE Function]
# @BRIEF Fetches all charts with pagination support.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
# [DEF:SupersetClientGetCharts:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches all charts with pagination support.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_charts(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_charts"):
validated_query = self._validate_query_params(query or {})
@@ -44,10 +46,11 @@ class SupersetChartsMixin:
)
return len(paginated_data), paginated_data
# #endregion SupersetClientGetCharts
# [/DEF:SupersetClientGetCharts:Function]
# #region SupersetClientExtractChartIdsFromLayout [C:1] [TYPE Function]
# @BRIEF Traverses dashboard layout metadata and extracts chart IDs from common keys.
# [DEF:SupersetClientExtractChartIdsFromLayout:Function]
# @COMPLEXITY: 1
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
def _extract_chart_ids_from_layout(
self, payload: Any
) -> set:
@@ -77,7 +80,8 @@ class SupersetChartsMixin:
walk(payload)
return found
# #endregion SupersetClientExtractChartIdsFromLayout
# [/DEF:SupersetClientExtractChartIdsFromLayout:Function]
# #endregion SupersetChartsMixin
# #endregion SupersetChartsMixin

118
backend/src/core/superset_client/_dashboards_crud.py
View File

@@ -1,6 +1,6 @@
# #region SupersetDashboardsCrudMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, crud, detail, export, import, delete]
# @LAYER: Infra
# @BRIEF Dashboard CRUD mixin for SupersetClient — detail, export, import, delete.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin]
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
@@ -11,10 +11,9 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, cast
from requests import Response
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
log = MarkerLogger("SupersetDashboardsCrud")
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsCrudMixin [C:3] [TYPE Class]
@@ -23,10 +22,11 @@ log = MarkerLogger("SupersetDashboardsCrud")
# @RELATION DEPENDS_ON -> [SupersetDashboardsFiltersMixin]
# @RELATION DEPENDS_ON -> [SupersetChartsMixin]
class SupersetDashboardsCrudMixin:
# #region SupersetClientGetDashboardDetail [C:3] [TYPE Function]
# @BRIEF Fetches detailed dashboard information including related charts and datasets.
# @RELATION CALLS -> [SupersetClientGetDashboard]
# @RELATION CALLS -> [SupersetClientGetChart]
# [DEF:SupersetClientGetDashboardDetail:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
# @RELATION: CALLS -> [SupersetClientGetDashboard]
# @RELATION: CALLS -> [SupersetClientGetChart]
def get_dashboard_detail(self, dashboard_ref: Union[int, str]) -> Dict:
with belief_scope(
"SupersetClient.get_dashboard_detail", f"ref={dashboard_ref}"
@@ -104,7 +104,9 @@ class SupersetDashboardsCrudMixin:
or "Chart",
})
except Exception as e:
log.explore("Failed to fetch dashboard charts", error=str(e), payload={"dashboard_ref": dashboard_ref})
app_logger.warning(
"[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", e,
)
try:
datasets_response = self.network.request(
@@ -142,7 +144,9 @@ class SupersetDashboardsCrudMixin:
"overview": fq_name,
})
except Exception as e:
log.explore("Failed to fetch dashboard datasets", error=str(e), payload={"dashboard_ref": dashboard_ref})
app_logger.warning(
"[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", e,
)
# Fallback: derive chart IDs from layout metadata
if not charts:
@@ -175,12 +179,9 @@ class SupersetDashboardsCrudMixin:
self._extract_chart_ids_from_layout(raw_json_metadata)
)
log.reason(
"Extracted fallback chart IDs from layout",
payload={
"chart_count": len(chart_ids_from_position),
"dashboard_ref": dashboard_ref,
},
app_logger.info(
"[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)",
len(chart_ids_from_position), dashboard_ref,
)
for chart_id in sorted(chart_ids_from_position):
@@ -198,7 +199,10 @@ class SupersetDashboardsCrudMixin:
or chart_data.get("viz_type") or "Chart",
})
except Exception as e:
log.explore("Failed to resolve fallback chart", error=str(e), payload={"chart_id": chart_id})
app_logger.warning(
"[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s",
chart_id, e,
)
# Backfill datasets from chart datasource IDs.
dataset_ids_from_charts = {
@@ -240,7 +244,10 @@ class SupersetDashboardsCrudMixin:
"overview": fq_name,
})
except Exception as e:
log.explore("Failed to resolve dataset from chart datasource", error=str(e), payload={"dataset_id": dataset_id})
app_logger.warning(
"[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s",
dataset_id, e,
)
unique_charts = {chart["id"]: chart for chart in charts}
unique_datasets = {dataset["id"]: dataset for dataset in datasets}
@@ -262,15 +269,18 @@ class SupersetDashboardsCrudMixin:
"dataset_count": len(unique_datasets),
}
# #endregion SupersetClientGetDashboardDetail
# [/DEF:SupersetClientGetDashboardDetail:Function]
# #region SupersetClientExportDashboard [C:3] [TYPE Function]
# @BRIEF Экспортирует дашборд в виде ZIP-архива.
# @SIDE_EFFECT Performs network I/O to download archive.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientExportDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Экспортирует дашборд в виде ZIP-архива.
# @SIDE_EFFECT: Performs network I/O to download archive.
# @RELATION: CALLS -> [APIClient]
def export_dashboard(self, dashboard_id: int) -> Tuple[bytes, str]:
with belief_scope("export_dashboard"):
log.reason("Exporting dashboard", payload={"dashboard_id": dashboard_id})
app_logger.info(
"[export_dashboard][Enter] Exporting dashboard %s.", dashboard_id
)
response = self.network.request(
method="GET",
endpoint="/dashboard/export/",
@@ -281,16 +291,20 @@ class SupersetDashboardsCrudMixin:
response = cast(Response, response)
self._validate_export_response(response, dashboard_id)
filename = self._resolve_export_filename(response, dashboard_id)
log.reflect("Dashboard exported", payload={"dashboard_id": dashboard_id, "filename": filename})
app_logger.info(
"[export_dashboard][Exit] Exported dashboard %s to %s.",
dashboard_id, filename,
)
return response.content, filename
# #endregion SupersetClientExportDashboard
# [/DEF:SupersetClientExportDashboard:Function]
# #region SupersetClientImportDashboard [C:3] [TYPE Function]
# @BRIEF Импортирует дашборд из ZIP-файла.
# @SIDE_EFFECT Performs network I/O to upload archive.
# @RELATION CALLS -> [SupersetClientDoImport]
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientImportDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Импортирует дашборд из ZIP-файла.
# @SIDE_EFFECT: Performs network I/O to upload archive.
# @RELATION: CALLS -> [SupersetClientDoImport]
# @RELATION: CALLS -> [APIClient]
def import_dashboard(
self,
file_name: Union[str, Path],
@@ -305,41 +319,55 @@ class SupersetDashboardsCrudMixin:
try:
return self._do_import(file_path)
except Exception as exc:
log.explore("First import attempt failed", error=str(exc), payload={"file_name": file_name})
app_logger.error(
"[import_dashboard][Failure] First import attempt failed: %s",
exc, exc_info=True,
)
if not self.delete_before_reimport:
raise
target_id = self._resolve_target_id_for_delete(dash_id, dash_slug)
if target_id is None:
log.explore("No ID available for delete-retry during import", error="Cannot delete-retry: neither dash_id nor dash_slug resolved", payload={"dash_id": dash_id, "dash_slug": dash_slug})
app_logger.error(
"[import_dashboard][Failure] No ID available for delete-retry."
)
raise
self.delete_dashboard(target_id)
log.reason(
"Deleted dashboard, retrying import",
payload={"target_id": target_id},
app_logger.info(
"[import_dashboard][State] Deleted dashboard ID %s, retrying import.",
target_id,
)
return self._do_import(file_path)
# #endregion SupersetClientImportDashboard
# [/DEF:SupersetClientImportDashboard:Function]
# #region SupersetClientDeleteDashboard [C:3] [TYPE Function]
# @BRIEF Удаляет дашборд по его ID или slug.
# @SIDE_EFFECT Deletes resource from upstream Superset environment.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientDeleteDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Удаляет дашборд по его ID или slug.
# @SIDE_EFFECT: Deletes resource from upstream Superset environment.
# @RELATION: CALLS -> [APIClient]
def delete_dashboard(self, dashboard_id: Union[int, str]) -> None:
with belief_scope("delete_dashboard"):
log.reason("Deleting dashboard", payload={"dashboard_id": dashboard_id})
app_logger.info(
"[delete_dashboard][Enter] Deleting dashboard %s.", dashboard_id
)
response = self.network.request(
method="DELETE", endpoint=f"/dashboard/{dashboard_id}"
)
response = cast(Dict, response)
if response.get("result", True) is not False:
log.reflect("Dashboard deleted", payload={"dashboard_id": dashboard_id})
app_logger.info(
"[delete_dashboard][Success] Dashboard %s deleted.", dashboard_id
)
else:
log.explore("Unexpected response while deleting dashboard", error=f"Unexpected API response: {response}", payload={"dashboard_id": dashboard_id, "response": response})
app_logger.warning(
"[delete_dashboard][Warning] Unexpected response while deleting %s: %s",
dashboard_id, response,
)
# #endregion SupersetClientDeleteDashboard
# [/DEF:SupersetClientDeleteDashboard:Function]
# #endregion SupersetDashboardsCrudMixin
# #endregion SupersetDashboardsCrudMixin

84
backend/src/core/superset_client/_dashboards_filters.py
View File

@@ -1,24 +1,24 @@
# #region SupersetDashboardsFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, filters, permalink, native-filters]
# @LAYER: Infra
# @BRIEF Dashboard native filter extraction mixin for SupersetClient.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
import json
from typing import Any, Dict, Optional, Union, cast
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
log = MarkerLogger("SupersetDashboardsFilters")
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsFiltersMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
class SupersetDashboardsFiltersMixin:
# #region SupersetClientGetDashboard [C:3] [TYPE Function]
# @BRIEF Fetches a single dashboard by ID or slug.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboard by ID or slug.
# @RELATION: CALLS -> [APIClient]
def get_dashboard(self, dashboard_ref: Union[int, str]) -> Dict:
with belief_scope("SupersetClient.get_dashboard", f"ref={dashboard_ref}"):
response = self.network.request(
@@ -26,11 +26,12 @@ class SupersetDashboardsFiltersMixin:
)
return cast(Dict, response)
# #endregion SupersetClientGetDashboard
# [/DEF:SupersetClientGetDashboard:Function]
# #region SupersetClientGetDashboardPermalinkState [C:2] [TYPE Function]
# @BRIEF Fetches stored dashboard permalink state by permalink key.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDashboardPermalinkState:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetches stored dashboard permalink state by permalink key.
# @RELATION: CALLS -> [APIClient]
def get_dashboard_permalink_state(self, permalink_key: str) -> Dict:
with belief_scope(
"SupersetClient.get_dashboard_permalink_state", f"key={permalink_key}"
@@ -40,11 +41,12 @@ class SupersetDashboardsFiltersMixin:
)
return cast(Dict, response)
# #endregion SupersetClientGetDashboardPermalinkState
# [/DEF:SupersetClientGetDashboardPermalinkState:Function]
# #region SupersetClientGetNativeFilterState [C:2] [TYPE Function]
# @BRIEF Fetches stored native filter state by filter state key.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetNativeFilterState:Function]
# @COMPLEXITY: 2
# @PURPOSE: Fetches stored native filter state by filter state key.
# @RELATION: CALLS -> [APIClient]
def get_native_filter_state(
self, dashboard_id: Union[int, str], filter_state_key: str
) -> Dict:
@@ -58,11 +60,12 @@ class SupersetDashboardsFiltersMixin:
)
return cast(Dict, response)
# #endregion SupersetClientGetNativeFilterState
# [/DEF:SupersetClientGetNativeFilterState:Function]
# #region SupersetClientExtractNativeFiltersFromPermalink [C:3] [TYPE Function]
# @BRIEF Extract native filters dataMask from a permalink key.
# @RELATION CALLS -> [SupersetClientGetDashboardPermalinkState]
# [DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters dataMask from a permalink key.
# @RELATION: CALLS -> [SupersetClientGetDashboardPermalinkState]
def extract_native_filters_from_permalink(self, permalink_key: str) -> Dict:
with belief_scope(
"SupersetClient.extract_native_filters_from_permalink",
@@ -92,11 +95,12 @@ class SupersetDashboardsFiltersMixin:
"permalink_key": permalink_key,
}
# #endregion SupersetClientExtractNativeFiltersFromPermalink
# [/DEF:SupersetClientExtractNativeFiltersFromPermalink:Function]
# #region SupersetClientExtractNativeFiltersFromKey [C:3] [TYPE Function]
# @BRIEF Extract native filters from a native_filters_key URL parameter.
# @RELATION CALLS -> [SupersetClientGetNativeFilterState]
# [DEF:SupersetClientExtractNativeFiltersFromKey:Function]
# @COMPLEXITY: 3
# @PURPOSE: Extract native filters from a native_filters_key URL parameter.
# @RELATION: CALLS -> [SupersetClientGetNativeFilterState]
def extract_native_filters_from_key(
self, dashboard_id: Union[int, str], filter_state_key: str
) -> Dict:
@@ -115,7 +119,10 @@ class SupersetDashboardsFiltersMixin:
try:
parsed_value = json.loads(value)
except json.JSONDecodeError as e:
log.explore("Failed to parse filter state JSON", error=str(e))
app_logger.warning(
"[extract_native_filters_from_key][Warning] Failed to parse filter state JSON: %s",
e,
)
parsed_value = {}
elif isinstance(value, dict):
parsed_value = value
@@ -147,12 +154,13 @@ class SupersetDashboardsFiltersMixin:
"filter_state_key": filter_state_key,
}
# #endregion SupersetClientExtractNativeFiltersFromKey
# [/DEF:SupersetClientExtractNativeFiltersFromKey:Function]
# #region SupersetClientParseDashboardUrlForFilters [C:3] [TYPE Function]
# @BRIEF Parse a Superset dashboard URL and extract native filter state if present.
# @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]
# @RELATION CALLS -> [SupersetClientExtractNativeFiltersFromKey]
# [DEF:SupersetClientParseDashboardUrlForFilters:Function]
# @COMPLEXITY: 3
# @PURPOSE: Parse a Superset dashboard URL and extract native filter state if present.
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromPermalink]
# @RELATION: CALLS -> [SupersetClientExtractNativeFiltersFromKey]
def parse_dashboard_url_for_filters(self, url: str) -> Dict:
with belief_scope(
"SupersetClient.parse_dashboard_url_for_filters", f"url={url}"
@@ -215,7 +223,11 @@ class SupersetDashboardsFiltersMixin:
if raw_id is not None:
resolved_id = int(raw_id)
except Exception as e:
log.explore("Failed to resolve dashboard slug to ID", error=str(e), payload={"slug": dashboard_ref})
app_logger.warning(
"[parse_dashboard_url_for_filters][Warning] Failed to resolve dashboard slug '%s' to ID: %s",
dashboard_ref,
e,
)
if resolved_id is not None:
filter_data = self.extract_native_filters_from_key(
@@ -226,7 +238,9 @@ class SupersetDashboardsFiltersMixin:
result["filters"] = filter_data
return result
else:
log.explore("Could not resolve dashboard_id from URL for native_filters_key", error="Dashboard ID could not be resolved from URL", payload={"dashboard_ref": dashboard_ref})
app_logger.warning(
"[parse_dashboard_url_for_filters][Warning] Could not resolve dashboard_id from URL for native_filters_key"
)
# Check for native_filters in query params (direct filter values)
native_filters = query_params.get("native_filters", [None])[0]
@@ -237,11 +251,15 @@ class SupersetDashboardsFiltersMixin:
result["filters"] = {"dataMask": parsed_filters}
return result
except json.JSONDecodeError as e:
log.explore("Failed to parse native_filters JSON", error=str(e))
app_logger.warning(
"[parse_dashboard_url_for_filters][Warning] Failed to parse native_filters JSON: %s",
e,
)
return result
# #endregion SupersetClientParseDashboardUrlForFilters
# [/DEF:SupersetClientParseDashboardUrlForFilters:Function]
# #endregion SupersetDashboardsFiltersMixin
# #endregion SupersetDashboardsFiltersMixin

80
backend/src/core/superset_client/_dashboards_list.py
View File

@@ -1,16 +1,15 @@
# #region SupersetDashboardsListMixin [C:3] [TYPE Module] [SEMANTICS superset, dashboards, list, pagination, summary]
# @LAYER: Infra
# @BRIEF Dashboard listing mixin for SupersetClient — paginated list, summary projection.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin]
import json
from typing import Any, Dict, List, Optional, Tuple, cast
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
log = MarkerLogger("SupersetDashboardsList")
app_logger = cast(Any, app_logger)
# #region SupersetDashboardsListMixin [C:3] [TYPE Class]
@@ -18,12 +17,13 @@ log = MarkerLogger("SupersetDashboardsList")
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetUserProjectionMixin]
class SupersetDashboardsListMixin:
# #region SupersetClientGetDashboards [C:3] [TYPE Function]
# @BRIEF Получает полный список дашбордов, автоматически обрабатывая пагинацию.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
# [DEF:SupersetClientGetDashboards:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список дашбордов, автоматически обрабатывая пагинацию.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_dashboards(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_dashboards"):
log.reason("Fetching dashboards", payload={"has_query": query is not None})
app_logger.info("[get_dashboards][Enter] Fetching dashboards.")
validated_query = self._validate_query_params(query or {})
if "columns" not in validated_query:
validated_query["columns"] = [
@@ -39,14 +39,15 @@ class SupersetDashboardsListMixin:
},
)
total_count = len(paginated_data)
log.reflect("Dashboards fetched", payload={"total_count": total_count})
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
return total_count, paginated_data
# #endregion SupersetClientGetDashboards
# [/DEF:SupersetClientGetDashboards:Function]
# #region SupersetClientGetDashboardsPage [C:3] [TYPE Function]
# @BRIEF Fetches a single dashboards page from Superset without iterating all pages.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDashboardsPage:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches a single dashboards page from Superset without iterating all pages.
# @RELATION: CALLS -> [APIClient]
def get_dashboards_page(
self, query: Optional[Dict] = None
) -> Tuple[int, List[Dict]]:
@@ -70,11 +71,12 @@ class SupersetDashboardsListMixin:
total_count = response_json.get("count", len(result))
return total_count, result
# #endregion SupersetClientGetDashboardsPage
# [/DEF:SupersetClientGetDashboardsPage:Function]
# #region SupersetClientGetDashboardsSummary [C:3] [TYPE Function]
# @BRIEF Fetches dashboard metadata optimized for the grid.
# @RELATION CALLS -> [SupersetClientGetDashboards]
# [DEF:SupersetClientGetDashboardsSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches dashboard metadata optimized for the grid.
# @RELATION: CALLS -> [SupersetClientGetDashboards]
def get_dashboards_summary(self, require_slug: bool = False) -> List[Dict]:
with belief_scope("SupersetClient.get_dashboards_summary"):
query: Dict[str, Any] = {}
@@ -122,37 +124,28 @@ class SupersetDashboardsListMixin:
})
if index < max_debug_samples:
log.reflect(
"Dashboard actor projection sample",
payload={
"env": getattr(self.env, "id", None),
"dashboard_id": dash.get("id"),
"raw_owners": raw_owners,
"raw_owner_usernames": raw_owner_usernames,
"raw_created_by": raw_created_by,
"raw_changed_by": raw_changed_by,
"raw_changed_by_name": raw_changed_by_name,
"projected_owners": owners,
"projected_created_by": projected_created_by,
"projected_modified_by": projected_modified_by,
},
app_logger.reflect(
"[REFLECT] Dashboard actor projection sample "
f"(env={getattr(self.env, 'id', None)}, dashboard_id={dash.get('id')}, "
f"raw_owners={raw_owners!r}, raw_owner_usernames={raw_owner_usernames!r}, "
f"raw_created_by={raw_created_by!r}, raw_changed_by={raw_changed_by!r}, "
f"raw_changed_by_name={raw_changed_by_name!r}, projected_owners={owners!r}, "
f"projected_created_by={projected_created_by!r}, projected_modified_by={projected_modified_by!r})"
)
log.reflect(
"Dashboard actor projection summary",
payload={
"env": getattr(self.env, "id", None),
"dashboards": len(result),
"sampled": min(len(result), max_debug_samples),
},
app_logger.reflect(
"[REFLECT] Dashboard actor projection summary "
f"(env={getattr(self.env, 'id', None)}, dashboards={len(result)}, "
f"sampled={min(len(result), max_debug_samples)})"
)
return result
# #endregion SupersetClientGetDashboardsSummary
# [/DEF:SupersetClientGetDashboardsSummary:Function]
# #region SupersetClientGetDashboardsSummaryPage [C:3] [TYPE Function]
# @BRIEF Fetches one page of dashboard metadata optimized for the grid.
# @RELATION CALLS -> [SupersetClientGetDashboardsPage]
# [DEF:SupersetClientGetDashboardsSummaryPage:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches one page of dashboard metadata optimized for the grid.
# @RELATION: CALLS -> [SupersetClientGetDashboardsPage]
def get_dashboards_summary_page(
self,
page: int,
@@ -204,7 +197,8 @@ class SupersetDashboardsListMixin:
return total_count, result
# #endregion SupersetClientGetDashboardsSummaryPage
# [/DEF:SupersetClientGetDashboardsSummaryPage:Function]
# #endregion SupersetDashboardsListMixin
# #endregion SupersetDashboardsListMixin

54
backend/src/core/superset_client/_databases.py
View File

@@ -1,26 +1,26 @@
# #region SupersetDatabasesMixin [C:3] [TYPE Module] [SEMANTICS superset, databases, list, get, summary, uuid]
# @LAYER: Infra
# @BRIEF Database domain mixin for SupersetClient — list, get, summary, by_uuid.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
from typing import Any, Dict, List, Optional, Tuple, cast
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
log = MarkerLogger("SupersetDatabases")
app_logger = cast(Any, app_logger)
# #region SupersetDatabasesMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing all database-related Superset API operations.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
class SupersetDatabasesMixin:
# #region SupersetClientGetDatabases [C:3] [TYPE Function]
# @BRIEF Получает полный список баз данных.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
# [DEF:SupersetClientGetDatabases:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список баз данных.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_databases(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_databases"):
log.reason("Fetching databases", payload={"has_query": query is not None})
app_logger.info("[get_databases][Enter] Fetching databases.")
validated_query = self._validate_query_params(query or {})
if "columns" not in validated_query:
validated_query["columns"] = []
@@ -33,32 +33,34 @@ class SupersetDatabasesMixin:
},
)
total_count = len(paginated_data)
log.reflect("Databases fetched", payload={"total_count": total_count})
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
return total_count, paginated_data
# #endregion SupersetClientGetDatabases
# [/DEF:SupersetClientGetDatabases:Function]
# #region SupersetClientGetDatabase [C:3] [TYPE Function]
# @BRIEF Получает информацию о конкретной базе данных по её ID.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDatabase:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретной базе данных по её ID.
# @RELATION: CALLS -> [APIClient]
def get_database(self, database_id: int) -> Dict:
with belief_scope("get_database"):
log.reason("Fetching database", payload={"database_id": database_id})
app_logger.info("[get_database][Enter] Fetching database %s.", database_id)
response = self.network.request(
method="GET", endpoint=f"/database/{database_id}"
)
response = cast(Dict, response)
log.reflect("Database fetched", payload={"database_id": database_id})
app_logger.info("[get_database][Exit] Got database %s.", database_id)
return response
# #endregion SupersetClientGetDatabase
# [/DEF:SupersetClientGetDatabase:Function]
# #region SupersetClientGetDatabasesSummary [C:3] [TYPE Function]
# @BRIEF Fetch a summary of databases including uuid, name, and engine.
# @RELATION CALLS -> [SupersetClientGetDatabases]
# [DEF:SupersetClientGetDatabasesSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetch a summary of databases including uuid, name, and engine.
# @RELATION: CALLS -> [SupersetClientGetDatabases]
def get_databases_summary(self) -> List[Dict]:
with belief_scope("SupersetClient.get_databases_summary"):
query = {"columns": ["id", "uuid", "database_name", "backend"]}
query = {"columns": ["uuid", "database_name", "backend"]}
_, databases = self.get_databases(query=query)
# Map 'backend' to 'engine' for consistency with contracts
@@ -67,18 +69,20 @@ class SupersetDatabasesMixin:
return databases
# #endregion SupersetClientGetDatabasesSummary
# [/DEF:SupersetClientGetDatabasesSummary:Function]
# #region SupersetClientGetDatabaseByUuid [C:3] [TYPE Function]
# @BRIEF Find a database by its UUID.
# @RELATION CALLS -> [SupersetClientGetDatabases]
# [DEF:SupersetClientGetDatabaseByUuid:Function]
# @COMPLEXITY: 3
# @PURPOSE: Find a database by its UUID.
# @RELATION: CALLS -> [SupersetClientGetDatabases]
def get_database_by_uuid(self, db_uuid: str) -> Optional[Dict]:
with belief_scope("SupersetClient.get_database_by_uuid", f"uuid={db_uuid}"):
query = {"filters": [{"col": "uuid", "op": "eq", "value": db_uuid}]}
_, databases = self.get_databases(query=query)
return databases[0] if databases else None
# #endregion SupersetClientGetDatabaseByUuid
# [/DEF:SupersetClientGetDatabaseByUuid:Function]
# #endregion SupersetDatabasesMixin
# #endregion SupersetDatabasesMixin

82
backend/src/core/superset_client/_datasets.py
View File

@@ -1,27 +1,27 @@
# #region SupersetDatasetsMixin [C:3] [TYPE Module] [SEMANTICS superset, datasets, list, get, detail, update]
# @LAYER: Infra
# @BRIEF Dataset domain mixin for SupersetClient — list, get, detail, update.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
import json
from typing import Any, Dict, List, Optional, Tuple, cast
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
log = MarkerLogger("SupersetDatasets")
app_logger = cast(Any, app_logger)
# #region SupersetDatasetsMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing basic dataset CRUD operations (list, get, detail, update).
# @RELATION DEPENDS_ON -> [SupersetClientBase]
class SupersetDatasetsMixin:
# #region SupersetClientGetDatasets [C:3] [TYPE Function]
# @BRIEF Получает полный список датасетов, автоматически обрабатывая пагинацию.
# @RELATION CALLS -> [SupersetClientFetchAllPages]
# [DEF:SupersetClientGetDatasets:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает полный список датасетов, автоматически обрабатывая пагинацию.
# @RELATION: CALLS -> [SupersetClientFetchAllPages]
def get_datasets(self, query: Optional[Dict] = None) -> Tuple[int, List[Dict]]:
with belief_scope("get_datasets"):
log.reason("Fetching datasets")
app_logger.info("[get_datasets][Enter] Fetching datasets.")
validated_query = self._validate_query_params(query)
paginated_data = self._fetch_all_pages(
@@ -32,14 +32,15 @@ class SupersetDatasetsMixin:
},
)
total_count = len(paginated_data)
log.reflect("Datasets fetched", payload={"total_count": total_count})
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
return total_count, paginated_data
# #endregion SupersetClientGetDatasets
# [/DEF:SupersetClientGetDatasets:Function]
# #region SupersetClientGetDatasetsSummary [C:3] [TYPE Function]
# @BRIEF Fetches dataset metadata optimized for the Dataset Hub grid.
# @RELATION CALLS -> [SupersetClientGetDatasets]
# [DEF:SupersetClientGetDatasetsSummary:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches dataset metadata optimized for the Dataset Hub grid.
# @RELATION: CALLS -> [SupersetClientGetDatasets]
def get_datasets_summary(self) -> List[Dict]:
with belief_scope("SupersetClient.get_datasets_summary"):
query = {"columns": ["id", "table_name", "schema", "database"]}
@@ -59,12 +60,13 @@ class SupersetDatasetsMixin:
)
return result
# #endregion SupersetClientGetDatasetsSummary
# [/DEF:SupersetClientGetDatasetsSummary:Function]
# #region SupersetClientGetDatasetDetail [C:3] [TYPE Function]
# @BRIEF Fetches detailed dataset information including columns and linked dashboards.
# @RELATION CALLS -> [SupersetClientGetDataset]
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDatasetDetail:Function]
# @COMPLEXITY: 3
# @PURPOSE: Fetches detailed dataset information including columns and linked dashboards.
# @RELATION: CALLS -> [SupersetClientGetDataset]
# @RELATION: CALLS -> [APIClient]
def get_dataset_detail(self, dataset_id: int) -> Dict:
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
@@ -139,7 +141,9 @@ class SupersetDatasetsMixin:
{"id": dash_id, "title": f"Dashboard {dash_id}", "slug": None}
)
except Exception as e:
log.explore("Failed to fetch related dashboards", error=str(e), payload={"dataset_id": dataset_id})
app_logger.warning(
f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}"
)
linked_dashboards = []
database_obj = dataset.get("database")
@@ -164,40 +168,37 @@ class SupersetDatasetsMixin:
"changed_on": dataset.get("changed_on"),
}
log.reflect(
"Dataset detail fetched",
payload={
"dataset_id": dataset_id,
"column_count": len(column_info),
"linked_dashboard_count": len(linked_dashboards),
},
app_logger.info(
f"[get_dataset_detail][Exit] Got dataset {dataset_id} with {len(column_info)} columns and {len(linked_dashboards)} linked dashboards"
)
return result
# #endregion SupersetClientGetDatasetDetail
# [/DEF:SupersetClientGetDatasetDetail:Function]
# #region SupersetClientGetDataset [C:3] [TYPE Function]
# @BRIEF Получает информацию о конкретном датасете по его ID.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientGetDataset:Function]
# @COMPLEXITY: 3
# @PURPOSE: Получает информацию о конкретном датасете по его ID.
# @RELATION: CALLS -> [APIClient]
def get_dataset(self, dataset_id: int) -> Dict:
with belief_scope("SupersetClient.get_dataset", f"id={dataset_id}"):
log.reason("Fetching dataset", payload={"dataset_id": dataset_id})
app_logger.info("[get_dataset][Enter] Fetching dataset %s.", dataset_id)
response = self.network.request(
method="GET", endpoint=f"/dataset/{dataset_id}"
)
response = cast(Dict, response)
log.reflect("Dataset fetched", payload={"dataset_id": dataset_id})
app_logger.info("[get_dataset][Exit] Got dataset %s.", dataset_id)
return response
# #endregion SupersetClientGetDataset
# [/DEF:SupersetClientGetDataset:Function]
# #region SupersetClientUpdateDataset [C:3] [TYPE Function]
# @BRIEF Обновляет данные датасета по его ID.
# @SIDE_EFFECT Modifies resource in upstream Superset environment.
# @RELATION CALLS -> [APIClient]
# [DEF:SupersetClientUpdateDataset:Function]
# @COMPLEXITY: 3
# @PURPOSE: Обновляет данные датасета по его ID.
# @SIDE_EFFECT: Modifies resource in upstream Superset environment.
# @RELATION: CALLS -> [APIClient]
def update_dataset(self, dataset_id: int, data: Dict) -> Dict:
with belief_scope("SupersetClient.update_dataset", f"id={dataset_id}"):
log.reason("Updating dataset", payload={"dataset_id": dataset_id})
app_logger.info("[update_dataset][Enter] Updating dataset %s.", dataset_id)
response = self.network.request(
method="PUT",
endpoint=f"/dataset/{dataset_id}",
@@ -205,10 +206,11 @@ class SupersetDatasetsMixin:
headers={"Content-Type": "application/json"},
)
response = cast(Dict, response)
log.reflect("Dataset updated", payload={"dataset_id": dataset_id})
app_logger.info("[update_dataset][Exit] Updated dataset %s.", dataset_id)
return response
# #endregion SupersetClientUpdateDataset
# [/DEF:SupersetClientUpdateDataset:Function]
# #endregion SupersetDatasetsMixin
# #endregion SupersetDatasetsMixin

58
backend/src/core/superset_client/_datasets_preview.py
View File

@@ -1,6 +1,6 @@
# #region SupersetDatasetsPreviewMixin [C:4] [TYPE Module] [SEMANTICS superset, datasets, preview, sql, compilation, filters]
# @LAYER: Infra
# @BRIEF Dataset preview compilation mixin for SupersetClient — build query context, compile SQL.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetDatasetsMixin]
@@ -8,20 +8,17 @@ import json
from copy import deepcopy
from typing import Any, Dict, List, Optional
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
from ..utils.network import SupersetAPIError
log = MarkerLogger("SupersetDatasetsPreview")
# #region SupersetDatasetsPreviewMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing dataset preview compilation and query context building.
class SupersetDatasetsPreviewMixin:
# #region SupersetClientCompileDatasetPreview [C:4] [TYPE Function]
# @BRIEF Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.
# [DEF:SupersetClientCompileDatasetPreview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Compile dataset preview SQL through the strongest supported Superset preview endpoint family and return normalized SQL output.
def compile_dataset_preview(self, dataset_id: int, template_params: Optional[Dict[str, Any]] = None, effective_filters: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
with belief_scope('SupersetClientCompileDatasetPreview'):
log.reason('Compiling dataset preview SQL', payload={"dataset_id": dataset_id})
app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientCompileDatasetPreview')
dataset_response = self.get_dataset(dataset_id)
dataset_record = dataset_response.get('result', dataset_response) if isinstance(dataset_response, dict) else {}
query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params or {}, effective_filters=effective_filters or [])
@@ -49,7 +46,7 @@ class SupersetDatasetsPreviewMixin:
elif isinstance(request_body, dict):
request_payload_keys = sorted(request_body.keys())
strategy_diagnostics = {'endpoint': endpoint_path, 'endpoint_kind': endpoint_kind, 'request_transport': request_transport, 'contains_root_datasource': endpoint_kind == 'v1_chart_data' and 'datasource' in query_context, 'contains_form_datasource': endpoint_kind.startswith('legacy_') and 'datasource' in legacy_form_data, 'contains_query_object_datasource': bool(query_context.get('queries')) and isinstance(query_context['queries'][0], dict) and ('datasource' in query_context['queries'][0]), 'request_param_keys': request_param_keys, 'request_payload_keys': request_payload_keys}
log.reason('Attempting dataset preview compilation strategy', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})
app_logger.reason('Attempting Superset dataset preview compilation strategy', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'request_params': request_params, 'request_payload': request_body, 'legacy_form_data': legacy_form_data if endpoint_kind.startswith('legacy_') else None, 'query_context': query_context if endpoint_kind == 'v1_chart_data' else None, 'template_param_count': len(template_params or {}), 'filter_count': len(effective_filters or [])})
try:
response = self.network.request(method='POST', endpoint=endpoint_path, params=request_params or None, data=request_body, headers=request_headers or None)
normalized = self._extract_compiled_sql_from_preview_response(response)
@@ -59,20 +56,22 @@ class SupersetDatasetsPreviewMixin:
normalized['endpoint_kind'] = endpoint_kind
normalized['dataset_id'] = dataset_id
normalized['strategy_attempts'] = strategy_attempts + [{**strategy_diagnostics, 'success': True}]
log.reflect('Dataset preview compilation returned normalized SQL payload', payload={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})
app_logger.reflect('Dataset preview compilation returned normalized SQL payload', extra={'dataset_id': dataset_id, **strategy_diagnostics, 'success': True, 'compiled_sql_length': len(str(normalized.get('compiled_sql') or '')), 'response_diagnostics': normalized.get('response_diagnostics')})
app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientCompileDatasetPreview')
return normalized
except Exception as exc:
failure_diagnostics = {**strategy_diagnostics, 'success': False, 'error': str(exc)}
strategy_attempts.append(failure_diagnostics)
log.explore('Dataset preview compilation strategy failed', payload={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body}, error=str(exc))
app_logger.explore('Superset dataset preview compilation strategy failed', extra={'dataset_id': dataset_id, **failure_diagnostics, 'request_params': request_params, 'request_payload': request_body})
raise SupersetAPIError(f'Superset preview compilation failed for all known strategies (attempts={strategy_attempts!r})')
# #endregion SupersetClientCompileDatasetPreview
# [/DEF:SupersetClientCompileDatasetPreview:Function]
# #region SupersetClientBuildDatasetPreviewLegacyFormData [C:4] [TYPE Function]
# @BRIEF Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.
# [DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build browser-style legacy form_data payload for Superset preview endpoints inferred from observed deployment traffic.
def build_dataset_preview_legacy_form_data(self, dataset_id: int, dataset_record: Dict[str, Any], template_params: Dict[str, Any], effective_filters: List[Dict[str, Any]]) -> Dict[str, Any]:
with belief_scope('SupersetClientBuildDatasetPreviewLegacyFormData'):
log.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
app_logger.reason('Belief protocol reasoning checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
query_context = self.build_dataset_preview_query_context(dataset_id=dataset_id, dataset_record=dataset_record, template_params=template_params, effective_filters=effective_filters)
query_object = deepcopy(query_context.get('queries', [{}])[0] if query_context.get('queries') else {})
legacy_form_data = deepcopy(query_context.get('form_data', {}))
@@ -94,13 +93,14 @@ class SupersetDatasetsPreviewMixin:
time_range = query_object.get('time_range')
if time_range:
legacy_form_data['time_range'] = time_range
log.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', payload={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})
log.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
app_logger.reflect('Built Superset legacy preview form_data payload from browser-observed request shape', extra={'dataset_id': dataset_id, 'legacy_endpoint_inference': 'POST /explore_json/form_data?form_data=... primary, POST /data?form_data=... fallback, based on observed browser traffic', 'contains_form_datasource': 'datasource' in legacy_form_data, 'legacy_form_data_keys': sorted(legacy_form_data.keys()), 'legacy_extra_filters': legacy_form_data.get('extra_filters', []), 'legacy_extra_form_data': legacy_form_data.get('extra_form_data', {})})
app_logger.reflect('Belief protocol postcondition checkpoint for SupersetClientBuildDatasetPreviewLegacyFormData')
return legacy_form_data
# #endregion SupersetClientBuildDatasetPreviewLegacyFormData
# [/DEF:SupersetClientBuildDatasetPreviewLegacyFormData:Function]
# #region SupersetClientBuildDatasetPreviewQueryContext [C:4] [TYPE Function]
# @BRIEF Build a reduced-scope chart-data query context for deterministic dataset preview compilation.
# [DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build a reduced-scope chart-data query context for deterministic dataset preview compilation.
def build_dataset_preview_query_context(
self,
dataset_id: int,
@@ -109,9 +109,9 @@ class SupersetDatasetsPreviewMixin:
effective_filters: List[Dict[str, Any]],
) -> Dict[str, Any]:
with belief_scope("SupersetClientBuildDatasetPreviewQueryContext"):
log.reason(
app_logger.reason(
"Building Superset dataset preview query context",
payload={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])},
extra={"dataset_id": dataset_id, "filter_count": len(effective_filters or [])},
)
normalized_template_params = deepcopy(template_params or {})
normalized_filter_payload = (
@@ -148,7 +148,10 @@ class SupersetDatasetsPreviewMixin:
for key, value in parsed_dataset_template_params.items():
normalized_template_params.setdefault(str(key), value)
except json.JSONDecodeError:
log.explore("Dataset template_params could not be parsed while building preview query context", error="Failed to parse template_params JSON", payload={"dataset_id": dataset_id})
app_logger.explore(
"Dataset template_params could not be parsed while building preview query context",
extra={"dataset_id": dataset_id},
)
extra_form_data: Dict[str, Any] = deepcopy(normalized_extra_form_data)
if normalized_filters:
@@ -204,9 +207,9 @@ class SupersetDatasetsPreviewMixin:
"result_type": result_type,
"force": True,
}
log.reflect(
app_logger.reflect(
"Built Superset dataset preview query context",
payload={
extra={
"dataset_id": dataset_id,
"datasource": datasource_payload,
"normalized_effective_filters": normalized_filters,
@@ -219,6 +222,7 @@ class SupersetDatasetsPreviewMixin:
)
return payload
# #endregion SupersetClientBuildDatasetPreviewQueryContext
# [/DEF:SupersetClientBuildDatasetPreviewQueryContext:Function]
# #endregion SupersetDatasetsPreviewMixin
# #endregion SupersetDatasetsPreviewMixin

25
backend/src/core/superset_client/_datasets_preview_filters.py
View File

@@ -1,25 +1,24 @@
# #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Module] [SEMANTICS superset, datasets, preview, filters, normalization, sql, extraction]
# @LAYER: Infra
# @BRIEF Filter normalization and SQL extraction helpers for dataset preview compilation.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin]
from copy import deepcopy
from typing import Any, Dict, List, Optional, Tuple
from ..cot_logger import MarkerLogger
from ..logger import belief_scope
from ..logger import logger as app_logger, belief_scope
from ..utils.network import SupersetAPIError
log = MarkerLogger("SupersetDatasetsPreviewFilters")
# #region SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
# @RELATION DEPENDS_ON -> [SupersetDatasetsPreviewMixin]
class SupersetDatasetsPreviewFiltersMixin:
# #region SupersetClientNormalizeEffectiveFiltersForQueryContext [C:3] [TYPE Function]
# @BRIEF Convert execution mappings into Superset chart-data filter objects.
# [DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# @COMPLEXITY: 3
# @PURPOSE: Convert execution mappings into Superset chart-data filter objects.
def _normalize_effective_filters_for_query_context(
self,
effective_filters: List[Dict[str, Any]],
@@ -98,9 +97,9 @@ class SupersetDatasetsPreviewFiltersMixin:
"outgoing_clauses": deepcopy(outgoing_clauses),
}
)
log.reason(
app_logger.reason(
"Normalized effective preview filter for Superset query context",
payload={
extra={
"filter_name": display_name,
"used_preserved_clauses": used_preserved_clauses,
"outgoing_clauses": outgoing_clauses,
@@ -118,10 +117,11 @@ class SupersetDatasetsPreviewFiltersMixin:
"diagnostics": diagnostics,
}
# #endregion SupersetClientNormalizeEffectiveFiltersForQueryContext
# [/DEF:SupersetClientNormalizeEffectiveFiltersForQueryContext:Function]
# #region SupersetClientExtractCompiledSqlFromPreviewResponse [C:3] [TYPE Function]
# @BRIEF Normalize compiled SQL from either chart-data or legacy form_data preview responses.
# [DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize compiled SQL from either chart-data or legacy form_data preview responses.
def _extract_compiled_sql_from_preview_response(
self, response: Any
) -> Dict[str, Any]:
@@ -191,7 +191,8 @@ class SupersetDatasetsPreviewFiltersMixin:
f"(diagnostics={response_diagnostics!r})"
)
# #endregion SupersetClientExtractCompiledSqlFromPreviewResponse
# [/DEF:SupersetClientExtractCompiledSqlFromPreviewResponse:Function]
# #endregion SupersetDatasetsPreviewFiltersMixin
# #endregion SupersetDatasetsPreviewFiltersMixin

24
backend/src/core/superset_client/_user_projection.py
View File

@@ -1,6 +1,6 @@
# #region SupersetUserProjection [C:2] [TYPE Module] [SEMANTICS superset, users, owners, projection, normalization]
# @LAYER: Infra
# @BRIEF User/owner payload normalization helpers for Superset client responses.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetClientBase]
from typing import Any, Dict, List, Optional, Union
@@ -10,8 +10,9 @@ from typing import Any, Dict, List, Optional, Union
# @BRIEF Mixin providing user/owner payload normalization for Superset API responses.
# @RELATION DEPENDS_ON -> [SupersetClientBase]
class SupersetUserProjectionMixin:
# #region SupersetClientExtractOwnerLabels [C:1] [TYPE Function]
# @BRIEF Normalize dashboard owners payload to stable display labels.
# [DEF:SupersetClientExtractOwnerLabels:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize dashboard owners payload to stable display labels.
def _extract_owner_labels(self, owners_payload: Any) -> List[str]:
if owners_payload is None:
return []
@@ -33,10 +34,11 @@ class SupersetUserProjectionMixin:
normalized.append(label)
return normalized
# #endregion SupersetClientExtractOwnerLabels
# [/DEF:SupersetClientExtractOwnerLabels:Function]
# #region SupersetClientExtractUserDisplay [C:1] [TYPE Function]
# @BRIEF Normalize user payload to a stable display name.
# [DEF:SupersetClientExtractUserDisplay:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize user payload to a stable display name.
def _extract_user_display(
self, preferred_value: Optional[str], user_payload: Optional[Dict]
) -> Optional[str]:
@@ -63,10 +65,11 @@ class SupersetUserProjectionMixin:
return email
return None
# #endregion SupersetClientExtractUserDisplay
# [/DEF:SupersetClientExtractUserDisplay:Function]
# #region SupersetClientSanitizeUserText [C:1] [TYPE Function]
# @BRIEF Convert scalar value to non-empty user-facing text.
# [DEF:SupersetClientSanitizeUserText:Function]
# @COMPLEXITY: 1
# @PURPOSE: Convert scalar value to non-empty user-facing text.
def _sanitize_user_text(self, value: Optional[Union[str, int]]) -> Optional[str]:
if value is None:
return None
@@ -75,7 +78,8 @@ class SupersetUserProjectionMixin:
return None
return normalized
# #endregion SupersetClientSanitizeUserText
# [/DEF:SupersetClientSanitizeUserText:Function]
# #endregion SupersetUserProjectionMixin
# #endregion SupersetUserProjection

56
backend/src/core/superset_profile_lookup.py
View File

@@ -1,20 +1,18 @@
# #region SupersetProfileLookup [C:3] [TYPE Module] [SEMANTICS superset, users, lookup, profile, pagination, normalization]
#
# @BRIEF Provides environment-scoped Superset account lookup adapter with stable normalized output.
# @LAYER Core
# @INVARIANT Adapter never leaks raw upstream payload shape to API consumers.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [SupersetAccountLookupAdapter]
#
#
# @INVARIANT: Adapter never leaks raw upstream payload shape to API consumers.
# [SECTION: IMPORTS]
import json
from typing import Any, Dict, List, Optional
from .logger import logger, belief_scope
from .utils.network import APIClient, AuthenticationError, SupersetAPIError
# [/SECTION]
# #region SupersetAccountLookupAdapter [C:3] [TYPE Class]
@@ -22,20 +20,20 @@ from .utils.network import APIClient, AuthenticationError, SupersetAPIError
# @RELATION DEPENDS_ON -> [APIClient]
# @RELATION DEPENDS_ON -> [SupersetProfileLookup]
class SupersetAccountLookupAdapter:
# #region __init__ [TYPE Function]
# @BRIEF Initializes lookup adapter with authenticated API client and environment context.
# @PRE network_client supports request(method, endpoint, params=...).
# @POST Adapter is ready to perform users lookup requests.
# [DEF:__init__:Function]
# @PURPOSE: Initializes lookup adapter with authenticated API client and environment context.
# @PRE: network_client supports request(method, endpoint, params=...).
# @POST: Adapter is ready to perform users lookup requests.
def __init__(self, network_client: APIClient, environment_id: str):
self.network_client = network_client
self.environment_id = str(environment_id or "")
# #endregion __init__
# [/DEF:__init__:Function]
# #region get_users_page [TYPE Function]
# @BRIEF Fetch one users page from Superset with passthrough search/sort parameters.
# @PRE page_index >= 0 and page_size >= 1.
# @POST Returns deterministic payload with normalized items and total count.
# @RETURN Dict[str, Any]
# [DEF:get_users_page:Function]
# @PURPOSE: Fetch one users page from Superset with passthrough search/sort parameters.
# @PRE: page_index >= 0 and page_size >= 1.
# @POST: Returns deterministic payload with normalized items and total count.
# @RETURN: Dict[str, Any]
def get_users_page(
self,
search: Optional[str] = None,
@@ -133,13 +131,13 @@ class SupersetAccountLookupAdapter:
)
raise selected_error
raise SupersetAPIError("Superset users lookup failed without explicit error")
# #endregion get_users_page
# [/DEF:get_users_page:Function]
# #region _normalize_lookup_payload [TYPE Function]
# @BRIEF Convert Superset users response variants into stable candidates payload.
# @PRE response can be dict/list in any supported upstream shape.
# @POST Output contains canonical keys: status, environment_id, page_index, page_size, total, items.
# @RETURN Dict[str, Any]
# [DEF:_normalize_lookup_payload:Function]
# @PURPOSE: Convert Superset users response variants into stable candidates payload.
# @PRE: response can be dict/list in any supported upstream shape.
# @POST: Output contains canonical keys: status, environment_id, page_index, page_size, total, items.
# @RETURN: Dict[str, Any]
def _normalize_lookup_payload(
self,
response: Any,
@@ -194,13 +192,13 @@ class SupersetAccountLookupAdapter:
"total": max(int(total), len(normalized_items)),
"items": normalized_items,
}
# #endregion _normalize_lookup_payload
# [/DEF:_normalize_lookup_payload:Function]
# #region normalize_user_payload [TYPE Function]
# @BRIEF Project raw Superset user object to canonical candidate shape.
# @PRE raw_user may have heterogenous key names between Superset versions.
# @POST Returns normalized candidate keys (environment_id, username, display_name, email, is_active).
# @RETURN Dict[str, Any]
# [DEF:normalize_user_payload:Function]
# @PURPOSE: Project raw Superset user object to canonical candidate shape.
# @PRE: raw_user may have heterogenous key names between Superset versions.
# @POST: Returns normalized candidate keys (environment_id, username, display_name, email, is_active).
# @RETURN: Dict[str, Any]
def normalize_user_payload(self, raw_user: Any) -> Dict[str, Any]:
if not isinstance(raw_user, dict):
raw_user = {}
@@ -232,7 +230,7 @@ class SupersetAccountLookupAdapter:
"email": email,
"is_active": is_active,
}
# #endregion normalize_user_payload
# [/DEF:normalize_user_payload:Function]
# #endregion SupersetAccountLookupAdapter
# #endregion SupersetProfileLookup
# #endregion SupersetProfileLookup

22
backend/src/core/task_manager/__tests__/test_context.py
View File

@@ -1,17 +1,19 @@
# #region TestContext [C:3] [TYPE Module] [SEMANTICS tests, task-context, background-tasks, sub-context]
# @BRIEF Verify TaskContext preserves optional background task scheduler across sub-context creation.
# @RELATION BELONGS_TO -> [SrcRoot]
# [DEF:TestContext:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 3
# @SEMANTICS: tests, task-context, background-tasks, sub-context
# @PURPOSE: Verify TaskContext preserves optional background task scheduler across sub-context creation.
from unittest.mock import MagicMock
from src.core.task_manager.context import TaskContext
# #region test_task_context_preserves_background_tasks_across_sub_context [TYPE Function]
# @BRIEF Plugins must be able to access background_tasks from both root and sub-context loggers.
# @PRE TaskContext is initialized with a BackgroundTasks-like object.
# @POST background_tasks remains available on root and derived sub-contexts.
# @RELATION BINDS_TO -> [TestContext]
# [DEF:test_task_context_preserves_background_tasks_across_sub_context:Function]
# @RELATION: BINDS_TO -> TestContext
# @PURPOSE: Plugins must be able to access background_tasks from both root and sub-context loggers.
# @PRE: TaskContext is initialized with a BackgroundTasks-like object.
# @POST: background_tasks remains available on root and derived sub-contexts.
def test_task_context_preserves_background_tasks_across_sub_context():
background_tasks = MagicMock()
context = TaskContext(
@@ -25,5 +27,5 @@ def test_task_context_preserves_background_tasks_across_sub_context():
assert context.background_tasks is background_tasks
assert sub_context.background_tasks is background_tasks
# #endregion test_task_context_preserves_background_tasks_across_sub_context
# #endregion TestContext
# [/DEF:test_task_context_preserves_background_tasks_across_sub_context:Function]
# [/DEF:TestContext:Module]

56
backend/src/core/task_manager/__tests__/test_task_logger.py
View File

@@ -1,7 +1,7 @@
# #region __tests__/test_task_logger [TYPE Module]
# @BRIEF Contract testing for TaskLogger
# @RELATION VERIFIES -> [../task_logger.py]
# #endregion __tests__/test_task_logger
# [DEF:__tests__/test_task_logger:Module]
# @RELATION: VERIFIES -> ../task_logger.py
# @PURPOSE: Contract testing for TaskLogger
# [/DEF:__tests__/test_task_logger:Module]
import pytest
from unittest.mock import MagicMock
@@ -17,20 +17,20 @@ def task_logger(mock_add_log):
return TaskLogger(task_id="test_123", add_log_fn=mock_add_log, source="test_plugin")
# @TEST_CONTRACT: TaskLoggerModel -> Invariants
# #region test_task_logger_initialization [TYPE Function]
# @BRIEF Verify TaskLogger initializes with correct task_id and state.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_task_logger_initialization:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger initializes with correct task_id and state.
def test_task_logger_initialization(task_logger):
"""Verify TaskLogger is bound to specific task_id and source."""
assert task_logger._task_id == "test_123"
assert task_logger._default_source == "test_plugin"
# @TEST_CONTRACT: invariants -> "All specific log methods (info, error) delegate to _log"
# #endregion test_task_logger_initialization
# [/DEF:test_task_logger_initialization:Function]
# #region test_log_methods_delegation [TYPE Function]
# @BRIEF Verify TaskLogger delegates log method calls to the underlying persistence service.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_log_methods_delegation:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger delegates log method calls to the underlying persistence service.
def test_log_methods_delegation(task_logger, mock_add_log):
"""Verify info, error, warning, debug delegate to internal _log."""
task_logger.info("info message", metadata={"k": "v"})
@@ -70,11 +70,11 @@ def test_log_methods_delegation(task_logger, mock_add_log):
)
# @TEST_CONTRACT: invariants -> "with_source creates a new logger with the same task_id"
# #endregion test_log_methods_delegation
# [/DEF:test_log_methods_delegation:Function]
# #region test_with_source [TYPE Function]
# @BRIEF Verify TaskLogger.with_source returns a new logger with the correct source attribution.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_with_source:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger.with_source returns a new logger with the correct source attribution.
def test_with_source(task_logger):
"""Verify with_source returns a new instance with updated default source."""
new_logger = task_logger.with_source("new_source")
@@ -84,33 +84,33 @@ def test_with_source(task_logger):
assert new_logger is not task_logger
# @TEST_EDGE: missing_task_id -> raises TypeError
# #endregion test_with_source
# [/DEF:test_with_source:Function]
# #region test_missing_task_id [TYPE Function]
# @BRIEF Verify TaskLogger raises or handles missing task_id gracefully.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_missing_task_id:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger raises or handles missing task_id gracefully.
def test_missing_task_id():
with pytest.raises(TypeError):
TaskLogger(add_log_fn=lambda x: x)
# @TEST_EDGE: invalid_add_log_fn -> raises TypeError
# (Python doesn't strictly enforce this at init, but let's verify it fails on call if not callable)
# #endregion test_missing_task_id
# [/DEF:test_missing_task_id:Function]
# #region test_invalid_add_log_fn [TYPE Function]
# @BRIEF Verify TaskLogger raises ValueError for invalid add_log_fn parameter.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_invalid_add_log_fn:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger raises ValueError for invalid add_log_fn parameter.
def test_invalid_add_log_fn():
logger = TaskLogger(task_id="msg", add_log_fn=None)
with pytest.raises(TypeError):
logger.info("test")
# @TEST_INVARIANT: consistent_delegation
# #endregion test_invalid_add_log_fn
# [/DEF:test_invalid_add_log_fn:Function]
# #region test_progress_log [TYPE Function]
# @BRIEF Verify TaskLogger correctly logs progress updates with percentage and message.
# @RELATION BINDS_TO -> [__tests__/test_task_logger]
# [DEF:test_progress_log:Function]
# @RELATION: BINDS_TO -> __tests__/test_task_logger
# @PURPOSE: Verify TaskLogger correctly logs progress updates with percentage and message.
def test_progress_log(task_logger, mock_add_log):
"""Verify progress method correctly formats metadata."""
task_logger.progress("Step 1", 45.5)
@@ -128,4 +128,4 @@ def test_progress_log(task_logger, mock_add_log):
task_logger.progress("Step low", -10)
assert mock_add_log.call_args[1]["metadata"]["progress"] == 0
# #endregion test_progress_log
# [/DEF:test_progress_log:Function]

34
backend/src/core/task_manager/cleanup.py
View File

@@ -1,6 +1,6 @@
# #region TaskCleanupModule [C:3] [TYPE Module] [SEMANTICS task, cleanup, retention, logs]
# @BRIEF Implements task cleanup and retention policies, including associated logs.
# @LAYER Core
# @LAYER: Core
# @RELATION DEPENDS_ON -> [TaskPersistenceService]
# @RELATION DEPENDS_ON -> [TaskLogPersistenceService]
# @RELATION DEPENDS_ON -> [ConfigManager]
@@ -16,10 +16,10 @@ from ..config_manager import ConfigManager
# @RELATION DEPENDS_ON -> [TaskLogPersistenceService]
# @RELATION DEPENDS_ON -> [ConfigManager]
class TaskCleanupService:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the cleanup service with dependencies.
# @PRE persistence_service and config_manager are valid.
# @POST Cleanup service is ready.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the cleanup service with dependencies.
# @PRE: persistence_service and config_manager are valid.
# @POST: Cleanup service is ready.
def __init__(
self,
persistence_service: TaskPersistenceService,
@@ -29,12 +29,12 @@ class TaskCleanupService:
self.persistence_service = persistence_service
self.log_persistence_service = log_persistence_service
self.config_manager = config_manager
# #endregion __init__
# [/DEF:__init__:Function]
# #region run_cleanup [TYPE Function]
# @BRIEF Deletes tasks older than the configured retention period and their logs.
# @PRE Config manager has valid settings.
# @POST Old tasks and their logs are deleted from persistence.
# [DEF:run_cleanup:Function]
# @PURPOSE: Deletes tasks older than the configured retention period and their logs.
# @PRE: Config manager has valid settings.
# @POST: Old tasks and their logs are deleted from persistence.
def run_cleanup(self):
with belief_scope("TaskCleanupService.run_cleanup"):
settings = self.config_manager.get_config().settings
@@ -54,12 +54,12 @@ class TaskCleanupService:
self.persistence_service.delete_tasks(to_delete)
logger.info(f"Deleted {len(to_delete)} tasks and their logs exceeding limit of {settings.task_retention_limit}")
# #endregion run_cleanup
# [/DEF:run_cleanup:Function]
# #region delete_task_with_logs [TYPE Function]
# @BRIEF Delete a single task and all its associated logs.
# @PRE task_id is a valid task ID.
# @POST Task and all its logs are deleted.
# [DEF:delete_task_with_logs:Function]
# @PURPOSE: Delete a single task and all its associated logs.
# @PRE: task_id is a valid task ID.
# @POST: Task and all its logs are deleted.
# @PARAM: task_id (str) - The task ID to delete.
def delete_task_with_logs(self, task_id: str) -> None:
"""Delete a single task and all its associated logs."""
@@ -71,7 +71,7 @@ class TaskCleanupService:
self.persistence_service.delete_tasks([task_id])
logger.info(f"Deleted task {task_id} and its associated logs")
# #endregion delete_task_with_logs
# [/DEF:delete_task_with_logs:Function]
# #endregion TaskCleanupService
# #endregion TaskCleanupModule
# #endregion TaskCleanupModule

101
backend/src/core/task_manager/context.py
View File

@@ -1,30 +1,27 @@
# #region TaskContextModule [C:5] [TYPE Module] [SEMANTICS task, context, plugin, execution, logger]
# @BRIEF Provides execution context passed to plugins during task execution.
# @LAYER Core
# @PRE Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries.
# @POST Plugins receive context instances with stable logger and parameter accessors.
# @SIDE_EFFECT Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts.
# @DATA_CONTRACT Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @INVARIANT Each TaskContext is bound to a single task execution.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [TaskLoggerModule]
# @RELATION DEPENDS_ON -> [TaskManager]
# @INVARIANT: Each TaskContext is bound to a single task execution.
# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries.
# @POST: Plugins receive context instances with stable logger and parameter accessors.
# @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# [SECTION: IMPORTS]
# [SECTION: IMPORTS]
from typing import Dict, Any, Callable, Optional
from .task_logger import TaskLogger
from ..logger import belief_scope
# [/SECTION]
# #region TaskContext [C:5] [TYPE Class] [SEMANTICS context, task, execution, plugin]
# @BRIEF A container passed to plugin.execute() providing the logger and other task-specific utilities.
# @PRE Constructor receives non-empty task_id, callable add_log_fn, and params mapping.
# @POST Instance exposes immutable task identity with logger, params, and optional background task access.
# @SIDE_EFFECT Emits structured task logs through TaskLogger on plugin interactions.
# @DATA_CONTRACT Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @INVARIANT logger is always a valid TaskLogger instance.
# @INVARIANT: logger is always a valid TaskLogger instance.
# @PRE: Constructor receives non-empty task_id, callable add_log_fn, and params mapping.
# @POST: Instance exposes immutable task identity with logger, params, and optional background task access.
# @RELATION DEPENDS_ON -> [TaskLogger]
# @SIDE_EFFECT: Emits structured task logs through TaskLogger on plugin interactions.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @UX_STATE: Idle -> Active -> Complete
#
# @TEST_CONTRACT: TaskContextContract ->
@@ -52,10 +49,10 @@ class TaskContext:
# ... plugin logic
"""
# #region __init__ [TYPE Function]
# @BRIEF Initialize the TaskContext with task-specific resources.
# @PRE task_id is a valid task identifier, add_log_fn is callable.
# @POST TaskContext is ready to be passed to plugin.execute().
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskContext with task-specific resources.
# @PRE: task_id is a valid task identifier, add_log_fn is callable.
# @POST: TaskContext is ready to be passed to plugin.execute().
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
# @PARAM: params (Dict) - Task parameters.
@@ -76,59 +73,59 @@ class TaskContext:
task_id=task_id, add_log_fn=add_log_fn, source=default_source
)
# #endregion __init__
# [/DEF:__init__:Function]
# #region task_id [TYPE Function]
# @BRIEF Get the task ID.
# @PRE TaskContext must be initialized.
# @POST Returns the task ID string.
# @RETURN str - The task ID.
# [DEF:task_id:Function]
# @PURPOSE: Get the task ID.
# @PRE: TaskContext must be initialized.
# @POST: Returns the task ID string.
# @RETURN: str - The task ID.
@property
def task_id(self) -> str:
with belief_scope("task_id"):
return self._task_id
# #endregion task_id
# [/DEF:task_id:Function]
# #region logger [TYPE Function]
# @BRIEF Get the TaskLogger instance for this context.
# @PRE TaskContext must be initialized.
# @POST Returns the TaskLogger instance.
# @RETURN TaskLogger - The logger instance.
# [DEF:logger:Function]
# @PURPOSE: Get the TaskLogger instance for this context.
# @PRE: TaskContext must be initialized.
# @POST: Returns the TaskLogger instance.
# @RETURN: TaskLogger - The logger instance.
@property
def logger(self) -> TaskLogger:
with belief_scope("logger"):
return self._logger
# #endregion logger
# [/DEF:logger:Function]
# #region params [TYPE Function]
# @BRIEF Get the task parameters.
# @PRE TaskContext must be initialized.
# @POST Returns the parameters dictionary.
# @RETURN Dict[str, Any] - The task parameters.
# [DEF:params:Function]
# @PURPOSE: Get the task parameters.
# @PRE: TaskContext must be initialized.
# @POST: Returns the parameters dictionary.
# @RETURN: Dict[str, Any] - The task parameters.
@property
def params(self) -> Dict[str, Any]:
with belief_scope("params"):
return self._params
# #endregion params
# [/DEF:params:Function]
# #region background_tasks [TYPE Function]
# @BRIEF Expose optional background task scheduler for plugins that dispatch deferred side effects.
# @PRE TaskContext must be initialized.
# @POST Returns BackgroundTasks-like object or None.
# [DEF:background_tasks:Function]
# @PURPOSE: Expose optional background task scheduler for plugins that dispatch deferred side effects.
# @PRE: TaskContext must be initialized.
# @POST: Returns BackgroundTasks-like object or None.
@property
def background_tasks(self) -> Optional[Any]:
with belief_scope("background_tasks"):
return self._background_tasks
# #endregion background_tasks
# [/DEF:background_tasks:Function]
# #region get_param [TYPE Function]
# @BRIEF Get a specific parameter value with optional default.
# @PRE TaskContext must be initialized.
# @POST Returns parameter value or default.
# [DEF:get_param:Function]
# @PURPOSE: Get a specific parameter value with optional default.
# @PRE: TaskContext must be initialized.
# @POST: Returns parameter value or default.
# @PARAM: key (str) - Parameter key.
# @PARAM: default (Any) - Default value if key not found.
# @RETURN: Any - Parameter value or default.
@@ -136,12 +133,12 @@ class TaskContext:
with belief_scope("get_param"):
return self._params.get(key, default)
# #endregion get_param
# [/DEF:get_param:Function]
# #region create_sub_context [TYPE Function]
# @BRIEF Create a sub-context with a different default source.
# @PRE source is a non-empty string.
# @POST Returns new TaskContext with different logger source.
# [DEF:create_sub_context:Function]
# @PURPOSE: Create a sub-context with a different default source.
# @PRE: source is a non-empty string.
# @POST: Returns new TaskContext with different logger source.
# @PARAM: source (str) - New default source for logging.
# @RETURN: TaskContext - New context with different source.
def create_sub_context(self, source: str) -> "TaskContext":
@@ -155,7 +152,7 @@ class TaskContext:
background_tasks=self._background_tasks,
)
# #endregion create_sub_context
# [/DEF:create_sub_context:Function]
# #endregion TaskContext

418
backend/src/core/task_manager/manager.py
View File

@@ -1,11 +1,10 @@
# #region TaskManagerModule [C:5] [TYPE Module] [SEMANTICS task, manager, lifecycle, execution, state]
# @BRIEF Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
# @LAYER Core
# @PRE Plugin loader and database sessions are initialized.
# @POST Orchestrates task execution and persistence.
# @SIDE_EFFECT Spawns worker threads and flushes logs to DB.
# @DATA_CONTRACT Input[plugin_id, params] -> Model[Task, LogEntry]
# @INVARIANT Task IDs are unique.
# @LAYER: Core
# @PRE: Plugin loader and database sessions are initialized.
# @POST: Orchestrates task execution and persistence.
# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.
# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]
# @RELATION DEPENDS_ON -> [PluginLoader]
# @RELATION DEPENDS_ON -> [TaskPersistenceService]
# @RELATION DEPENDS_ON -> [TaskLogPersistenceService]
@@ -13,6 +12,7 @@
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [JobLifecycle]
# @RELATION DEPENDS_ON -> [EventBus]
# @INVARIANT: Task IDs are unique.
# @TEST_CONTRACT: TaskManagerRuntime -> {
# required_fields: {plugin_loader: PluginLoader},
# optional_fields: {},
@@ -25,7 +25,6 @@
# @TEST_EDGE: external_failure -> {"db_unavailable": true}
# @TEST_INVARIANT: logger_compliance -> verifies: [valid_module]
# [SECTION: IMPORTS]
import asyncio
import threading
import inspect
@@ -37,22 +36,11 @@ from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats
from .persistence import TaskPersistenceService, TaskLogPersistenceService
from .context import TaskContext
from ..logger import logger, belief_scope, should_log_task_level
from ..cot_logger import MarkerLogger, get_trace_id, set_trace_id
log = MarkerLogger("TaskManager")
# [/SECTION]
# #region TaskManager [C:5] [TYPE Class] [SEMANTICS task, manager, lifecycle, execution, state]
# @BRIEF Manages the lifecycle of tasks, including their creation, execution, and state tracking.
# @LAYER Core
# @PRE Plugin loader resolves plugin ids and persistence services are available for task state hydration.
# @POST In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.
# @SIDE_EFFECT Spawns worker threads, flushes logs to database, and mutates task states.
# @DATA_CONTRACT Input[plugin_id, params] -> Output[Task]
# @INVARIANT Task IDs are unique within the registry.
# @INVARIANT Each task has exactly one status at any time.
# @INVARIANT Log entries are never deleted after being added to a task.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [TaskPersistenceService]
# @RELATION DEPENDS_ON -> [TaskLogPersistenceService]
# @RELATION DEPENDS_ON -> [PluginLoader]
@@ -60,6 +48,13 @@ log = MarkerLogger("TaskManager")
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [JobLifecycle]
# @RELATION DEPENDS_ON -> [EventBus]
# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.
# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
# @SIDE_EFFECT: Spawns worker threads, flushes logs to database, and mutates task states.
# @DATA_CONTRACT: Input[plugin_id, params] -> Output[Task]
class TaskManager:
"""
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
@@ -68,53 +63,57 @@ class TaskManager:
# Log flush interval in seconds
LOG_FLUSH_INTERVAL = 2.0
# #region TaskGraph [C:5] [TYPE Block]
# @BRIEF Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.
# @PRE Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.
# @POST Each live task id resolves to at most one active Task node and optional pause future.
# @SIDE_EFFECT Mutates the in-memory task registry and loads persisted state during manager startup.
# @DATA_CONTRACT Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]
# @INVARIANT Registry membership is keyed by unique task id and survives log streaming side channels.
# @RELATION DEPENDS_ON -> [Task]
# @RELATION DEPENDS_ON -> [TaskPersistenceService]
# #endregion TaskGraph
# [DEF:TaskGraph:Block]
# @COMPLEXITY: 5
# @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.
# @RELATION: [DEPENDS_ON] ->[Task]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.
# @POST: Each live task id resolves to at most one active Task node and optional pause future.
# @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.
# @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]
# @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.
# [/DEF:TaskGraph:Block]
# #region EventBus [C:5] [TYPE Block]
# @BRIEF Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.
# @PRE Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.
# @POST Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.
# @SIDE_EFFECT Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.
# @DATA_CONTRACT Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]
# @INVARIANT Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.
# @RELATION DEPENDS_ON -> [LogEntry]
# @RELATION DEPENDS_ON -> [TaskLogPersistenceService]
# #endregion EventBus
# [DEF:EventBus:Block]
# @COMPLEXITY: 5
# @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.
# @RELATION: [DEPENDS_ON] ->[LogEntry]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.
# @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.
# @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.
# @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]
# @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.
# [/DEF:EventBus:Block]
# #region JobLifecycle [C:5] [TYPE Block]
# @BRIEF Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.
# @PRE Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.
# @POST Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.
# @SIDE_EFFECT Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.
# @DATA_CONTRACT Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]
# @INVARIANT A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [EventBus]
# @RELATION DEPENDS_ON -> [TaskContext]
# @RELATION DEPENDS_ON -> [PluginLoader]
# #endregion JobLifecycle
# [DEF:JobLifecycle:Block]
# @COMPLEXITY: 5
# @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.
# @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.
# @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.
# @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]
# @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.
# [/DEF:JobLifecycle:Block]
# #region __init__ [C:5] [TYPE Function]
# @BRIEF Initialize the TaskManager with dependencies.
# @PRE plugin_loader is initialized.
# @POST TaskManager is ready to accept tasks.
# @SIDE_EFFECT Starts background flusher thread and loads persisted task state into memory.
# @RELATION CALLS -> [TaskPersistenceService.load_tasks]
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [EventBus]
# [DEF:__init__:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
# @PRE: plugin_loader is initialized.
# @POST: TaskManager is ready to accept tasks.
# @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @PARAM: plugin_loader - The plugin loader instance.
def __init__(self, plugin_loader):
with belief_scope("TaskManager.__init__"):
log.reason("Initializing task manager runtime services")
logger.reason("Initializing task manager runtime services")
self.plugin_loader = plugin_loader
self.tasks: Dict[str, Task] = {}
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
@@ -143,33 +142,35 @@ class TaskManager:
# Load persisted tasks on startup
self.load_persisted_tasks()
log.reflect(
logger.reflect(
"Task manager runtime initialized",
payload={"task_count": len(self.tasks)},
extra={"task_count": len(self.tasks)},
)
# #endregion __init__
# [/DEF:__init__:Function]
# #region _flusher_loop [C:3] [TYPE Function]
# @BRIEF Background thread that periodically flushes log buffer to database.
# @PRE TaskManager is initialized.
# @POST Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION CALLS -> [TaskManager._flush_logs]
# @RELATION DEPENDS_ON -> [EventBus]
# [DEF:_flusher_loop:Function]
# @COMPLEXITY: 3
# @PURPOSE: Background thread that periodically flushes log buffer to database.
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION: [CALLS] ->[TaskManager._flush_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flusher_loop(self):
"""Background thread that flushes log buffer to database."""
while not self._flusher_stop_event.is_set():
self._flush_logs()
self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)
# #endregion _flusher_loop
# [/DEF:_flusher_loop:Function]
# #region _flush_logs [C:3] [TYPE Function]
# @BRIEF Flush all buffered logs to the database.
# @PRE None.
# @POST All buffered logs are written to task_logs table.
# @RELATION CALLS -> [TaskLogPersistenceService.add_logs]
# @RELATION DEPENDS_ON -> [EventBus]
# [DEF:_flush_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Flush all buffered logs to the database.
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flush_logs(self):
"""Flush all buffered logs to the database."""
with self._log_buffer_lock:
@@ -182,9 +183,8 @@ class TaskManager:
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
log.reflect("Flushed logs to database", payload={"task_id": task_id, "count": len(logs)})
logger.debug(f"Flushed {len(logs)} logs for task {task_id}")
except Exception as e:
log.explore("Failed to flush logs to database", error=str(e), payload={"task_id": task_id})
logger.error(f"Failed to flush logs for task {task_id}: {e}")
# Re-add logs to buffer on failure
with self._log_buffer_lock:
@@ -192,12 +192,13 @@ class TaskManager:
self._log_buffer[task_id] = []
self._log_buffer[task_id].extend(logs)
# #endregion _flush_logs
# [/DEF:_flush_logs:Function]
# #region _flush_task_logs [C:3] [TYPE Function]
# @BRIEF Flush logs for a specific task immediately.
# @PRE task_id exists.
# @POST Task's buffered logs are written to database.
# [DEF:_flush_task_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Flush logs for a specific task immediately.
# @PRE: task_id exists.
# @POST: Task's buffered logs are written to database.
# @PARAM: task_id (str) - The task ID.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
@@ -210,17 +211,16 @@ class TaskManager:
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
log.reflect("Flushed task logs immediately", payload={"task_id": task_id, "count": len(logs)})
except Exception as e:
log.explore("Failed to flush task logs immediately", error=str(e), payload={"task_id": task_id})
logger.error(f"Failed to flush logs for task {task_id}: {e}")
# #endregion _flush_task_logs
# [/DEF:_flush_task_logs:Function]
# #region create_task [C:3] [TYPE Function]
# @BRIEF Creates and queues a new task for execution.
# @PRE Plugin with plugin_id exists. Params are valid.
# @POST Task is created, added to registry, and scheduled for execution.
# [DEF:create_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Creates and queues a new task for execution.
# @PRE: Plugin with plugin_id exists. Params are valid.
# @POST: Task is created, added to registry, and scheduled for execution.
# @PARAM: plugin_id (str) - The ID of the plugin to run.
# @PARAM: params (Dict[str, Any]) - Parameters for the plugin.
# @PARAM: user_id (Optional[str]) - ID of the user requesting the task.
@@ -234,50 +234,52 @@ class TaskManager:
) -> Task:
with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"):
if not self.plugin_loader.has_plugin(plugin_id):
log.explore("Plugin not found", error=f"Plugin with ID '{plugin_id}' not found.")
logger.error(f"Plugin with ID '{plugin_id}' not found.")
raise ValueError(f"Plugin with ID '{plugin_id}' not found.")
self.plugin_loader.get_plugin(plugin_id)
if not isinstance(params, dict):
log.explore("Invalid task params type", error="Task parameters must be a dictionary.")
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
log.reason("Creating task node and scheduling execution", payload={"plugin_id": plugin_id})
logger.reason("Creating task node and scheduling execution")
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
self.tasks[task.id] = task
self.persistence_service.persist_task(task)
trace_id = get_trace_id()
logger.info(f"Task {task.id} created and scheduled for execution")
self.loop.create_task(
self._run_task(task.id, trace_id=trace_id)
self._run_task(task.id)
) # Schedule task for execution
log.reflect(
logger.reflect(
"Task creation persisted and execution scheduled",
payload={"task_id": task.id, "plugin_id": plugin_id},
extra={"task_id": task.id, "plugin_id": plugin_id},
)
return task
# #endregion create_task
# [/DEF:create_task:Function]
# #region _run_task [C:3] [TYPE Function]
# @BRIEF Internal method to execute a task with TaskContext support.
# @PRE Task exists in registry.
# @POST Task is executed, status updated to SUCCESS or FAILED.
# [DEF:_run_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Internal method to execute a task with TaskContext support.
# @PRE: Task exists in registry.
# @POST: Task is executed, status updated to SUCCESS or FAILED.
# @PARAM: task_id (str) - The ID of the task to run.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[EventBus]
async def _run_task(self, task_id: str, trace_id: str = None):
if trace_id:
set_trace_id(trace_id)
async def _run_task(self, task_id: str):
with belief_scope("TaskManager._run_task", f"task_id={task_id}"):
task = self.tasks[task_id]
plugin = self.plugin_loader.get_plugin(task.plugin_id)
log.reason(
logger.reason(
"Transitioning task to running state",
payload={"task_id": task_id, "plugin_id": task.plugin_id},
extra={"task_id": task_id, "plugin_id": task.plugin_id},
)
logger.info(
f"Starting execution of task {task_id} for plugin '{plugin.name}'"
)
task.status = TaskStatus.RUNNING
task.started_at = datetime.utcnow()
@@ -323,6 +325,7 @@ class TaskManager:
self.executor, plugin.execute, params
)
logger.info(f"Task {task_id} completed successfully")
task.status = TaskStatus.SUCCESS
self._add_log(
task_id,
@@ -330,8 +333,8 @@ class TaskManager:
f"Task completed successfully for plugin '{plugin.name}'",
source="system",
)
log.reflect("Task completed successfully", payload={"task_id": task_id, "plugin_id": task.plugin_id})
except Exception as e:
logger.error(f"Task {task_id} failed: {e}")
task.status = TaskStatus.FAILED
self._add_log(
task_id,
@@ -340,23 +343,26 @@ class TaskManager:
source="system",
metadata={"error_type": type(e).__name__},
)
log.explore("Task failed during execution", error=str(e), payload={"task_id": task_id, "plugin_id": task.plugin_id})
finally:
task.finished_at = datetime.utcnow()
# Flush any remaining buffered logs before persisting task
self._flush_task_logs(task_id)
self.persistence_service.persist_task(task)
log.reflect(
"Task execution finished",
payload={"task_id": task_id, "status": str(task.status)},
logger.info(
f"Task {task_id} execution finished with status: {task.status}"
)
logger.reflect(
"Task lifecycle reached persisted terminal state",
extra={"task_id": task_id, "status": str(task.status)},
)
# #endregion _run_task
# [/DEF:_run_task:Function]
# #region resolve_task [C:3] [TYPE Function]
# @BRIEF Resumes a task that is awaiting mapping.
# @PRE Task exists and is in AWAITING_MAPPING state.
# @POST Task status updated to RUNNING, params updated, execution resumed.
# [DEF:resolve_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Resumes a task that is awaiting mapping.
# @PRE: Task exists and is in AWAITING_MAPPING state.
# @POST: Task status updated to RUNNING, params updated, execution resumed.
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.
# @THROWS: ValueError if task not found or not awaiting mapping.
@@ -378,12 +384,13 @@ class TaskManager:
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# #endregion resolve_task
# [/DEF:resolve_task:Function]
# #region wait_for_resolution [C:3] [TYPE Function]
# @BRIEF Pauses execution and waits for a resolution signal.
# @PRE Task exists.
# @POST Execution pauses until future is set.
# [DEF:wait_for_resolution:Function]
# @COMPLEXITY: 3
# @PURPOSE: Pauses execution and waits for a resolution signal.
# @PRE: Task exists.
# @POST: Execution pauses until future is set.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
@@ -403,12 +410,13 @@ class TaskManager:
if task_id in self.task_futures:
del self.task_futures[task_id]
# #endregion wait_for_resolution
# [/DEF:wait_for_resolution:Function]
# #region wait_for_input [C:3] [TYPE Function]
# @BRIEF Pauses execution and waits for user input.
# @PRE Task exists.
# @POST Execution pauses until future is set via resume_task_with_password.
# [DEF:wait_for_input:Function]
# @COMPLEXITY: 3
# @PURPOSE: Pauses execution and waits for user input.
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def wait_for_input(self, task_id: str):
@@ -426,12 +434,13 @@ class TaskManager:
if task_id in self.task_futures:
del self.task_futures[task_id]
# #endregion wait_for_input
# [/DEF:wait_for_input:Function]
# #region get_task [C:3] [TYPE Function]
# @BRIEF Retrieves a task by its ID.
# @PRE task_id is a string.
# @POST Returns Task object or None.
# [DEF:get_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Retrieves a task by its ID.
# @PRE: task_id is a string.
# @POST: Returns Task object or None.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: Optional[Task] - The task or None.
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
@@ -439,24 +448,26 @@ class TaskManager:
with belief_scope("TaskManager.get_task", f"task_id={task_id}"):
return self.tasks.get(task_id)
# #endregion get_task
# [/DEF:get_task:Function]
# #region get_all_tasks [C:3] [TYPE Function]
# @BRIEF Retrieves all registered tasks.
# @PRE None.
# @POST Returns list of all Task objects.
# @RETURN List[Task] - All tasks.
# @RELATION DEPENDS_ON -> [TaskGraph]
# [DEF:get_all_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Retrieves all registered tasks.
# @PRE: None.
# @POST: Returns list of all Task objects.
# @RETURN: List[Task] - All tasks.
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_all_tasks(self) -> List[Task]:
with belief_scope("TaskManager.get_all_tasks"):
return list(self.tasks.values())
# #endregion get_all_tasks
# [/DEF:get_all_tasks:Function]
# #region get_tasks [C:3] [TYPE Function]
# @BRIEF Retrieves tasks with pagination and optional status filter.
# @PRE limit and offset are non-negative integers.
# @POST Returns a list of tasks sorted by start_time descending.
# [DEF:get_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Retrieves tasks with pagination and optional status filter.
# @PRE: limit and offset are non-negative integers.
# @POST: Returns a list of tasks sorted by start_time descending.
# @PARAM: limit (int) - Maximum number of tasks to return.
# @PARAM: offset (int) - Number of tasks to skip.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
@@ -496,12 +507,13 @@ class TaskManager:
tasks.sort(key=sort_key, reverse=True)
return tasks[offset : offset + limit]
# #endregion get_tasks
# [/DEF:get_tasks:Function]
# #region get_task_logs [C:3] [TYPE Function]
# @BRIEF Retrieves logs for a specific task (from memory for running, persistence for completed).
# @PRE task_id is a string.
# @POST Returns list of LogEntry or TaskLog objects.
# [DEF:get_task_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).
# @PRE: task_id is a string.
# @POST: Returns list of LogEntry or TaskLog objects.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
# @RETURN: List[LogEntry] - List of log entries.
@@ -533,12 +545,13 @@ class TaskManager:
# For running/pending tasks, return from memory
return task.logs if task else []
# #endregion get_task_logs
# [/DEF:get_task_logs:Function]
# #region get_task_log_stats [C:3] [TYPE Function]
# @BRIEF Get statistics about logs for a task.
# @PRE task_id is a valid task ID.
# @POST Returns LogStats with counts by level and source.
# [DEF:get_task_log_stats:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]
@@ -547,12 +560,13 @@ class TaskManager:
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
return self.log_persistence_service.get_log_stats(task_id)
# #endregion get_task_log_stats
# [/DEF:get_task_log_stats:Function]
# #region get_task_log_sources [C:3] [TYPE Function]
# @BRIEF Get unique sources for a task's logs.
# @PRE task_id is a valid task ID.
# @POST Returns list of unique source strings.
# [DEF:get_task_log_sources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]
@@ -561,12 +575,13 @@ class TaskManager:
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
return self.log_persistence_service.get_sources(task_id)
# #endregion get_task_log_sources
# [/DEF:get_task_log_sources:Function]
# #region _add_log [C:3] [TYPE Function]
# @BRIEF Adds a log entry to a task buffer and notifies subscribers.
# @PRE Task exists.
# @POST Log added to buffer and pushed to queues (if level meets task_log_level filter).
# [DEF:_add_log:Function]
# @COMPLEXITY: 3
# @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.
# @PRE: Task exists.
# @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).
# @PARAM: task_id (str) - ID of the task.
# @PARAM: level (str) - Log level.
# @PARAM: message (str) - Log message.
@@ -616,12 +631,13 @@ class TaskManager:
for queue in self.subscribers[task_id]:
self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)
# #endregion _add_log
# [/DEF:_add_log:Function]
# #region subscribe_logs [C:3] [TYPE Function]
# @BRIEF Subscribes to real-time logs for a task.
# @PRE task_id is a string.
# @POST Returns an asyncio.Queue for log entries.
# [DEF:subscribe_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Subscribes to real-time logs for a task.
# @PRE: task_id is a string.
# @POST: Returns an asyncio.Queue for log entries.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: asyncio.Queue - Queue for log entries.
# @RELATION: [DEPENDS_ON] ->[EventBus]
@@ -633,12 +649,13 @@ class TaskManager:
self.subscribers[task_id].append(queue)
return queue
# #endregion subscribe_logs
# [/DEF:subscribe_logs:Function]
# #region unsubscribe_logs [C:3] [TYPE Function]
# @BRIEF Unsubscribes from real-time logs for a task.
# @PRE task_id is a string, queue is asyncio.Queue.
# @POST Queue removed from subscribers.
# [DEF:unsubscribe_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Unsubscribes from real-time logs for a task.
# @PRE: task_id is a string, queue is asyncio.Queue.
# @POST: Queue removed from subscribers.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: queue (asyncio.Queue) - Queue to remove.
# @RELATION: [DEPENDS_ON] ->[EventBus]
@@ -650,14 +667,15 @@ class TaskManager:
if not self.subscribers[task_id]:
del self.subscribers[task_id]
# #endregion unsubscribe_logs
# [/DEF:unsubscribe_logs:Function]
# #region load_persisted_tasks [C:3] [TYPE Function]
# @BRIEF Load persisted tasks using persistence service.
# @PRE None.
# @POST Persisted tasks loaded into self.tasks.
# @RELATION CALLS -> [TaskPersistenceService.load_tasks]
# @RELATION DEPENDS_ON -> [TaskGraph]
# [DEF:load_persisted_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Load persisted tasks using persistence service.
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def load_persisted_tasks(self) -> None:
with belief_scope("TaskManager.load_persisted_tasks"):
loaded_tasks = self.persistence_service.load_tasks(limit=100)
@@ -665,12 +683,13 @@ class TaskManager:
if task.id not in self.tasks:
self.tasks[task.id] = task
# #endregion load_persisted_tasks
# [/DEF:load_persisted_tasks:Function]
# #region await_input [C:3] [TYPE Function]
# @BRIEF Transition a task to AWAITING_INPUT state with input request.
# @PRE Task exists and is in RUNNING state.
# @POST Task status changed to AWAITING_INPUT, input_request set, persisted.
# [DEF:await_input:Function]
# @COMPLEXITY: 3
# @PURPOSE: Transition a task to AWAITING_INPUT state with input request.
# @PRE: Task exists and is in RUNNING state.
# @POST: Task status changed to AWAITING_INPUT, input_request set, persisted.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: input_request (Dict) - Details about required input.
# @THROWS: ValueError if task not found or not RUNNING.
@@ -697,12 +716,13 @@ class TaskManager:
metadata={"input_request": input_request},
)
# #endregion await_input
# [/DEF:await_input:Function]
# #region resume_task_with_password [C:3] [TYPE Function]
# @BRIEF Resume a task that is awaiting input with provided passwords.
# @PRE Task exists and is in AWAITING_INPUT state.
# @POST Task status changed to RUNNING, passwords injected, task resumed.
# [DEF:resume_task_with_password:Function]
# @COMPLEXITY: 3
# @PURPOSE: Resume a task that is awaiting input with provided passwords.
# @PRE: Task exists and is in AWAITING_INPUT state.
# @POST: Task status changed to RUNNING, passwords injected, task resumed.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.
# @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.
@@ -740,12 +760,13 @@ class TaskManager:
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# #endregion resume_task_with_password
# [/DEF:resume_task_with_password:Function]
# #region clear_tasks [C:3] [TYPE Function]
# @BRIEF Clears tasks based on status filter (also deletes associated logs).
# @PRE status is Optional[TaskStatus].
# @POST Tasks matching filter (or all non-active) cleared from registry and database.
# [DEF:clear_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Clears tasks based on status filter (also deletes associated logs).
# @PRE: status is Optional[TaskStatus].
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: int - Number of tasks cleared.
# @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]
@@ -791,10 +812,11 @@ class TaskManager:
if tasks_to_remove:
self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)
log.reflect("Tasks cleared from registry and database", payload={"count": len(tasks_to_remove)})
logger.info(f"Cleared {len(tasks_to_remove)} tasks.")
return len(tasks_to_remove)
# #endregion clear_tasks
# [/DEF:clear_tasks:Function]
# #endregion TaskManager
# #endregion TaskManagerModule

16
backend/src/core/task_manager/models.py
View File

@@ -1,19 +1,17 @@
# #region TaskManagerModels [C:3] [TYPE Module] [SEMANTICS task, models, pydantic, enum, state]
# @BRIEF Defines the data models and enumerations used by the Task Manager.
# @LAYER Core
# @INVARIANT Task IDs are immutable once created.
# @LAYER: Core
# @RELATION USED_BY -> [TaskManager]
# @RELATION USED_BY -> [TaskManagerPackage]
# @INVARIANT: Task IDs are immutable once created.
# @CONSTRAINT: Must use Pydantic for data validation.
# [SECTION: IMPORTS]
import uuid
from datetime import datetime
from enum import Enum
from typing import Dict, Any, List, Optional
from pydantic import BaseModel, Field
# [/SECTION]
# #region TaskStatus [TYPE Enum]
@@ -127,17 +125,17 @@ class Task(BaseModel):
# Result payload can be dict/list/scalar depending on plugin and legacy records.
result: Optional[Any] = None
# #region __init__ [TYPE Function]
# @BRIEF Initializes the Task model and validates input_request for AWAITING_INPUT status.
# @PRE If status is AWAITING_INPUT, input_request must be provided.
# @POST Task instance is created or ValueError is raised.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the Task model and validates input_request for AWAITING_INPUT status.
# @PRE: If status is AWAITING_INPUT, input_request must be provided.
# @POST: Task instance is created or ValueError is raised.
# @PARAM: **data - Keyword arguments for model initialization.
def __init__(self, **data):
super().__init__(**data)
if self.status == TaskStatus.AWAITING_INPUT and not self.input_request:
raise ValueError("input_request is required when status is AWAITING_INPUT")
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion Task

261
backend/src/core/task_manager/persistence.py
View File

@@ -1,16 +1,15 @@
# #region TaskPersistenceModule [C:5] [TYPE Module] [SEMANTICS persistence, sqlite, sqlalchemy, task, storage]
# @BRIEF Handles the persistence of tasks using SQLAlchemy and the tasks.db database.
# @LAYER Core
# @PRE Tasks database must be initialized with TaskRecord and TaskLogRecord schemas.
# @POST Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT Performs database I/O on tasks.db.
# @DATA_CONTRACT Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @INVARIANT Database schema must match the TaskRecord model structure.
# @LAYER: Core
# @PRE: Tasks database must be initialized with TaskRecord and TaskLogRecord schemas.
# @POST: Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION DEPENDS_ON -> [TaskManager]
# @RELATION DEPENDS_ON -> [TaskGraph]
# @RELATION DEPENDS_ON -> [TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
# [SECTION: IMPORTS]
from datetime import datetime
from typing import List, Optional
import json
@@ -22,25 +21,20 @@ from ...models.mapping import Environment
from ..database import TasksSessionLocal
from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats
from ..logger import logger, belief_scope
from ..cot_logger import MarkerLogger
log = MarkerLogger("TaskPersistence")
log_pl = MarkerLogger("TaskLogPersistence")
# [/SECTION]
# #region TaskPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, sqlalchemy]
# @BRIEF Provides methods to save, load, and delete task records in tasks.db using SQLAlchemy models.
# @PRE TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.
# @POST Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.
# @SIDE_EFFECT Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]
# @INVARIANT Persistence must handle potentially missing task fields natively.
# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, Task inputs must expose id/plugin_id/status/params/result/logs fields, and TaskRecord plus Environment schemas must be available.
# @POST: Persist operations leave matching TaskRecord rows committed or rolled back without leaking sessions, load operations return reconstructed Task objects from stored TaskRecord rows, and delete operations remove only the addressed task rows.
# @SIDE_EFFECT: Opens SQLAlchemy sessions, reads and writes task_records rows, resolves environment foreign keys against environments, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT: Input[Task | List[Task] | List[str] | Query(limit:int,status:Optional[TaskStatus])] -> Model[TaskRecord, Environment] -> Output[None | List[Task]]
# @RELATION DEPENDS_ON -> [TasksSessionLocal]
# @RELATION DEPENDS_ON -> [TaskRecord]
# @RELATION DEPENDS_ON -> [Environment]
# @RELATION DEPENDS_ON -> [TaskManager]
# @RELATION DEPENDS_ON -> [TaskGraph]
# @INVARIANT: Persistence must handle potentially missing task fields natively.
#
# @TEST_CONTRACT: TaskPersistenceContract ->
# {
@@ -56,10 +50,11 @@ log_pl = MarkerLogger("TaskLogPersistence")
# @TEST_EDGE: load_corrupt_json_params -> handled gracefully
# @TEST_INVARIANT: accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]
class TaskPersistenceService:
# #region _json_load_if_needed [C:1] [TYPE Function]
# @BRIEF Safely load JSON strings from DB if necessary
# @PRE value is an arbitrary database value
# @POST Returns parsed JSON object, list, string, or primitive
# [DEF:_json_load_if_needed:Function]
# @COMPLEXITY: 1
# @PURPOSE: Safely load JSON strings from DB if necessary
# @PRE: value is an arbitrary database value
# @POST: Returns parsed JSON object, list, string, or primitive
@staticmethod
def _json_load_if_needed(value):
with belief_scope("TaskPersistenceService._json_load_if_needed"):
@@ -77,12 +72,13 @@ class TaskPersistenceService:
return value
return value
# #endregion _json_load_if_needed
# [/DEF:_json_load_if_needed:Function]
# #region _parse_datetime [C:1] [TYPE Function]
# @BRIEF Safely parse a datetime string from the database
# @PRE value is an ISO string or datetime object
# @POST Returns datetime object or None
# [DEF:_parse_datetime:Function]
# @COMPLEXITY: 1
# @PURPOSE: Safely parse a datetime string from the database
# @PRE: value is an ISO string or datetime object
# @POST: Returns datetime object or None
@staticmethod
def _parse_datetime(value):
with belief_scope("TaskPersistenceService._parse_datetime"):
@@ -95,14 +91,15 @@ class TaskPersistenceService:
return None
return None
# #endregion _parse_datetime
# [/DEF:_parse_datetime:Function]
# #region _resolve_environment_id [C:3] [TYPE Function]
# @BRIEF Resolve environment id into existing environments.id value to satisfy FK constraints.
# @PRE Session is active
# @POST Returns existing environments.id or None when unresolved.
# @DATA_CONTRACT Input[env_id: Optional[str]] -> Output[Optional[str]]
# @RELATION DEPENDS_ON -> [Environment]
# [DEF:_resolve_environment_id:Function]
# @COMPLEXITY: 3
# @PURPOSE: Resolve environment id into existing environments.id value to satisfy FK constraints.
# @PRE: Session is active
# @POST: Returns existing environments.id or None when unresolved.
# @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]
# @RELATION: [DEPENDS_ON] ->[Environment]
@staticmethod
def _resolve_environment_id(
session: Session, env_id: Optional[str]
@@ -144,30 +141,31 @@ class TaskPersistenceService:
return None
# #endregion _resolve_environment_id
# [/DEF:_resolve_environment_id:Function]
# #region __init__ [C:3] [TYPE Function]
# @BRIEF Initializes the persistence service.
# @PRE None.
# @POST Service is ready.
# [DEF:__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initializes the persistence service.
# @PRE: None.
# @POST: Service is ready.
def __init__(self):
with belief_scope("TaskPersistenceService.__init__"):
# We use TasksSessionLocal from database.py
pass
# #endregion __init__
# [/DEF:__init__:Function]
# #region persist_task [C:3] [TYPE Function]
# @BRIEF Persists or updates a single task in the database.
# @PRE isinstance(task, Task)
# @POST Task record created or updated in database.
# [DEF:persist_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Persists or updates a single task in the database.
# @PRE: isinstance(task, Task)
# @POST: Task record created or updated in database.
# @PARAM: task (Task) - The task object to persist.
# @SIDE_EFFECT: Writes to task_records table in tasks.db
# @DATA_CONTRACT: Input[Task] -> Model[TaskRecord]
# @RELATION: [CALLS] ->[_resolve_environment_id]
def persist_task(self, task: Task) -> None:
with belief_scope("TaskPersistenceService.persist_task", f"task_id={task.id}"):
log.reason("Persisting task to database", payload={"task_id": task.id})
session: Session = TasksSessionLocal()
try:
record = (
@@ -204,8 +202,8 @@ class TaskPersistenceService:
# Store logs as JSON, converting datetime to string
record.logs = []
for log_entry in task.logs:
log_dict = log_entry.dict()
for log in task.logs:
log_dict = log.dict()
if isinstance(log_dict.get("timestamp"), datetime):
log_dict["timestamp"] = log_dict["timestamp"].isoformat()
# Also clean up any datetimes in context
@@ -215,41 +213,39 @@ class TaskPersistenceService:
# Extract error if failed
if task.status == TaskStatus.FAILED:
for log_entry in reversed(task.logs):
if log_entry.level == "ERROR":
record.error = log_entry.message
for log in reversed(task.logs):
if log.level == "ERROR":
record.error = log.message
break
session.commit()
log.reflect("Task persisted successfully", payload={"task_id": task.id})
except Exception as e:
session.rollback()
log.explore("Failed to persist task", error=str(e), payload={"task_id": task.id})
logger.error(f"Failed to persist task {task.id}: {e}")
finally:
session.close()
# #endregion persist_task
# [/DEF:persist_task:Function]
# #region persist_tasks [C:3] [TYPE Function]
# @BRIEF Persists multiple tasks.
# @PRE isinstance(tasks, list)
# @POST All tasks in list are persisted.
# [DEF:persist_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Persists multiple tasks.
# @PRE: isinstance(tasks, list)
# @POST: All tasks in list are persisted.
# @PARAM: tasks (List[Task]) - The list of tasks to persist.
# @RELATION: [CALLS] ->[persist_task]
def persist_tasks(self, tasks: List[Task]) -> None:
with belief_scope("TaskPersistenceService.persist_tasks"):
log.reason("Persisting multiple tasks", payload={"count": len(tasks)})
for task in tasks:
self.persist_task(task)
log.reflect("All tasks persisted", payload={"count": len(tasks)})
# #endregion persist_tasks
# [/DEF:persist_tasks:Function]
# #region load_tasks [C:3] [TYPE Function]
# @BRIEF Loads tasks from the database.
# @PRE limit is an integer.
# @POST Returns list of Task objects.
# [DEF:load_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Loads tasks from the database.
# @PRE: limit is an integer.
# @POST: Returns list of Task objects.
# @PARAM: limit (int) - Max tasks to load.
# @PARAM: status (Optional[TaskStatus]) - Filter by status.
# @RETURN: List[Task] - The loaded tasks.
@@ -260,7 +256,6 @@ class TaskPersistenceService:
self, limit: int = 100, status: Optional[TaskStatus] = None
) -> List[Task]:
with belief_scope("TaskPersistenceService.load_tasks"):
log.reason("Loading tasks from database", payload={"limit": limit, "status": str(status) if status else None})
session: Session = TasksSessionLocal()
try:
query = session.query(TaskRecord)
@@ -304,20 +299,19 @@ class TaskPersistenceService:
)
loaded_tasks.append(task)
except Exception as e:
log.explore("Failed to reconstruct task from record", error=str(e), payload={"record_id": record.id})
logger.error(f"Failed to reconstruct task {record.id}: {e}")
log.reflect("Tasks loaded from database", payload={"count": len(loaded_tasks)})
return loaded_tasks
finally:
session.close()
# #endregion load_tasks
# [/DEF:load_tasks:Function]
# #region delete_tasks [C:3] [TYPE Function]
# @BRIEF Deletes specific tasks from the database.
# @PRE task_ids is a list of strings.
# @POST Specified task records deleted from database.
# [DEF:delete_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Deletes specific tasks from the database.
# @PRE: task_ids is a list of strings.
# @POST: Specified task records deleted from database.
# @PARAM: task_ids (List[str]) - List of task IDs to delete.
# @SIDE_EFFECT: Deletes rows from task_records table.
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
@@ -325,30 +319,35 @@ class TaskPersistenceService:
if not task_ids:
return
with belief_scope("TaskPersistenceService.delete_tasks"):
log.reason("Deleting tasks from database", payload={"count": len(task_ids)})
session: Session = TasksSessionLocal()
try:
session.query(TaskRecord).filter(TaskRecord.id.in_(task_ids)).delete(
synchronize_session=False
)
session.commit()
log.reflect("Tasks deleted from database", payload={"count": len(task_ids)})
except Exception as e:
session.rollback()
log.explore("Failed to delete tasks", error=str(e))
logger.error(f"Failed to delete tasks: {e}")
finally:
session.close()
# #endregion delete_tasks
# [/DEF:delete_tasks:Function]
# #endregion TaskPersistenceService
# @INVARIANT Log entries are batch-inserted for performance.
# #region TaskLogPersistenceService [C:5] [TYPE Class] [SEMANTICS persistence, service, database, log, sqlalchemy]
# @BRIEF Provides methods to store, query, summarize, and delete task log rows in the task_logs table.
# @PRE: TasksSessionLocal must provide an active SQLAlchemy session, task_id inputs must identify task log rows, LogEntry batches must expose timestamp/level/source/message/metadata fields, and LogFilter inputs must provide pagination and filter attributes used by queries.
# @POST: add_logs commits all provided log entries or rolls back on failure, query methods return TaskLog or LogStats views reconstructed from TaskLogRecord rows, and delete methods remove only log rows matching the supplied task identifiers.
# @SIDE_EFFECT: Opens SQLAlchemy sessions, inserts, reads, aggregates, and deletes task_logs rows, serializes log metadata to JSON, commits or rolls back transactions, and emits error logs on persistence failures.
# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION DEPENDS_ON -> [TaskLogRecord]
# @RELATION DEPENDS_ON -> [TasksSessionLocal]
# @RELATION DEPENDS_ON -> [TaskManager]
# @RELATION DEPENDS_ON -> [EventBus]
# @INVARIANT: Log entries are batch-inserted for performance.
#
# @TEST_CONTRACT: TaskLogPersistenceContract ->
# {
@@ -368,19 +367,21 @@ class TaskLogPersistenceService:
Supports batch inserts, filtering, and statistics.
"""
# #region __init__ [C:3] [TYPE Function]
# @BRIEF Initializes the TaskLogPersistenceService
# @PRE config is provided or defaults are used
# @POST Service is ready for log persistence
# [DEF:__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initializes the TaskLogPersistenceService
# @PRE: config is provided or defaults are used
# @POST: Service is ready for log persistence
def __init__(self, config=None):
pass
# #endregion __init__
# [/DEF:__init__:Function]
# #region add_logs [C:3] [TYPE Function]
# @BRIEF Batch insert log entries for a task.
# @PRE logs is a list of LogEntry objects.
# @POST All logs inserted into task_logs table.
# [DEF:add_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Batch insert log entries for a task.
# @PRE: logs is a list of LogEntry objects.
# @POST: All logs inserted into task_logs table.
# @PARAM: task_id (str) - The task ID.
# @PARAM: logs (List[LogEntry]) - Log entries to insert.
# @SIDE_EFFECT: Writes to task_logs table.
@@ -390,36 +391,34 @@ class TaskLogPersistenceService:
if not logs:
return
with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"):
log_pl.reason("Batch inserting log entries", payload={"task_id": task_id, "count": len(logs)})
session: Session = TasksSessionLocal()
try:
for log_entry in logs:
for log in logs:
record = TaskLogRecord(
task_id=task_id,
timestamp=log_entry.timestamp,
level=log_entry.level,
source=log_entry.source or "system",
message=log_entry.message,
metadata_json=json.dumps(log_entry.metadata)
if log_entry.metadata
timestamp=log.timestamp,
level=log.level,
source=log.source or "system",
message=log.message,
metadata_json=json.dumps(log.metadata)
if log.metadata
else None,
)
session.add(record)
session.commit()
log_pl.reflect("Log entries persisted", payload={"task_id": task_id, "count": len(logs)})
except Exception as e:
session.rollback()
log_pl.explore("Failed to persist log entries", error=str(e), payload={"task_id": task_id})
logger.error(f"Failed to add logs for task {task_id}: {e}")
finally:
session.close()
# #endregion add_logs
# [/DEF:add_logs:Function]
# #region get_logs [C:3] [TYPE Function]
# @BRIEF Query logs for a task with filtering and pagination.
# @PRE task_id is a valid task ID.
# @POST Returns list of TaskLog objects matching filters.
# [DEF:get_logs:Function]
# @COMPLEXITY: 3
# @PURPOSE: Query logs for a task with filtering and pagination.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of TaskLog objects matching filters.
# @PARAM: task_id (str) - The task ID.
# @PARAM: log_filter (LogFilter) - Filter parameters.
# @RETURN: List[TaskLog] - Filtered log entries.
@@ -429,7 +428,6 @@ class TaskLogPersistenceService:
# @RELATION: [DEPENDS_ON] ->[TaskLog]
def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
log_pl.reason("Querying task logs", payload={"task_id": task_id, "filter_level": log_filter.level})
session: Session = TasksSessionLocal()
try:
query = session.query(TaskLogRecord).filter(
@@ -474,17 +472,17 @@ class TaskLogPersistenceService:
)
)
log_pl.reflect("Task logs retrieved", payload={"task_id": task_id, "count": len(logs)})
return logs
finally:
session.close()
# #endregion get_logs
# [/DEF:get_logs:Function]
# #region get_log_stats [C:3] [TYPE Function]
# @BRIEF Get statistics about logs for a task.
# @PRE task_id is a valid task ID.
# @POST Returns LogStats with counts by level and source.
# [DEF:get_log_stats:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
# @PRE: task_id is a valid task ID.
# @POST: Returns LogStats with counts by level and source.
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]
@@ -494,7 +492,6 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"
):
log_pl.reason("Aggregating log statistics", payload={"task_id": task_id})
session: Session = TasksSessionLocal()
try:
# Get total count
@@ -526,19 +523,19 @@ class TaskLogPersistenceService:
by_source = {source: count for source, count in source_counts}
log_pl.reflect("Log statistics computed", payload={"task_id": task_id, "total": total_count})
return LogStats(
total_count=total_count, by_level=by_level, by_source=by_source
)
finally:
session.close()
# #endregion get_log_stats
# [/DEF:get_log_stats:Function]
# #region get_sources [C:3] [TYPE Function]
# @BRIEF Get unique sources for a task's logs.
# @PRE task_id is a valid task ID.
# @POST Returns list of unique source strings.
# [DEF:get_sources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
# @PRE: task_id is a valid task ID.
# @POST: Returns list of unique source strings.
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]
@@ -547,7 +544,6 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.get_sources", f"task_id={task_id}"
):
log_pl.reason("Querying distinct log sources", payload={"task_id": task_id})
session: Session = TasksSessionLocal()
try:
from sqlalchemy import distinct
@@ -557,18 +553,17 @@ class TaskLogPersistenceService:
.filter(TaskLogRecord.task_id == task_id)
.all()
)
result = [s[0] for s in sources]
log_pl.reflect("Log sources retrieved", payload={"task_id": task_id, "sources": result})
return result
return [s[0] for s in sources]
finally:
session.close()
# #endregion get_sources
# [/DEF:get_sources:Function]
# #region delete_logs_for_task [C:3] [TYPE Function]
# @BRIEF Delete all logs for a specific task.
# @PRE task_id is a valid task ID.
# @POST All logs for the task are deleted.
# [DEF:delete_logs_for_task:Function]
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for a specific task.
# @PRE: task_id is a valid task ID.
# @POST: All logs for the task are deleted.
# @PARAM: task_id (str) - The task ID.
# @SIDE_EFFECT: Deletes from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
@@ -576,27 +571,25 @@ class TaskLogPersistenceService:
with belief_scope(
"TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"
):
log_pl.reason("Deleting logs for task", payload={"task_id": task_id})
session: Session = TasksSessionLocal()
try:
session.query(TaskLogRecord).filter(
TaskLogRecord.task_id == task_id
).delete(synchronize_session=False)
session.commit()
log_pl.reflect("Task logs deleted", payload={"task_id": task_id})
except Exception as e:
session.rollback()
log_pl.explore("Failed to delete task logs", error=str(e), payload={"task_id": task_id})
logger.error(f"Failed to delete logs for task {task_id}: {e}")
finally:
session.close()
# #endregion delete_logs_for_task
# [/DEF:delete_logs_for_task:Function]
# #region delete_logs_for_tasks [C:3] [TYPE Function]
# @BRIEF Delete all logs for multiple tasks.
# @PRE task_ids is a list of task IDs.
# @POST All logs for the tasks are deleted.
# [DEF:delete_logs_for_tasks:Function]
# @COMPLEXITY: 3
# @PURPOSE: Delete all logs for multiple tasks.
# @PRE: task_ids is a list of task IDs.
# @POST: All logs for the tasks are deleted.
# @PARAM: task_ids (List[str]) - List of task IDs.
# @SIDE_EFFECT: Deletes rows from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
@@ -604,22 +597,20 @@ class TaskLogPersistenceService:
if not task_ids:
return
with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"):
log_pl.reason("Deleting logs for multiple tasks", payload={"count": len(task_ids)})
session: Session = TasksSessionLocal()
try:
session.query(TaskLogRecord).filter(
TaskLogRecord.task_id.in_(task_ids)
).delete(synchronize_session=False)
session.commit()
log_pl.reflect("Logs deleted for multiple tasks", payload={"count": len(task_ids)})
except Exception as e:
session.rollback()
log_pl.explore("Failed to delete logs for tasks", error=str(e))
logger.error(f"Failed to delete logs for tasks: {e}")
finally:
session.close()
# #endregion delete_logs_for_tasks
# [/DEF:delete_logs_for_tasks:Function]
# #endregion TaskLogPersistenceService
# #endregion TaskPersistenceModule

88
backend/src/core/task_manager/task_logger.py
View File

@@ -1,21 +1,19 @@
# #region TaskLoggerModule [C:2] [TYPE Module] [SEMANTICS task, logger, context, plugin, attribution]
# @BRIEF Provides a dedicated logger for tasks with automatic source attribution.
# @LAYER Core
# @INVARIANT Each TaskLogger instance is bound to a specific task_id and default source.
# @LAYER: Core
# @RELATION DEPENDS_ON -> [TaskManager]
# @RELATION DEPENDS_ON -> [EventBus]
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
# [SECTION: IMPORTS]
from typing import Dict, Any, Optional, Callable
# [/SECTION]
# #region TaskLogger [C:2] [TYPE Class] [SEMANTICS logger, task, source, attribution]
# @BRIEF A wrapper around TaskManager._add_log that carries task_id and source context.
# @INVARIANT All log calls include the task_id and source.
# @RELATION DEPENDS_ON -> [TaskManager]
# @RELATION DEPENDS_ON -> [EventBus]
# @RELATION USED_BY -> [TaskContext]
# @INVARIANT: All log calls include the task_id and source.
# @UX_STATE: Idle -> Logging -> (system records log)
#
# @TEST_CONTRACT: TaskLoggerContract ->
@@ -45,10 +43,10 @@ class TaskLogger:
api_logger.info("Fetching dashboards")
"""
# #region __init__ [TYPE Function]
# @BRIEF Initialize the TaskLogger with task context.
# @PRE add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
# @POST TaskLogger is ready to log messages.
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskLogger with task context.
# @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
# @POST: TaskLogger is ready to log messages.
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
# @PARAM: source (str) - Default source for logs (default: "plugin").
@@ -57,12 +55,12 @@ class TaskLogger:
self._add_log = add_log_fn
self._default_source = source
# #endregion __init__
# [/DEF:__init__:Function]
# #region with_source [TYPE Function]
# @BRIEF Create a sub-logger with a different default source.
# @PRE source is a non-empty string.
# @POST Returns new TaskLogger with the same task_id but different source.
# [DEF:with_source:Function]
# @PURPOSE: Create a sub-logger with a different default source.
# @PRE: source is a non-empty string.
# @POST: Returns new TaskLogger with the same task_id but different source.
# @PARAM: source (str) - New default source.
# @RETURN: TaskLogger - New logger instance.
def with_source(self, source: str) -> "TaskLogger":
@@ -71,12 +69,12 @@ class TaskLogger:
task_id=self._task_id, add_log_fn=self._add_log, source=source
)
# #endregion with_source
# [/DEF:with_source:Function]
# #region _log [TYPE Function]
# @BRIEF Internal method to log a message at a given level.
# @PRE level is a valid log level string.
# @POST Log entry added via add_log_fn.
# [DEF:_log:Function]
# @PURPOSE: Internal method to log a message at a given level.
# @PRE: level is a valid log level string.
# @POST: Log entry added via add_log_fn.
# @PARAM: level (str) - Log level (DEBUG, INFO, WARNING, ERROR).
# @PARAM: message (str) - Log message.
# @PARAM: source (Optional[str]) - Override source for this log entry.
@@ -98,12 +96,12 @@ class TaskLogger:
metadata=metadata,
)
# #endregion _log
# [/DEF:_log:Function]
# #region debug [TYPE Function]
# @BRIEF Log a DEBUG level message.
# @PRE message is a string.
# @POST Log entry added via internally with DEBUG level.
# [DEF:debug:Function]
# @PURPOSE: Log a DEBUG level message.
# @PRE: message is a string.
# @POST: Log entry added via internally with DEBUG level.
# @PARAM: message (str) - Log message.
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
@@ -115,12 +113,12 @@ class TaskLogger:
) -> None:
self._log("DEBUG", message, source, metadata)
# #endregion debug
# [/DEF:debug:Function]
# #region info [TYPE Function]
# @BRIEF Log an INFO level message.
# @PRE message is a string.
# @POST Log entry added internally with INFO level.
# [DEF:info:Function]
# @PURPOSE: Log an INFO level message.
# @PRE: message is a string.
# @POST: Log entry added internally with INFO level.
# @PARAM: message (str) - Log message.
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
@@ -132,12 +130,12 @@ class TaskLogger:
) -> None:
self._log("INFO", message, source, metadata)
# #endregion info
# [/DEF:info:Function]
# #region warning [TYPE Function]
# @BRIEF Log a WARNING level message.
# @PRE message is a string.
# @POST Log entry added internally with WARNING level.
# [DEF:warning:Function]
# @PURPOSE: Log a WARNING level message.
# @PRE: message is a string.
# @POST: Log entry added internally with WARNING level.
# @PARAM: message (str) - Log message.
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
@@ -149,12 +147,12 @@ class TaskLogger:
) -> None:
self._log("WARNING", message, source, metadata)
# #endregion warning
# [/DEF:warning:Function]
# #region error [TYPE Function]
# @BRIEF Log an ERROR level message.
# @PRE message is a string.
# @POST Log entry added internally with ERROR level.
# [DEF:error:Function]
# @PURPOSE: Log an ERROR level message.
# @PRE: message is a string.
# @POST: Log entry added internally with ERROR level.
# @PARAM: message (str) - Log message.
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
@@ -166,12 +164,12 @@ class TaskLogger:
) -> None:
self._log("ERROR", message, source, metadata)
# #endregion error
# [/DEF:error:Function]
# #region progress [TYPE Function]
# @BRIEF Log a progress update with percentage.
# @PRE percent is between 0 and 100.
# @POST Log entry with progress metadata added.
# [DEF:progress:Function]
# @PURPOSE: Log a progress update with percentage.
# @PRE: percent is between 0 and 100.
# @POST: Log entry with progress metadata added.
# @PARAM: message (str) - Progress message.
# @PARAM: percent (float) - Progress percentage (0-100).
# @PARAM: source (Optional[str]) - Override source.
@@ -182,7 +180,7 @@ class TaskLogger:
metadata = {"progress": min(100, max(0, percent))}
self._log("INFO", message, source, metadata)
# #endregion progress
# [/DEF:progress:Function]
# #endregion TaskLogger

200
backend/src/core/utils/async_network.py
View File

@@ -1,22 +1,20 @@
# #region AsyncNetworkModule [C:5] [TYPE Module] [SEMANTICS network, httpx, async, superset, authentication, cache]
# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER Infra
# @PRE Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @INVARIANT Async client reuses cached auth tokens per environment credentials and invalidates on 401.
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
#
# @BRIEF Provides async Superset API client with shared auth-token cache to avoid per-request re-login.
# @LAYER: Infra
# @PRE: Config payloads contain a Superset base URL and authentication fields needed for login.
# @POST: Async network clients reuse cached auth tokens and expose stable async request/error translation flow.
# @SIDE_EFFECT: Performs upstream HTTP I/O and mutates process-local auth cache entries.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> Output[authenticated async Superset HTTP interactions]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# @INVARIANT: Async client reuses cached auth tokens per environment credentials and invalidates on 401.
# [SECTION: IMPORTS]
from typing import Optional, Dict, Any, Union
import asyncio
import httpx
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
from .network import (
AuthenticationError,
DashboardNotFoundError,
@@ -25,9 +23,7 @@ from .network import (
SupersetAPIError,
SupersetAuthCache,
)
# [/SECTION]
log = MarkerLogger("AsyncNetwork")
# #region AsyncAPIClient [C:3] [TYPE Class]
# @BRIEF Async Superset API client backed by httpx.AsyncClient with shared auth cache.
@@ -38,13 +34,14 @@ class AsyncAPIClient:
DEFAULT_TIMEOUT = 30
_auth_locks: Dict[tuple[str, str, bool], asyncio.Lock] = {}
# #region AsyncAPIClient.__init__ [C:3] [TYPE Function]
# @BRIEF Initialize async API client for one environment.
# @PRE config contains base_url and auth payload.
# @POST Client is ready for async request/authentication flow.
# @DATA_CONTRACT Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION CALLS -> [AsyncAPIClient._normalize_base_url]
# @RELATION DEPENDS_ON -> [SupersetAuthCache]
# [DEF:AsyncAPIClient.__init__:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize async API client for one environment.
# @PRE: config contains base_url and auth payload.
# @POST: Client is ready for async request/authentication flow.
# @DATA_CONTRACT: Input[config: Dict[str, Any]] -> self._auth_cache_key[str]
# @RELATION: [CALLS] ->[AsyncAPIClient._normalize_base_url]
# @RELATION: [DEPENDS_ON] ->[SupersetAuthCache]
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
@@ -63,21 +60,23 @@ class AsyncAPIClient:
verify_ssl,
)
# #endregion AsyncAPIClient.__init__
# [/DEF:AsyncAPIClient.__init__:Function]
# #region AsyncAPIClient._normalize_base_url [C:1] [TYPE Function]
# @BRIEF Normalize base URL for Superset API root construction.
# @POST Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
# [DEF:AsyncAPIClient._normalize_base_url:Function]
# @COMPLEXITY: 1
# @PURPOSE: Normalize base URL for Superset API root construction.
# @POST: Returns canonical base URL without trailing slash and duplicate /api/v1 suffix.
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
normalized = normalized[:-len("/api/v1")]
return normalized.rstrip("/")
# #endregion AsyncAPIClient._normalize_base_url
# [/DEF:AsyncAPIClient._normalize_base_url:Function]
# #region AsyncAPIClient._build_api_url [C:1] [TYPE Function]
# @BRIEF Build full API URL from relative Superset endpoint.
# @POST Returns absolute URL for upstream request.
# [DEF:AsyncAPIClient._build_api_url:Function]
# @COMPLEXITY: 1
# @PURPOSE: Build full API URL from relative Superset endpoint.
# @POST: Returns absolute URL for upstream request.
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -87,11 +86,12 @@ class AsyncAPIClient:
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
return f"{self.base_url}{normalized_endpoint}"
return f"{self.api_base_url}{normalized_endpoint}"
# #endregion AsyncAPIClient._build_api_url
# [/DEF:AsyncAPIClient._build_api_url:Function]
# #region AsyncAPIClient._get_auth_lock [C:1] [TYPE Function]
# @BRIEF Return per-cache-key async lock to serialize fresh login attempts.
# @POST Returns stable asyncio.Lock instance.
# [DEF:AsyncAPIClient._get_auth_lock:Function]
# @COMPLEXITY: 1
# @PURPOSE: Return per-cache-key async lock to serialize fresh login attempts.
# @POST: Returns stable asyncio.Lock instance.
@classmethod
def _get_auth_lock(cls, cache_key: tuple[str, str, bool]) -> asyncio.Lock:
existing_lock = cls._auth_locks.get(cache_key)
@@ -100,40 +100,24 @@ class AsyncAPIClient:
created_lock = asyncio.Lock()
cls._auth_locks[cache_key] = created_lock
return created_lock
# #endregion AsyncAPIClient._get_auth_lock
# [/DEF:AsyncAPIClient._get_auth_lock:Function]
# #region AsyncAPIClient.authenticate [C:3] [TYPE Function]
# @BRIEF Authenticate against Superset and cache access/csrf tokens.
# @POST Client tokens are populated and reusable across requests.
# @SIDE_EFFECT Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT None -> Output[Dict[str, str]]
# @RELATION CALLS -> [SupersetAuthCache.get]
# @RELATION CALLS -> [SupersetAuthCache.set]
# @RELATION CALLS -> [AsyncAPIClient._get_auth_lock]
# [DEF:AsyncAPIClient.authenticate:Function]
# @COMPLEXITY: 3
# @PURPOSE: Authenticate against Superset and cache access/csrf tokens.
# @POST: Client tokens are populated and reusable across requests.
# @SIDE_EFFECT: Performs network requests to Superset authentication endpoints.
# @DATA_CONTRACT: None -> Output[Dict[str, str]]
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RELATION: [CALLS] ->[AsyncAPIClient._get_auth_lock]
async def authenticate(self) -> Dict[str, str]:
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
# Always fetch a fresh CSRF token to establish session cookie in this httpx client.
# The cached access_token may still be valid; we avoid a full POST login.
try:
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = await self._client.get(
csrf_url,
headers={"Authorization": f"Bearer {self._tokens['access_token']}"},
)
csrf_response.raise_for_status()
self._tokens["csrf_token"] = csrf_response.json()["result"]
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
self._authenticated = True
log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url})
return self._tokens
except Exception as csrf_err:
log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err))
SupersetAuthCache.invalidate(self._auth_cache_key)
self._tokens = {}
self._authenticated = False
self._authenticated = True
app_logger.info("[async_authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
auth_lock = self._get_auth_lock(self._auth_cache_key)
async with auth_lock:
@@ -141,11 +125,11 @@ class AsyncAPIClient:
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
self._authenticated = True
log.reason("Reusing cached Superset auth tokens (after lock wait)", payload={"base_url": self.base_url})
app_logger.info("[async_authenticate][CacheHitAfterWait] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
with belief_scope("AsyncAPIClient.authenticate"):
log.reason("Performing full authentication", payload={"base_url": self.base_url})
app_logger.info("[async_authenticate][Enter] Authenticating to %s", self.base_url)
try:
login_url = f"{self.api_base_url}/security/login"
response = await self._client.post(login_url, json=self.auth)
@@ -165,7 +149,7 @@ class AsyncAPIClient:
}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reflect("Authenticated successfully")
app_logger.info("[async_authenticate][Exit] Authenticated successfully.")
return self._tokens
except httpx.HTTPStatusError as exc:
SupersetAuthCache.invalidate(self._auth_cache_key)
@@ -179,12 +163,13 @@ class AsyncAPIClient:
except (httpx.HTTPError, KeyError) as exc:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {exc}") from exc
# #endregion AsyncAPIClient.authenticate
# [/DEF:AsyncAPIClient.authenticate:Function]
# #region AsyncAPIClient.get_headers [C:3] [TYPE Function]
# @BRIEF Return authenticated Superset headers for async requests.
# @POST Headers include Authorization and CSRF tokens.
# @RELATION CALLS -> [AsyncAPIClient.authenticate]
# [DEF:AsyncAPIClient.get_headers:Function]
# @COMPLEXITY: 3
# @PURPOSE: Return authenticated Superset headers for async requests.
# @POST: Headers include Authorization and CSRF tokens.
# @RELATION: [CALLS] ->[AsyncAPIClient.authenticate]
async def get_headers(self) -> Dict[str, str]:
if not self._authenticated:
await self.authenticate()
@@ -194,15 +179,16 @@ class AsyncAPIClient:
"Referer": self.base_url,
"Content-Type": "application/json",
}
# #endregion AsyncAPIClient.get_headers
# [/DEF:AsyncAPIClient.get_headers:Function]
# #region AsyncAPIClient.request [C:3] [TYPE Function]
# @BRIEF Perform one authenticated async Superset API request.
# @POST Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT Performs network I/O.
# @RELATION CALLS -> [AsyncAPIClient.get_headers]
# @RELATION CALLS -> [AsyncAPIClient._handle_http_error]
# @RELATION CALLS -> [AsyncAPIClient._handle_network_error]
# [DEF:AsyncAPIClient.request:Function]
# @COMPLEXITY: 3
# @PURPOSE: Perform one authenticated async Superset API request.
# @POST: Returns JSON payload or raw httpx.Response when raw_response=true.
# @SIDE_EFFECT: Performs network I/O.
# @RELATION: [CALLS] ->[AsyncAPIClient.get_headers]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_http_error]
# @RELATION: [CALLS] ->[AsyncAPIClient._handle_network_error]
async def request(
self,
method: str,
@@ -230,18 +216,19 @@ class AsyncAPIClient:
self._handle_http_error(exc, endpoint)
except httpx.HTTPError as exc:
self._handle_network_error(exc, full_url)
# #endregion AsyncAPIClient.request
# [/DEF:AsyncAPIClient.request:Function]
# #region AsyncAPIClient._handle_http_error [C:3] [TYPE Function]
# @BRIEF Translate upstream HTTP errors into stable domain exceptions.
# @POST Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT Input[httpx.HTTPStatusError] -> Exception
# @RELATION CALLS -> [AsyncAPIClient._is_dashboard_endpoint]
# @RELATION DEPENDS_ON -> [DashboardNotFoundError]
# @RELATION DEPENDS_ON -> [SupersetAPIError]
# @RELATION DEPENDS_ON -> [PermissionDeniedError]
# @RELATION DEPENDS_ON -> [AuthenticationError]
# @RELATION DEPENDS_ON -> [NetworkError]
# [DEF:AsyncAPIClient._handle_http_error:Function]
# @COMPLEXITY: 3
# @PURPOSE: Translate upstream HTTP errors into stable domain exceptions.
# @POST: Raises domain-specific exception for caller flow control.
# @DATA_CONTRACT: Input[httpx.HTTPStatusError] -> Exception
# @RELATION: [CALLS] ->[AsyncAPIClient._is_dashboard_endpoint]
# @RELATION: [DEPENDS_ON] ->[DashboardNotFoundError]
# @RELATION: [DEPENDS_ON] ->[SupersetAPIError]
# @RELATION: [DEPENDS_ON] ->[PermissionDeniedError]
# @RELATION: [DEPENDS_ON] ->[AuthenticationError]
# @RELATION: [DEPENDS_ON] ->[NetworkError]
def _handle_http_error(self, exc: httpx.HTTPStatusError, endpoint: str) -> None:
with belief_scope("AsyncAPIClient._handle_http_error"):
status_code = exc.response.status_code
@@ -261,11 +248,12 @@ class AsyncAPIClient:
if status_code == 401:
raise AuthenticationError() from exc
raise SupersetAPIError(f"API Error {status_code}: {exc.response.text}") from exc
# #endregion AsyncAPIClient._handle_http_error
# [/DEF:AsyncAPIClient._handle_http_error:Function]
# #region AsyncAPIClient._is_dashboard_endpoint [C:2] [TYPE Function]
# @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @POST Returns true only for dashboard-specific endpoints.
# [DEF:AsyncAPIClient._is_dashboard_endpoint:Function]
# @COMPLEXITY: 2
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @POST: Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -278,13 +266,14 @@ class AsyncAPIClient:
if normalized_endpoint.startswith("/api/v1/"):
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
# #endregion AsyncAPIClient._is_dashboard_endpoint
# [/DEF:AsyncAPIClient._is_dashboard_endpoint:Function]
# #region AsyncAPIClient._handle_network_error [C:3] [TYPE Function]
# @BRIEF Translate generic httpx errors into NetworkError.
# @POST Raises NetworkError with URL context.
# @DATA_CONTRACT Input[httpx.HTTPError] -> NetworkError
# @RELATION DEPENDS_ON -> [NetworkError]
# [DEF:AsyncAPIClient._handle_network_error:Function]
# @COMPLEXITY: 3
# @PURPOSE: Translate generic httpx errors into NetworkError.
# @POST: Raises NetworkError with URL context.
# @DATA_CONTRACT: Input[httpx.HTTPError] -> NetworkError
# @RELATION: [DEPENDS_ON] ->[NetworkError]
def _handle_network_error(self, exc: httpx.HTTPError, url: str) -> None:
with belief_scope("AsyncAPIClient._handle_network_error"):
if isinstance(exc, httpx.TimeoutException):
@@ -294,14 +283,17 @@ class AsyncAPIClient:
else:
message = f"Unknown network error: {exc}"
raise NetworkError(message, url=url) from exc
# #endregion AsyncAPIClient._handle_network_error
# [/DEF:AsyncAPIClient._handle_network_error:Function]
# #region AsyncAPIClient.aclose [C:3] [TYPE Function]
# @BRIEF Close underlying httpx client.
# @POST Client resources are released.
# @SIDE_EFFECT Closes network connections.
# @RELATION DEPENDS_ON -> [AsyncAPIClient.__init__]
# [DEF:AsyncAPIClient.aclose:Function]
# @COMPLEXITY: 3
# @PURPOSE: Close underlying httpx client.
# @POST: Client resources are released.
# @SIDE_EFFECT: Closes network connections.
# @RELATION: [DEPENDS_ON] ->[AsyncAPIClient.__init__]
async def aclose(self) -> None:
await self._client.aclose()
# #endregion AsyncAPIClient.aclose
# [/DEF:AsyncAPIClient.aclose:Function]
# #endregion AsyncAPIClient
# #endregion AsyncNetworkModule

99
backend/src/core/utils/dataset_mapper.py
View File

@@ -1,37 +1,32 @@
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, mapping, postgresql, xlsx, superset]
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [backend.core.superset_client]
# @RELATION DEPENDS_ON -> [pandas]
# @RELATION DEPENDS_ON -> [psycopg2]
#
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> backend.core.superset_client
# @RELATION DEPENDS_ON -> pandas
# @RELATION DEPENDS_ON -> psycopg2
# @PUBLIC_API: DatasetMapper
# [SECTION: IMPORTS]
import pandas as pd # type: ignore
import psycopg2 # type: ignore
from typing import Dict, Optional, Any
from ..logger import belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("DatasetMapper")
from ..logger import logger as app_logger, belief_scope
# #region DatasetMapper [TYPE Class]
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
class DatasetMapper:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the mapper.
# @POST Объект DatasetMapper инициализирован.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapper.
# @POST: Объект DatasetMapper инициализирован.
def __init__(self):
pass
# #endregion __init__
# [/DEF:__init__:Function]
# #region get_postgres_comments [TYPE Function]
# @BRIEF Извлекает комментарии к колонкам из системного каталога PostgreSQL.
# @PRE db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
# @PRE table_name и table_schema должны быть строками.
# @POST Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
# [DEF:get_postgres_comments:Function]
# @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.
# @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
# @PRE: table_name и table_schema должны быть строками.
# @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
# @THROW: Exception - При ошибках подключения или выполнения запроса к БД.
# @PARAM: db_config (Dict) - Конфигурация для подключения к БД.
# @PARAM: table_name (str) - Имя таблицы.
@@ -39,7 +34,7 @@ class DatasetMapper:
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
def get_postgres_comments(self, db_config: Dict, table_name: str, table_schema: str) -> Dict[str, str]:
with belief_scope("Fetch comments from PostgreSQL"):
log.reason("Fetching comments from PostgreSQL", payload={"table_schema": table_schema, "table_name": table_name})
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
query = f"""
SELECT
cols.column_name,
@@ -85,42 +80,42 @@ class DatasetMapper:
for row in cursor.fetchall():
if row[1]:
comments[row[0]] = row[1]
log.reflect("PostgreSQL comments fetched", payload={"count": len(comments)})
app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments))
except Exception as e:
log.explore("Failed to fetch PostgreSQL comments", payload={"table_schema": table_schema, "table_name": table_name}, error=str(e))
app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True)
raise
return comments
# #endregion get_postgres_comments
# [/DEF:get_postgres_comments:Function]
# #region load_excel_mappings [TYPE Function]
# @BRIEF Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
# @PRE file_path должен указывать на существующий XLSX файл.
# @POST Возвращается словарь с меппингами из файла.
# [DEF:load_excel_mappings:Function]
# @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
# @PRE: file_path должен указывать на существующий XLSX файл.
# @POST: Возвращается словарь с меппингами из файла.
# @THROW: Exception - При ошибках чтения файла или парсинга.
# @PARAM: file_path (str) - Путь к XLSX файлу.
# @RETURN: Dict[str, str] - Словарь с меппингами.
def load_excel_mappings(self, file_path: str) -> Dict[str, str]:
with belief_scope("Load mappings from Excel"):
log.reason("Loading mappings from Excel", payload={"file_path": file_path})
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
try:
df = pd.read_excel(file_path)
mappings = df.set_index('column_name')['verbose_name'].to_dict()
log.reflect("Excel mappings loaded", payload={"count": len(mappings)})
app_logger.info("[load_excel_mappings][Success] Loaded %d mappings.", len(mappings))
return mappings
except Exception as e:
log.explore("Failed to load Excel mappings", payload={"file_path": file_path}, error=str(e))
app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True)
raise
# #endregion load_excel_mappings
# [/DEF:load_excel_mappings:Function]
# #region run_mapping [TYPE Function]
# @BRIEF Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
# @PRE superset_client должен быть авторизован.
# @PRE dataset_id должен быть существующим ID в Superset.
# @POST Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION CALLS -> [self.get_postgres_comments]
# @RELATION CALLS -> [self.load_excel_mappings]
# @RELATION CALLS -> [superset_client.get_dataset]
# @RELATION CALLS -> [superset_client.update_dataset]
# [DEF:run_mapping:Function]
# @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
# @PRE: superset_client должен быть авторизован.
# @PRE: dataset_id должен быть существующим ID в Superset.
# @POST: Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION: CALLS -> self.get_postgres_comments
# @RELATION: CALLS -> self.load_excel_mappings
# @RELATION: CALLS -> superset_client.get_dataset
# @RELATION: CALLS -> superset_client.update_dataset
# @PARAM: superset_client (Any) - Клиент Superset.
# @PARAM: dataset_id (int) - ID датасета для обновления.
# @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').
@@ -130,20 +125,18 @@ class DatasetMapper:
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: Optional[Dict] = None, excel_path: Optional[str] = None, table_name: Optional[str] = None, table_schema: Optional[str] = None):
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
log.reason("Starting dataset mapping", payload={"dataset_id": dataset_id, "source": source})
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
mappings: Dict[str, str] = {}
try:
if source in ['postgres', 'both']:
if not (postgres_config and table_name and table_schema):
raise ValueError("Postgres config is required.")
assert postgres_config and table_name and table_schema, "Postgres config is required."
mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))
if source in ['excel', 'both']:
if not excel_path:
raise ValueError("Excel path is required.")
assert excel_path, "Excel path is required."
mappings.update(self.load_excel_mappings(excel_path))
if source not in ['postgres', 'excel', 'both']:
log.explore("Invalid mapping source", payload={"source": source}, error=f"Invalid source: {source}")
app_logger.error("[run_mapping][Failure] Invalid source: %s.", source)
return
dataset_response = superset_client.get_dataset(dataset_id)
@@ -228,14 +221,14 @@ class DatasetMapper:
payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}
superset_client.update_dataset(dataset_id, payload_for_update)
log.reflect("Dataset verbose_name updated", payload={"dataset_id": dataset_id})
app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id)
else:
log.reflect("No changes in verbose_name, skipping update", payload={"dataset_id": dataset_id})
app_logger.info("[run_mapping][State] No changes in columns' verbose_name, skipping update.")
except (FileNotFoundError, Exception) as e:
log.explore("Dataset mapping failed", payload={"dataset_id": dataset_id, "source": source}, error=str(e))
except (AssertionError, FileNotFoundError, Exception) as e:
app_logger.error("[run_mapping][Failure] %s", e, exc_info=True)
return
# #endregion run_mapping
# [/DEF:run_mapping:Function]
# #endregion DatasetMapper
# #endregion DatasetMapperModule
# #endregion DatasetMapperModule

208
backend/src/core/utils/fileio.py
View File

@@ -1,13 +1,11 @@
# #region FileIO [TYPE Module]
# #region FileIO [TYPE Module] [SEMANTICS file, io, zip, yaml, temp, archive, utility]
#
# @TIER: STANDARD
# @SEMANTICS: file, io, zip, yaml, temp, archive, utility
# @PURPOSE: Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.
# @BRIEF Предоставляет набор утилит для управления файловыми операциями, включая работу с временными файлами, архивами ZIP, файлами YAML и очистку директорий.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: create_temp_file, remove_empty_directories, read_dashboard_from_disk, calculate_crc32, RetentionPolicy, archive_exports, save_and_unpack_dashboard, update_yamls, create_dashboard_export, sanitize_filename, get_filename_from_headers, consolidate_archive_folders
# [SECTION: IMPORTS]
import os
import re
import zipfile
@@ -20,10 +18,6 @@ import shutil
import zlib
from dataclasses import dataclass
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("FileIO")
# #region InvalidZipFormatError [TYPE Class]
# @BRIEF Exception raised when a file is not a valid ZIP archive.
@@ -33,13 +27,9 @@ class InvalidZipFormatError(Exception):
# #region create_temp_file [TYPE Function]
# @BRIEF Контекстный менеджер для создания временного файла или директории с гарантированным удалением.
# @PRE suffix должен быть строкой, определяющей тип ресурса.
# @POST Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @PARAM: content (Optional[bytes]) - Бинарное содержимое для записи во временный файл.
# @PARAM: suffix (str) - Суффикс ресурса. Если `.dir`, создается директория.
# @PARAM: mode (str) - Режим записи в файл (e.g., 'wb').
# @PRE: suffix должен быть строкой, определяющей тип ресурса.
# @POST: Временный ресурс создан и путь к нему возвращен; ресурс удален после выхода из контекста.
# @YIELDS: Path - Путь к временному ресурсу.
# @THROW: IOError - При ошибках создания ресурса.
@contextmanager
def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode: str = 'wb', dry_run = False) -> Generator[Path, None, None]:
with belief_scope("Create temporary resource"):
@@ -49,7 +39,7 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode
if is_dir:
with tempfile.TemporaryDirectory(suffix=suffix) as temp_dir:
resource_path = Path(temp_dir)
log.reason("Created temporary directory", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][State] Created temporary directory: %s", resource_path)
yield resource_path
else:
fd, temp_path_str = tempfile.mkstemp(suffix=suffix)
@@ -57,70 +47,63 @@ def create_temp_file(content: Optional[bytes] = None, suffix: str = ".zip", mode
os.close(fd)
if content:
resource_path.write_bytes(content)
log.reason("Created temporary file", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][State] Created temporary file: %s", resource_path)
yield resource_path
finally:
if resource_path and resource_path.exists() and not dry_run:
try:
if resource_path.is_dir():
shutil.rmtree(resource_path)
log.reflect("Cleaned up temporary directory", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][Cleanup] Removed temporary directory: %s", resource_path)
else:
resource_path.unlink()
log.reflect("Cleaned up temporary file", payload={"path": str(resource_path)})
app_logger.debug("[create_temp_file][Cleanup] Removed temporary file: %s", resource_path)
except OSError as e:
log.explore("Error during temp resource cleanup", payload={"path": str(resource_path)}, error=str(e))
app_logger.error("[create_temp_file][Failure] Error during cleanup of %s: %s", resource_path, e)
# #endregion create_temp_file
# #region remove_empty_directories [TYPE Function]
# @BRIEF Рекурсивно удаляет все пустые поддиректории, начиная с указанного пути.
# @PRE root_dir должен быть путем к существующей директории.
# @POST Все пустые поддиректории удалены, возвращено их количество.
# @PARAM: root_dir (str) - Путь к корневой директории для очистки.
# @RETURN: int - Количество удаленных директорий.
# @PRE: root_dir должен быть путем к существующей директории.
# @POST: Все пустые поддиректории удалены, возвращено их количество.
def remove_empty_directories(root_dir: str) -> int:
with belief_scope(f"Remove empty directories in {root_dir}"):
app_logger.info("[remove_empty_directories][Enter] Starting cleanup of empty directories in %s", root_dir)
removed_count = 0
if not os.path.isdir(root_dir):
log.explore("Empty directory cleanup skipped - directory not found", payload={"root_dir": root_dir}, error=f"Directory not found: {root_dir}")
app_logger.error("[remove_empty_directories][Failure] Directory not found: %s", root_dir)
return 0
for current_dir, _, _ in os.walk(root_dir, topdown=False):
if not os.listdir(current_dir):
try:
os.rmdir(current_dir)
removed_count += 1
log.reason("Removed empty directory", payload={"directory": current_dir})
app_logger.info("[remove_empty_directories][State] Removed empty directory: %s", current_dir)
except OSError as e:
log.explore("Failed to remove empty directory", payload={"directory": current_dir}, error=str(e))
log.reflect("Empty directory cleanup complete", payload={"removed_count": removed_count})
app_logger.error("[remove_empty_directories][Failure] Failed to remove %s: %s", current_dir, e)
app_logger.info("[remove_empty_directories][Exit] Removed %d empty directories.", removed_count)
return removed_count
# #endregion remove_empty_directories
# #region read_dashboard_from_disk [TYPE Function]
# @BRIEF Читает бинарное содержимое файла с диска.
# @PRE file_path должен указывать на существующий файл.
# @POST Возвращает байты содержимого и имя файла.
# @PARAM: file_path (str) - Путь к файлу.
# @RETURN: Tuple[bytes, str] - Кортеж (содержимое, имя файла).
# @THROW: FileNotFoundError - Если файл не найден.
# @PRE: file_path должен указывать на существующий файл.
# @POST: Возвращает байты содержимого и имя файла.
def read_dashboard_from_disk(file_path: str) -> Tuple[bytes, str]:
with belief_scope(f"Read dashboard from {file_path}"):
path = Path(file_path)
assert path.is_file(), f"Файл дашборда не найден: {file_path}"
log.reason("Reading dashboard file from disk", payload={"file_path": file_path})
app_logger.info("[read_dashboard_from_disk][Enter] Reading file: %s", file_path)
content = path.read_bytes()
if not content:
log.explore("Dashboard file is empty", payload={"file_path": file_path}, error="Empty dashboard file")
app_logger.warning("[read_dashboard_from_disk][Warning] File is empty: %s", file_path)
return content, path.name
# #endregion read_dashboard_from_disk
# #region calculate_crc32 [TYPE Function]
# @BRIEF Вычисляет контрольную сумму CRC32 для файла.
# @PRE file_path должен быть объектом Path к существующему файлу.
# @POST Возвращает 8-значную hex-строку CRC32.
# @PARAM: file_path (Path) - Путь к файлу.
# @RETURN: str - 8-значное шестнадцатеричное представление CRC32.
# @THROW: IOError - При ошибках чтения файла.
# @PRE: file_path должен быть объектом Path к существующему файлу.
# @POST: Возвращает 8-значную hex-строку CRC32.
def calculate_crc32(file_path: Path) -> str:
with belief_scope(f"Calculate CRC32 for {file_path}"):
with open(file_path, 'rb') as f:
@@ -128,7 +111,6 @@ def calculate_crc32(file_path: Path) -> str:
return f"{crc32_value:08x}"
# #endregion calculate_crc32
# [SECTION: DATA_CLASSES]
# #region RetentionPolicy [TYPE DataClass]
# @BRIEF Определяет политику хранения для архивов (ежедневные, еженедельные, ежемесячные).
@dataclass
@@ -137,35 +119,31 @@ class RetentionPolicy:
weekly: int = 4
monthly: int = 12
# #endregion RetentionPolicy
# [/SECTION]
# #region archive_exports [TYPE Function]
# @BRIEF Управляет архивом экспортированных файлов, применяя политику хранения и дедупликацию.
# @PRE output_dir должен быть путем к существующей директории.
# @POST Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> [apply_retention_policy]
# @RELATION CALLS -> [calculate_crc32]
# @PARAM: output_dir (str) - Директория с архивами.
# @PARAM: policy (RetentionPolicy) - Политика хранения.
# @PARAM: deduplicate (bool) - Флаг для включения удаления дубликатов по CRC32.
# @PRE: output_dir должен быть путем к существующей директории.
# @POST: Старые или дублирующиеся архивы удалены согласно политике.
# @RELATION CALLS -> apply_retention_policy
# @RELATION CALLS -> calculate_crc32
def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool = False) -> None:
with belief_scope(f"Archive exports in {output_dir}"):
output_path = Path(output_dir)
if not output_path.is_dir():
log.explore("Archive directory not found", payload={"output_dir": output_dir}, error=f"Archive directory not found: {output_dir}")
app_logger.warning("[archive_exports][Skip] Archive directory not found: %s", output_dir)
return
log.reason("Managing archive exports", payload={"output_dir": output_dir})
app_logger.info("[archive_exports][Enter] Managing archive in %s", output_dir)
# 1. Collect all zip files
zip_files = list(output_path.glob("*.zip"))
if not zip_files:
log.reason("No zip files found in archive directory", payload={"output_dir": output_dir})
app_logger.info("[archive_exports][State] No zip files found in %s", output_dir)
return
# 2. Deduplication
if deduplicate:
log.reason("Starting archive deduplication")
app_logger.info("[archive_exports][State] Starting deduplication...")
checksums = {}
files_to_remove = []
@@ -177,19 +155,19 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
crc = calculate_crc32(file_path)
if crc in checksums:
files_to_remove.append(file_path)
log.reason("Duplicate archive found (same checksum)", payload={"file": file_path.name, "original": checksums[crc].name})
app_logger.debug("[archive_exports][State] Duplicate found: %s (same as %s)", file_path.name, checksums[crc].name)
else:
checksums[crc] = file_path
except Exception as e:
log.explore("Failed to calculate CRC32 for archive", payload={"file": str(file_path)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to calculate CRC32 for %s: %s", file_path, e)
for f in files_to_remove:
try:
f.unlink()
zip_files.remove(f)
log.reason("Removed duplicate archive", payload={"file": f.name})
app_logger.info("[archive_exports][State] Removed duplicate: %s", f.name)
except OSError as e:
log.explore("Failed to remove duplicate archive", payload={"file": str(f)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to remove duplicate %s: %s", f, e)
# 3. Retention Policy
files_with_dates = []
@@ -217,18 +195,15 @@ def archive_exports(output_dir: str, policy: RetentionPolicy, deduplicate: bool
if file_path not in files_to_keep:
try:
file_path.unlink()
log.reason("Removed archive by retention policy", payload={"file": file_path.name})
app_logger.info("[archive_exports][State] Removed by retention policy: %s", file_path.name)
except OSError as e:
log.explore("Failed to remove archive by retention policy", payload={"file": str(file_path)}, error=str(e))
app_logger.error("[archive_exports][Failure] Failed to remove %s: %s", file_path, e)
# #endregion archive_exports
# #region apply_retention_policy [TYPE Function]
# @BRIEF (Helper) Применяет политику хранения к списку файлов, возвращая те, что нужно сохранить.
# @PRE files_with_dates is a list of (Path, date) tuples.
# @POST Returns a set of files to keep.
# @PARAM: files_with_dates (List[Tuple[Path, date]]) - Список файлов с датами.
# @PARAM: policy (RetentionPolicy) - Политика хранения.
# @RETURN: set - Множество путей к файлам, которые должны быть сохранены.
# @PRE: files_with_dates is a list of (Path, date) tuples.
# @POST: Returns a set of files to keep.
def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: RetentionPolicy) -> set:
with belief_scope("Apply retention policy"):
# Сортируем по дате (от новой к старой)
@@ -253,54 +228,43 @@ def apply_retention_policy(files_with_dates: List[Tuple[Path, date]], policy: Re
files_to_keep.update(daily_files)
files_to_keep.update(weekly_files[:policy.weekly])
files_to_keep.update(monthly_files[:policy.monthly])
log.reflect("Retention policy applied", payload={"files_to_keep": len(files_to_keep)})
app_logger.debug("[apply_retention_policy][State] Keeping %d files according to retention policy", len(files_to_keep))
return files_to_keep
# #endregion apply_retention_policy
# #region save_and_unpack_dashboard [TYPE Function]
# @BRIEF Сохраняет бинарное содержимое ZIP-архива на диск и опционально распаковывает его.
# @PRE zip_content должен быть байтами валидного ZIP-архива.
# @POST ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
# @PARAM: zip_content (bytes) - Содержимое ZIP-архива.
# @PARAM: output_dir (Union[str, Path]) - Директория для сохранения.
# @PARAM: unpack (bool) - Флаг, нужно ли распаковывать архив.
# @PARAM: original_filename (Optional[str]) - Исходное имя файла для сохранения.
# @RETURN: Tuple[Path, Optional[Path]] - Путь к ZIP-файлу и, если применимо, путь к директории с распаковкой.
# @THROW: InvalidZipFormatError - При ошибке формата ZIP.
# @PRE: zip_content должен быть байтами валидного ZIP-архива.
# @POST: ZIP-файл сохранен, и если unpack=True, он распакован в output_dir.
def save_and_unpack_dashboard(zip_content: bytes, output_dir: Union[str, Path], unpack: bool = False, original_filename: Optional[str] = None) -> Tuple[Path, Optional[Path]]:
with belief_scope("Save and unpack dashboard"):
log.reason("Processing dashboard ZIP save", payload={"unpack": unpack})
app_logger.info("[save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: %s", unpack)
try:
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
zip_name = sanitize_filename(original_filename) if original_filename else f"dashboard_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip"
zip_path = output_path / zip_name
zip_path.write_bytes(zip_content)
log.reason("Dashboard ZIP saved to disk", payload={"zip_path": str(zip_path)})
app_logger.info("[save_and_unpack_dashboard][State] Dashboard saved to: %s", zip_path)
if unpack:
with zipfile.ZipFile(zip_path, 'r') as zip_ref:
zip_ref.extractall(output_path)
log.reason("Dashboard unpacked to directory", payload={"output_path": str(output_path)})
app_logger.info("[save_and_unpack_dashboard][State] Dashboard unpacked to: %s", output_path)
return zip_path, output_path
return zip_path, None
except zipfile.BadZipFile as e:
log.explore("Invalid ZIP archive format", payload={"output_dir": str(output_dir)}, error=str(e))
app_logger.error("[save_and_unpack_dashboard][Failure] Invalid ZIP archive: %s", e)
raise InvalidZipFormatError(f"Invalid ZIP file: {e}") from e
# #endregion save_and_unpack_dashboard
# #region update_yamls [TYPE Function]
# @BRIEF Обновляет конфигурации в YAML-файлах, заменяя значения или применяя regex.
# @PRE path должен быть существующей директорией.
# @POST Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> [_update_yaml_file]
# @THROW: FileNotFoundError - Если `path` не существует.
# @PARAM: db_configs (Optional[List[Dict]]) - Список конфигураций для замены.
# @PARAM: path (str) - Путь к директории с YAML файлами.
# @PARAM: regexp_pattern (Optional[LiteralString]) - Паттерн для поиска.
# @PARAM: replace_string (Optional[LiteralString]) - Строка для замены.
# @PRE: path должен быть существующей директорией.
# @POST: Все YAML файлы в директории обновлены согласно переданным параметрам.
# @RELATION CALLS -> _update_yaml_file
def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str = "dashboards", regexp_pattern: Optional[LiteralString] = None, replace_string: Optional[LiteralString] = None) -> None:
with belief_scope("Update YAML configurations"):
log.reason("Starting YAML configuration update", payload={"path": path})
app_logger.info("[update_yamls][Enter] Starting YAML configuration update.")
dir_path = Path(path)
assert dir_path.is_dir(), f"Путь {path} не существует или не является директорией"
@@ -312,12 +276,8 @@ def update_yamls(db_configs: Optional[List[Dict[str, Any]]] = None, path: str =
# #region _update_yaml_file [TYPE Function]
# @BRIEF (Helper) Обновляет один YAML файл.
# @PRE file_path должен быть объектом Path к существующему YAML файлу.
# @POST Файл обновлен согласно переданным конфигурациям или регулярному выражению.
# @PARAM: file_path (Path) - Путь к файлу.
# @PARAM: db_configs (List[Dict]) - Конфигурации.
# @PARAM: regexp_pattern (Optional[str]) - Паттерн.
# @PARAM: replace_string (Optional[str]) - Замена.
# @PRE: file_path должен быть объектом Path к существующему YAML файлу.
# @POST: Файл обновлен согласно переданным конфигурациям или регулярному выражению.
def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_pattern: Optional[str], replace_string: Optional[str]) -> None:
with belief_scope(f"Update YAML file: {file_path}"):
# Читаем содержимое файла
@@ -325,7 +285,7 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
except Exception as e:
log.explore("Failed to read YAML file", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Failed to read %s: %s", file_path, e)
return
# Если задан pattern и replace_string, применяем замену по регулярному выражению
if regexp_pattern and replace_string:
@@ -334,9 +294,9 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
if new_content != content:
with open(file_path, 'w', encoding='utf-8') as f:
f.write(new_content)
log.reason("Updated YAML file using regex pattern", payload={"file_path": str(file_path)})
app_logger.info("[_update_yaml_file][State] Updated %s using regex pattern", file_path)
except Exception as e:
log.explore("Error applying regex to YAML file", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Error applying regex to %s: %s", file_path, e)
# Если заданы конфигурации, заменяем значения (поддержка old/new)
if db_configs:
try:
@@ -357,37 +317,33 @@ def _update_yaml_file(file_path: Path, db_configs: List[Dict[str, Any]], regexp_
# Группы: 1=ключ+разделитель, 2=открывающая кавычка (опц), 3=значение, 4=закрывающая кавычка (опц)
pattern = rf'({key_pattern}\s*:\s*)(["\']?)({val_pattern})(["\']?)'
# #region replacer [TYPE Function]
# @BRIEF Функция замены, сохраняющая кавычки если они были.
# @PRE match должен быть объектом совпадения регулярного выражения.
# @POST Возвращает строку с новым значением, сохраняя префикс и кавычки.
# [DEF:replacer:Function]
# @PURPOSE: Функция замены, сохраняющая кавычки если они были.
# @PRE: match должен быть объектом совпадения регулярного выражения.
# @POST: Возвращает строку с новым значением, сохраняя префикс и кавычки.
def replacer(match):
prefix = match.group(1)
quote_open = match.group(2)
quote_close = match.group(4)
return f"{prefix}{quote_open}{new_val}{quote_close}"
# #endregion replacer
# [/DEF:replacer:Function]
modified_content = re.sub(pattern, replacer, modified_content)
log.reason("Replaced YAML config value", payload={"key": key, "file_path": str(file_path)})
app_logger.info("[_update_yaml_file][State] Replaced '%s' with '%s' for key %s in %s", old_val, new_val, key, file_path)
# Записываем обратно изменённый контент без парсинга YAML, сохраняем оригинальное форматирование
with open(file_path, 'w', encoding='utf-8') as f:
f.write(modified_content)
except Exception as e:
log.explore("Error performing raw replacement in YAML", payload={"file_path": str(file_path)}, error=str(e))
app_logger.error("[_update_yaml_file][Failure] Error performing raw replacement in %s: %s", file_path, e)
# #endregion _update_yaml_file
# #region create_dashboard_export [TYPE Function]
# @BRIEF Создает ZIP-архив из указанных исходных путей.
# @PRE source_paths должен содержать существующие пути.
# @POST ZIP-архив создан по пути zip_path.
# @PARAM: zip_path (Union[str, Path]) - Путь для сохранения ZIP архива.
# @PARAM: source_paths (List[Union[str, Path]]) - Список исходных путей для архивации.
# @PARAM: exclude_extensions (Optional[List[str]]) - Список расширений для исключения.
# @RETURN: bool - `True` при успехе, `False` при ошибке.
# @PRE: source_paths должен содержать существующие пути.
# @POST: ZIP-архив создан по пути zip_path.
def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union[str, Path]], exclude_extensions: Optional[List[str]] = None) -> bool:
with belief_scope(f"Create dashboard export: {zip_path}"):
log.reason("Packing dashboard export", payload={"source_paths": [str(p) for p in source_paths], "zip_path": str(zip_path)})
app_logger.info("[create_dashboard_export][Enter] Packing dashboard: %s -> %s", source_paths, zip_path)
try:
exclude_ext = [ext.lower() for ext in exclude_extensions or []]
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
@@ -398,19 +354,17 @@ def create_dashboard_export(zip_path: Union[str, Path], source_paths: List[Union
if item.is_file() and item.suffix.lower() not in exclude_ext:
arcname = item.relative_to(src_path.parent)
zipf.write(item, arcname)
log.reflect("Dashboard archive created", payload={"zip_path": str(zip_path)})
app_logger.info("[create_dashboard_export][Exit] Archive created: %s", zip_path)
return True
except (IOError, zipfile.BadZipFile, AssertionError) as e:
log.explore("Dashboard archive creation failed", payload={"zip_path": str(zip_path)}, error=str(e))
app_logger.error("[create_dashboard_export][Failure] Error: %s", e, exc_info=True)
return False
# #endregion create_dashboard_export
# #region sanitize_filename [TYPE Function]
# @BRIEF Очищает строку от символов, недопустимых в именах файлов.
# @PRE filename должен быть строкой.
# @POST Возвращает строку без спецсимволов.
# @PARAM: filename (str) - Исходное имя файла.
# @RETURN: str - Очищенная строка.
# @PRE: filename должен быть строкой.
# @POST: Возвращает строку без спецсимволов.
def sanitize_filename(filename: str) -> str:
with belief_scope(f"Sanitize filename: {filename}"):
return re.sub(r'[\\/*?:"<>|]', "_", filename).strip()
@@ -418,10 +372,8 @@ def sanitize_filename(filename: str) -> str:
# #region get_filename_from_headers [TYPE Function]
# @BRIEF Извлекает имя файла из HTTP заголовка 'Content-Disposition'.
# @PRE headers должен быть словарем заголовков.
# @POST Возвращает имя файла или None, если заголовок отсутствует.
# @PARAM: headers (dict) - Словарь HTTP заголовков.
# @RETURN: Optional[str] - Имя файла or `None`.
# @PRE: headers должен быть словарем заголовков.
# @POST: Возвращает имя файла или None, если заголовок отсутствует.
def get_filename_from_headers(headers: dict) -> Optional[str]:
with belief_scope("Get filename from headers"):
content_disposition = headers.get("Content-Disposition", "")
@@ -432,16 +384,14 @@ def get_filename_from_headers(headers: dict) -> Optional[str]:
# #region consolidate_archive_folders [TYPE Function]
# @BRIEF Консолидирует директории архивов на основе общего слага в имени.
# @PRE root_directory должен быть объектом Path к существующей директории.
# @POST Директории с одинаковым префиксом объединены в одну.
# @THROW: TypeError, ValueError - Если `root_directory` невалиден.
# @PARAM: root_directory (Path) - Корневая директория для консолидации.
# @PRE: root_directory должен быть объектом Path к существующей директории.
# @POST: Директории с одинаковым префиксом объединены в одну.
def consolidate_archive_folders(root_directory: Path) -> None:
with belief_scope(f"Consolidate archives in {root_directory}"):
assert isinstance(root_directory, Path), "root_directory must be a Path object."
assert root_directory.is_dir(), "root_directory must be an existing directory."
log.reason("Consolidating archive folders", payload={"root_directory": str(root_directory)})
app_logger.info("[consolidate_archive_folders][Enter] Consolidating archives in %s", root_directory)
# Собираем все директории с архивами
archive_dirs = []
for item in root_directory.iterdir():
@@ -464,7 +414,7 @@ def consolidate_archive_folders(root_directory: Path) -> None:
# Создаем целевую директорию
target_dir = root_directory / slug
target_dir.mkdir(exist_ok=True)
log.reason("Consolidating archive directories", payload={"slug": slug, "dir_count": len(dirs), "target": str(target_dir)})
app_logger.info("[consolidate_archive_folders][State] Consolidating %d directories under %s", len(dirs), target_dir)
# Перемещаем содержимое
for source_dir in dirs:
if source_dir == target_dir:
@@ -477,13 +427,13 @@ def consolidate_archive_folders(root_directory: Path) -> None:
else:
shutil.move(str(item), str(dest_item))
except Exception as e:
log.explore("Failed to move item during consolidation", payload={"source": str(item), "dest": str(dest_item)}, error=str(e))
app_logger.error("[consolidate_archive_folders][Failure] Failed to move %s to %s: %s", item, dest_item, e)
# Удаляем исходную директорию
try:
source_dir.rmdir()
log.reason("Removed source directory after consolidation", payload={"directory": str(source_dir)})
app_logger.info("[consolidate_archive_folders][State] Removed source directory: %s", source_dir)
except Exception as e:
log.explore("Failed to remove source directory after consolidation", payload={"directory": str(source_dir)}, error=str(e))
app_logger.error("[consolidate_archive_folders][Failure] Failed to remove source directory %s: %s", source_dir, e)
# #endregion consolidate_archive_folders
# #endregion FileIO
# #endregion FileIO

18
backend/src/core/utils/matching.py
View File

@@ -1,24 +1,18 @@
# #region FuzzyMatching [TYPE Module] [SEMANTICS fuzzy, matching, rapidfuzz, database, mapping]
#
# @BRIEF Provides utility functions for fuzzy matching database names.
# @LAYER Core
# @INVARIANT Confidence scores are returned as floats between 0.0 and 1.0.
# @RELATION DEPENDS_ON -> [rapidfuzz]
#
# @LAYER: Core
# @RELATION DEPENDS_ON -> rapidfuzz
#
# @INVARIANT: Confidence scores are returned as floats between 0.0 and 1.0.
# [SECTION: IMPORTS]
from rapidfuzz import fuzz, process
from typing import List, Dict
# [/SECTION]
# #region suggest_mappings [TYPE Function]
# @BRIEF Suggests mappings between source and target databases using fuzzy matching.
# @PRE source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST Returns a list of suggested mappings with confidence scores.
# @PARAM: source_databases (List[Dict]) - Databases from the source environment.
# @PARAM: target_databases (List[Dict]) - Databases from the target environment.
# @PARAM: threshold (int) - Minimum confidence score (0-100).
# @RETURN: List[Dict] - Suggested mappings.
# @PRE: source_databases and target_databases are lists of dictionaries with 'uuid' and 'database_name'.
# @POST: Returns a list of suggested mappings with confidence scores.
def suggest_mappings(source_databases: List[Dict], target_databases: List[Dict], threshold: int = 60) -> List[Dict]:
"""
Suggest mappings between source and target databases using fuzzy matching.

309
backend/src/core/utils/network.py
View File

@@ -1,11 +1,10 @@
# #region NetworkModule [C:3] [TYPE Module] [SEMANTICS network, http, client, api, requests, session, authentication]
# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
#
# @BRIEF Инкапсулирует низкоуровневую HTTP-логику для взаимодействия с Superset API, включая аутентификацию, управление сессией, retry-логику и обработку ошибок.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [LoggerModule]
# @PUBLIC_API: APIClient
# [SECTION: IMPORTS]
from typing import Optional, Dict, Any, List, Union, cast, Tuple
import json
import io
@@ -17,81 +16,81 @@ from requests.adapters import HTTPAdapter
import urllib3
from urllib3.util.retry import Retry
from ..logger import logger as app_logger, belief_scope
from ..cot_logger import MarkerLogger
# [/SECTION]
log = MarkerLogger("Network")
# #region SupersetAPIError [C:1] [TYPE Class]
# @BRIEF Base exception for all Superset API related errors.
class SupersetAPIError(Exception):
# #region __init__ [C:1] [TYPE Function]
# @BRIEF Initializes the exception with a message and context.
# @PRE message is a string, context is a dict.
# @POST Exception is initialized with context.
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the exception with a message and context.
# @PRE: message is a string, context is a dict.
# @POST: Exception is initialized with context.
def __init__(self, message: str = "Superset API error", **context: Any):
with belief_scope("SupersetAPIError.__init__"):
self.context = context
super().__init__(f"[API_FAILURE] {message} | Context: {self.context}")
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion SupersetAPIError
# #region AuthenticationError [C:1] [TYPE Class]
# @BRIEF Exception raised when authentication fails.
class AuthenticationError(SupersetAPIError):
# #region __init__ [C:1] [TYPE Function]
# @BRIEF Initializes the authentication error.
# @PRE message is a string, context is a dict.
# @POST AuthenticationError is initialized.
# [DEF:__init__:Function]
# @COMPLEXITY: 1
# @PURPOSE: Initializes the authentication error.
# @PRE: message is a string, context is a dict.
# @POST: AuthenticationError is initialized.
def __init__(self, message: str = "Authentication failed", **context: Any):
with belief_scope("AuthenticationError.__init__"):
super().__init__(message, type="authentication", **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion AuthenticationError
# #region PermissionDeniedError [TYPE Class]
# @BRIEF Exception raised when access is denied.
class PermissionDeniedError(AuthenticationError):
# #region __init__ [TYPE Function]
# @BRIEF Initializes the permission denied error.
# @PRE message is a string, context is a dict.
# @POST PermissionDeniedError is initialized.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the permission denied error.
# @PRE: message is a string, context is a dict.
# @POST: PermissionDeniedError is initialized.
def __init__(self, message: str = "Permission denied", **context: Any):
with belief_scope("PermissionDeniedError.__init__"):
super().__init__(message, **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion PermissionDeniedError
# #region DashboardNotFoundError [TYPE Class]
# @BRIEF Exception raised when a dashboard cannot be found.
class DashboardNotFoundError(SupersetAPIError):
# #region __init__ [TYPE Function]
# @BRIEF Initializes the not found error with resource ID.
# @PRE resource_id is provided.
# @POST DashboardNotFoundError is initialized.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the not found error with resource ID.
# @PRE: resource_id is provided.
# @POST: DashboardNotFoundError is initialized.
def __init__(self, resource_id: Union[int, str], message: str = "Dashboard not found", **context: Any):
with belief_scope("DashboardNotFoundError.__init__"):
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
# #endregion __init__
# [/DEF:__init__:Function]
# #endregion DashboardNotFoundError
# #region NetworkError [TYPE Class]
# @BRIEF Exception raised when a network level error occurs.
class NetworkError(Exception):
# #region NetworkError.__init__ [TYPE Function]
# @BRIEF Initializes the network error.
# @PRE message is a string.
# @POST NetworkError is initialized.
# [DEF:NetworkError.__init__:Function]
# @PURPOSE: Initializes the network error.
# @PRE: message is a string.
# @POST: NetworkError is initialized.
def __init__(self, message: str = "Network connection failed", **context: Any):
with belief_scope("NetworkError.__init__"):
self.context = context
super().__init__(f"[NETWORK_FAILURE] {message} | Context: {self.context}")
# #endregion NetworkError.__init__
# [/DEF:NetworkError.__init__:Function]
# #endregion NetworkError
# #region SupersetAuthCache [TYPE Class]
# @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials.
# @PRE base_url and username are stable strings.
# @POST Cached entries expire automatically by TTL and can be reused across requests.
# @PRE: base_url and username are stable strings.
# @POST: Cached entries expire automatically by TTL and can be reused across requests.
class SupersetAuthCache:
TTL_SECONDS = 300
@@ -106,7 +105,7 @@ class SupersetAuthCache:
return (str(base_url or "").strip(), username, bool(verify_ssl))
@classmethod
# #region SupersetAuthCache.get [TYPE Function]
# [DEF:SupersetAuthCache.get:Function]
def get(cls, key: Tuple[str, str, bool]) -> Optional[Dict[str, str]]:
now = time.time()
with cls._lock:
@@ -125,10 +124,10 @@ class SupersetAuthCache:
"access_token": str(tokens.get("access_token") or ""),
"csrf_token": str(tokens.get("csrf_token") or ""),
}
# #endregion SupersetAuthCache.get
# [/DEF:SupersetAuthCache.get:Function]
@classmethod
# #region SupersetAuthCache.set [TYPE Function]
# [DEF:SupersetAuthCache.set:Function]
def set(cls, key: Tuple[str, str, bool], tokens: Dict[str, str], ttl_seconds: Optional[int] = None) -> None:
normalized_ttl = max(int(ttl_seconds or cls.TTL_SECONDS), 1)
with cls._lock:
@@ -139,7 +138,7 @@ class SupersetAuthCache:
},
"expires_at": time.time() + normalized_ttl,
}
# #endregion SupersetAuthCache.set
# [/DEF:SupersetAuthCache.set:Function]
@classmethod
def invalidate(cls, key: Tuple[str, str, bool]) -> None:
@@ -154,8 +153,8 @@ class SupersetAuthCache:
class APIClient:
DEFAULT_TIMEOUT = 30
# #region APIClient.__init__ [TYPE Function]
# @BRIEF Инициализирует API клиент с конфигурацией, сессией и логгером.
# [DEF:APIClient.__init__:Function]
# @PURPOSE: Инициализирует API клиент с конфигурацией, сессией и логгером.
# @PARAM: config (Dict[str, Any]) - Конфигурация.
# @PARAM: verify_ssl (bool) - Проверять ли SSL.
# @PARAM: timeout (int) - Таймаут запросов.
@@ -163,7 +162,7 @@ class APIClient:
# @POST: APIClient instance is initialized with a session.
def __init__(self, config: Dict[str, Any], verify_ssl: bool = True, timeout: int = DEFAULT_TIMEOUT):
with belief_scope("__init__"):
log.reason("Initializing APIClient")
app_logger.info("[APIClient.__init__][Entry] Initializing APIClient.")
self.base_url: str = self._normalize_base_url(config.get("base_url", ""))
self.api_base_url: str = f"{self.base_url}/api/v1"
self.auth = config.get("auth")
@@ -176,14 +175,14 @@ class APIClient:
verify_ssl,
)
self._authenticated = False
log.reflect("APIClient initialized")
# #endregion APIClient.__init__
app_logger.info("[APIClient.__init__][Exit] APIClient initialized.")
# [/DEF:APIClient.__init__:Function]
# #region _init_session [TYPE Function]
# @BRIEF Создает и настраивает `requests.Session` с retry-логикой.
# @PRE self.request_settings must be initialized.
# @POST Returns a configured requests.Session instance.
# @RETURN requests.Session - Настроенная сессия.
# [DEF:_init_session:Function]
# @PURPOSE: Создает и настраивает `requests.Session` с retry-логикой.
# @PRE: self.request_settings must be initialized.
# @POST: Returns a configured requests.Session instance.
# @RETURN: requests.Session - Настроенная сессия.
def _init_session(self) -> requests.Session:
with belief_scope("_init_session"):
session = requests.Session()
@@ -216,32 +215,32 @@ class APIClient:
if not self.request_settings["verify_ssl"]:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
log.reflect("SSL verification disabled for session", payload={"base_url": self.base_url})
app_logger.warning("[_init_session][State] SSL verification disabled.")
# When verify_ssl is false, we should also disable hostname verification
session.verify = False
else:
session.verify = True
return session
# #endregion _init_session
# [/DEF:_init_session:Function]
# #region _normalize_base_url [TYPE Function]
# @BRIEF Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.
# @PRE raw_url can be empty.
# @POST Returns canonical base URL suitable for building API endpoints.
# @RETURN str
# [DEF:_normalize_base_url:Function]
# @PURPOSE: Normalize Superset environment URL to base host/path without trailing slash and /api/v1 suffix.
# @PRE: raw_url can be empty.
# @POST: Returns canonical base URL suitable for building API endpoints.
# @RETURN: str
def _normalize_base_url(self, raw_url: str) -> str:
normalized = str(raw_url or "").strip().rstrip("/")
if normalized.lower().endswith("/api/v1"):
normalized = normalized[:-len("/api/v1")]
return normalized.rstrip("/")
# #endregion _normalize_base_url
# [/DEF:_normalize_base_url:Function]
# #region _build_api_url [TYPE Function]
# @BRIEF Build absolute Superset API URL for endpoint using canonical /api/v1 base.
# @PRE endpoint is relative path or absolute URL.
# @POST Returns full URL without accidental duplicate slashes.
# @RETURN str
# [DEF:_build_api_url:Function]
# @PURPOSE: Build absolute Superset API URL for endpoint using canonical /api/v1 base.
# @PRE: endpoint is relative path or absolute URL.
# @POST: Returns full URL without accidental duplicate slashes.
# @RETURN: str
def _build_api_url(self, endpoint: str) -> str:
normalized_endpoint = str(endpoint or "").strip()
if normalized_endpoint.startswith("http://") or normalized_endpoint.startswith("https://"):
@@ -251,97 +250,65 @@ class APIClient:
if normalized_endpoint.startswith("/api/v1/") or normalized_endpoint == "/api/v1":
return f"{self.base_url}{normalized_endpoint}"
return f"{self.api_base_url}{normalized_endpoint}"
# #endregion _build_api_url
# [/DEF:_build_api_url:Function]
# #region APIClient.authenticate [TYPE Function]
# @BRIEF Выполняет аутентификацию в Superset API и получает access и CSRF токены.
# @PRE self.auth and self.base_url must be valid.
# @POST `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN Dict[str, str] - Словарь с токенами.
# [DEF:APIClient.authenticate:Function]
# @PURPOSE: Выполняет аутентификацию в Superset API и получает access и CSRF токены.
# @PRE: self.auth and self.base_url must be valid.
# @POST: `self._tokens` заполнен, `self._authenticated` установлен в `True`.
# @RETURN: Dict[str, str] - Словарь с токенами.
# @THROW: AuthenticationError, NetworkError - при ошибках.
# @RELATION: [CALLS] ->[SupersetAuthCache.get]
# @RELATION: [CALLS] ->[SupersetAuthCache.set]
# @RATIONALE: Auth cache stores access_token and csrf_token but not the session cookie.
# When a new APIClient instance reuses cached tokens, its fresh requests.Session
# has no session cookie — causing superset CSRF check to fail with
# "CSRF session token is missing". The fix is to always fetch a fresh CSRF token
# (GET /security/csrf_token/) even on cache hit, which establishes a session cookie
# in the new session while avoiding a full POST login.
# @REJECTED: Skipping csrf_token GET on cache hit — session cookie would be missing, causing
# all POST/PUT/DELETE requests to fail with CSRF error on new APIClient instances.
def authenticate(self) -> Dict[str, str]:
with belief_scope("authenticate"):
log.reason("Authenticating to Superset", payload={"base_url": self.base_url})
app_logger.info("[authenticate][Enter] Authenticating to %s", self.base_url)
cached_tokens = SupersetAuthCache.get(self._auth_cache_key)
if cached_tokens and cached_tokens.get("access_token") and cached_tokens.get("csrf_token"):
self._tokens = cached_tokens
# Always fetch a fresh CSRF token to establish session cookie in this session.
# The cached access_token may still be valid; we avoid a full POST login.
try:
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(
csrf_url,
headers={"Authorization": f"Bearer {self._tokens['access_token']}"},
timeout=self.request_settings["timeout"],
)
csrf_response.raise_for_status()
# Update CSRF token (may have rotated)
self._tokens["csrf_token"] = csrf_response.json()["result"]
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reason("Reused cached tokens + refreshed CSRF token", payload={"base_url": self.base_url})
except Exception as csrf_err:
# CSRF refresh failed — likely access_token expired, do full re-auth
log.explore("CSRF refresh failed, performing full re-auth", error=str(csrf_err))
SupersetAuthCache.invalidate(self._auth_cache_key)
# Fall through to full login below
self._tokens = {}
self._authenticated = False
if not self._authenticated:
try:
login_url = f"{self.api_base_url}/security/login"
# Log the payload keys and values (masking password)
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
log.reason("Performing full Superset login", payload={"login_url": login_url, "auth_payload": masked_auth})
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
if response.status_code != 200:
log.explore("Superset login returned non-200 status", payload={"status_code": response.status_code, "response": response.text}, error=f"HTTP {response.status_code}: {response.text[:200]}")
response.raise_for_status()
access_token = response.json()["access_token"]
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
csrf_response.raise_for_status()
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
log.reflect("Authenticated successfully")
return self._tokens
except requests.exceptions.HTTPError as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
status_code = e.response.status_code if e.response is not None else None
if status_code in [502, 503, 504]:
raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e
raise AuthenticationError(f"Authentication failed: {e}") from e
except (requests.exceptions.RequestException, KeyError) as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {e}") from e
self._authenticated = True
log.reflect("Authenticated successfully (from cached tokens)")
return self._tokens
# #endregion APIClient.authenticate
self._authenticated = True
app_logger.info("[authenticate][CacheHit] Reusing cached Superset auth tokens for %s", self.base_url)
return self._tokens
try:
login_url = f"{self.api_base_url}/security/login"
# Log the payload keys and values (masking password)
masked_auth = {k: ("******" if k == "password" else v) for k, v in self.auth.items()}
app_logger.info(f"[authenticate][Debug] Login URL: {login_url}")
app_logger.info(f"[authenticate][Debug] Auth payload: {masked_auth}")
response = self.session.post(login_url, json=self.auth, timeout=self.request_settings["timeout"])
if response.status_code != 200:
app_logger.error(f"[authenticate][Error] Status: {response.status_code}, Response: {response.text}")
response.raise_for_status()
access_token = response.json()["access_token"]
csrf_url = f"{self.api_base_url}/security/csrf_token/"
csrf_response = self.session.get(csrf_url, headers={"Authorization": f"Bearer {access_token}"}, timeout=self.request_settings["timeout"])
csrf_response.raise_for_status()
self._tokens = {"access_token": access_token, "csrf_token": csrf_response.json()["result"]}
self._authenticated = True
SupersetAuthCache.set(self._auth_cache_key, self._tokens)
app_logger.info("[authenticate][Exit] Authenticated successfully.")
return self._tokens
except requests.exceptions.HTTPError as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
status_code = e.response.status_code if e.response is not None else None
if status_code in [502, 503, 504]:
raise NetworkError(f"Environment unavailable during authentication (Status {status_code})", status_code=status_code) from e
raise AuthenticationError(f"Authentication failed: {e}") from e
except (requests.exceptions.RequestException, KeyError) as e:
SupersetAuthCache.invalidate(self._auth_cache_key)
raise NetworkError(f"Network or parsing error during authentication: {e}") from e
# [/DEF:APIClient.authenticate:Function]
@property
# #region headers [TYPE Function]
# @BRIEF Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE APIClient is initialized and authenticated or can be authenticated.
# @POST Returns headers including auth tokens.
# [DEF:headers:Function]
# @PURPOSE: Возвращает HTTP-заголовки для аутентифицированных запросов.
# @PRE: APIClient is initialized and authenticated or can be authenticated.
# @POST: Returns headers including auth tokens.
def headers(self) -> Dict[str, str]:
if not self._authenticated:
self.authenticate()
@@ -351,10 +318,10 @@ class APIClient:
"Referer": self.base_url,
"Content-Type": "application/json"
}
# #endregion headers
# [/DEF:headers:Function]
# #region request [TYPE Function]
# @BRIEF Выполняет универсальный HTTP-запрос к API.
# [DEF:request:Function]
# @PURPOSE: Выполняет универсальный HTTP-запрос к API.
# @PARAM: method (str) - HTTP метод.
# @PARAM: endpoint (str) - API эндпоинт.
# @PARAM: headers (Optional[Dict]) - Дополнительные заголовки.
@@ -381,10 +348,10 @@ class APIClient:
self._handle_http_error(e, endpoint)
except requests.exceptions.RequestException as e:
self._handle_network_error(e, full_url)
# #endregion request
# [/DEF:request:Function]
# #region _handle_http_error [TYPE Function]
# @BRIEF (Helper) Преобразует HTTP ошибки в кастомные исключения.
# [DEF:_handle_http_error:Function]
# @PURPOSE: (Helper) Преобразует HTTP ошибки в кастомные исключения.
# @PARAM: e (requests.exceptions.HTTPError) - Ошибка.
# @PARAM: endpoint (str) - Эндпоинт.
# @PRE: e must be a valid HTTPError with a response.
@@ -408,12 +375,12 @@ class APIClient:
if status_code == 401:
raise AuthenticationError() from e
raise SupersetAPIError(f"API Error {status_code}: {e.response.text}") from e
# #endregion _handle_http_error
# [/DEF:_handle_http_error:Function]
# #region _is_dashboard_endpoint [TYPE Function]
# @BRIEF Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @PRE endpoint may be relative or absolute.
# @POST Returns true only for dashboard-specific endpoints.
# [DEF:_is_dashboard_endpoint:Function]
# @PURPOSE: Determine whether an API endpoint represents a dashboard resource for 404 translation.
# @PRE: endpoint may be relative or absolute.
# @POST: Returns true only for dashboard-specific endpoints.
def _is_dashboard_endpoint(self, endpoint: str) -> bool:
normalized_endpoint = str(endpoint or "").strip().lower()
if not normalized_endpoint:
@@ -426,10 +393,10 @@ class APIClient:
if normalized_endpoint.startswith("/api/v1/"):
normalized_endpoint = normalized_endpoint[len("/api/v1"):]
return normalized_endpoint.startswith("/dashboard/") or normalized_endpoint == "/dashboard"
# #endregion _is_dashboard_endpoint
# [/DEF:_is_dashboard_endpoint:Function]
# #region _handle_network_error [TYPE Function]
# @BRIEF (Helper) Преобразует сетевые ошибки в `NetworkError`.
# [DEF:_handle_network_error:Function]
# @PURPOSE: (Helper) Преобразует сетевые ошибки в `NetworkError`.
# @PARAM: e (requests.exceptions.RequestException) - Ошибка.
# @PARAM: url (str) - URL.
# @PRE: e must be a RequestException.
@@ -443,10 +410,10 @@ class APIClient:
else:
msg = f"Unknown network error: {e}"
raise NetworkError(msg, url=url) from e
# #endregion _handle_network_error
# [/DEF:_handle_network_error:Function]
# #region upload_file [TYPE Function]
# @BRIEF Загружает файл на сервер через multipart/form-data.
# [DEF:upload_file:Function]
# @PURPOSE: Загружает файл на сервер через multipart/form-data.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: file_info (Dict[str, Any]) - Информация о файле.
# @PARAM: extra_data (Optional[Dict]) - Дополнительные данные.
@@ -474,10 +441,10 @@ class APIClient:
raise TypeError(f"Unsupported file_obj type: {type(file_obj)}")
return self._perform_upload(full_url, files_payload, extra_data, _headers, timeout)
# #endregion upload_file
# [/DEF:upload_file:Function]
# #region _perform_upload [TYPE Function]
# @BRIEF (Helper) Выполняет POST запрос с файлом.
# [DEF:_perform_upload:Function]
# @PURPOSE: (Helper) Выполняет POST запрос с файлом.
# @PARAM: url (str) - URL.
# @PARAM: files (Dict) - Файлы.
# @PARAM: data (Optional[Dict]) - Данные.
@@ -495,17 +462,17 @@ class APIClient:
try:
return response.json()
except Exception as json_e:
log.explore("Upload response not valid JSON", payload={"preview": response.text[:200]}, error=str(json_e))
app_logger.debug(f"[_perform_upload][Debug] Response is not valid JSON: {response.text[:200]}...")
raise SupersetAPIError(f"API error during upload: Response is not valid JSON: {json_e}") from json_e
return response.json()
except requests.exceptions.HTTPError as e:
raise SupersetAPIError(f"API error during upload: {e.response.text}") from e
except requests.exceptions.RequestException as e:
raise NetworkError(f"Network error during upload: {e}", url=url) from e
# #endregion _perform_upload
# [/DEF:_perform_upload:Function]
# #region fetch_paginated_count [TYPE Function]
# @BRIEF Получает общее количество элементов для пагинации.
# [DEF:fetch_paginated_count:Function]
# @PURPOSE: Получает общее количество элементов для пагинации.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: query_params (Dict) - Параметры запроса.
# @PARAM: count_field (str) - Поле с количеством.
@@ -516,10 +483,10 @@ class APIClient:
with belief_scope("fetch_paginated_count"):
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query_params)}))
return response_json.get(count_field, 0)
# #endregion fetch_paginated_count
# [/DEF:fetch_paginated_count:Function]
# #region fetch_paginated_data [TYPE Function]
# @BRIEF Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# [DEF:fetch_paginated_data:Function]
# @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
@@ -547,7 +514,7 @@ class APIClient:
if total_count is None:
total_count = response_json.get(count_field, len(first_page_results))
log.reason("Total count resolved from first page", payload={"total_count": total_count})
app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}")
# Fetch remaining pages
total_pages = (total_count + page_size - 1) // page_size
@@ -557,7 +524,7 @@ class APIClient:
results.extend(response_json.get(results_field, []))
return results
# #endregion fetch_paginated_data
# [/DEF:fetch_paginated_data:Function]
# #endregion APIClient

127
backend/src/core/utils/superset_compilation_adapter.py
View File

@@ -1,12 +1,12 @@
# #region SupersetCompilationAdapter [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, compilation_preview, sql_lab_launch, execution_truth]
# @BRIEF Interact with Superset preview compilation and SQL Lab execution endpoints using the current approved execution context.
# @LAYER Infra
# @PRE effective template params and dataset execution reference are available.
# @POST preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT performs upstream Superset preview and SQL Lab calls.
# @INVARIANT The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
# @LAYER: Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [CompiledPreview]
# @PRE: effective template params and dataset execution reference are available.
# @POST: preview and launch calls return Superset-originated artifacts or explicit errors.
# @SIDE_EFFECT: performs upstream Superset preview and SQL Lab calls.
# @INVARIANT: The adapter never fabricates compiled SQL locally; preview truth is delegated to Superset only.
from __future__ import annotations
@@ -52,38 +52,41 @@ class SqlLabLaunchPayload:
# #region SupersetCompilationAdapter [C:4] [TYPE Class]
# @BRIEF Delegate preview compilation and SQL Lab launch to Superset without local SQL fabrication.
# @PRE environment is configured and Superset is reachable for the target session.
# @POST adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT issues network requests to Superset API surfaces.
# @RELATION CALLS -> [SupersetClient]
# @PRE: environment is configured and Superset is reachable for the target session.
# @POST: adapter can return explicit ready/failed preview artifacts and canonical SQL Lab references.
# @SIDE_EFFECT: issues network requests to Superset API surfaces.
class SupersetCompilationAdapter:
# #region SupersetCompilationAdapter.__init__ [C:2] [TYPE Function]
# @BRIEF Bind adapter to one Superset environment and client instance.
# [DEF:SupersetCompilationAdapter.__init__:Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind adapter to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
# #endregion SupersetCompilationAdapter.__init__
# [/DEF:SupersetCompilationAdapter.__init__:Function]
# #region SupersetCompilationAdapter._supports_client_method [C:2] [TYPE Function]
# @BRIEF Detect explicitly implemented client capabilities without treating loose mocks as real methods.
# [DEF:SupersetCompilationAdapter._supports_client_method:Function]
# @COMPLEXITY: 2
# @PURPOSE: Detect explicitly implemented client capabilities without treating loose mocks as real methods.
def _supports_client_method(self, method_name: str) -> bool:
client_dict = getattr(self.client, "__dict__", {})
if method_name in client_dict:
return callable(client_dict[method_name])
return callable(getattr(type(self.client), method_name, None))
# #endregion SupersetCompilationAdapter._supports_client_method
# [/DEF:SupersetCompilationAdapter._supports_client_method:Function]
# #region SupersetCompilationAdapter.compile_preview [C:4] [TYPE Function]
# @BRIEF Request Superset-side compiled SQL preview for the current effective inputs.
# @PRE dataset_id and effective inputs are available for the current session.
# @POST returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT performs upstream preview requests.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[CompiledPreview]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_superset_preview]
# [DEF:SupersetCompilationAdapter.compile_preview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Request Superset-side compiled SQL preview for the current effective inputs.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_superset_preview]
# @PRE: dataset_id and effective inputs are available for the current session.
# @POST: returns a ready or failed preview artifact backed only by Superset-originated SQL or diagnostics.
# @SIDE_EFFECT: performs upstream preview requests.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[CompiledPreview]
def compile_preview(self, payload: PreviewCompilationPayload) -> CompiledPreview:
with belief_scope("SupersetCompilationAdapter.compile_preview"):
if payload.dataset_id <= 0:
@@ -169,25 +172,27 @@ class SupersetCompilationAdapter:
)
return preview
# #endregion SupersetCompilationAdapter.compile_preview
# [/DEF:SupersetCompilationAdapter.compile_preview:Function]
# #region SupersetCompilationAdapter.mark_preview_stale [C:2] [TYPE Function]
# @BRIEF Invalidate previous preview after mapping or value changes.
# @PRE preview is a persisted preview artifact or current in-memory snapshot.
# @POST preview status becomes stale without fabricating a replacement artifact.
# [DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
# @COMPLEXITY: 2
# @PURPOSE: Invalidate previous preview after mapping or value changes.
# @PRE: preview is a persisted preview artifact or current in-memory snapshot.
# @POST: preview status becomes stale without fabricating a replacement artifact.
def mark_preview_stale(self, preview: CompiledPreview) -> CompiledPreview:
preview.preview_status = PreviewStatus.STALE
return preview
# #endregion SupersetCompilationAdapter.mark_preview_stale
# [/DEF:SupersetCompilationAdapter.mark_preview_stale:Function]
# #region SupersetCompilationAdapter.create_sql_lab_session [C:4] [TYPE Function]
# @BRIEF Create the canonical audited execution session after all launch gates pass.
# @PRE compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[str]
# @RELATION CALLS -> [SupersetCompilationAdapter._request_sql_lab_session]
# [DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
# @COMPLEXITY: 4
# @PURPOSE: Create the canonical audited execution session after all launch gates pass.
# @RELATION: [CALLS] ->[SupersetCompilationAdapter._request_sql_lab_session]
# @PRE: compiled_sql is Superset-originated and launch gates are already satisfied.
# @POST: returns one canonical SQL Lab session reference from Superset.
# @SIDE_EFFECT: performs upstream SQL Lab execution/session creation.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[str]
def create_sql_lab_session(self, payload: SqlLabLaunchPayload) -> str:
with belief_scope("SupersetCompilationAdapter.create_sql_lab_session"):
compiled_sql = str(payload.compiled_sql or "").strip()
@@ -239,15 +244,16 @@ class SupersetCompilationAdapter:
)
return sql_lab_session_ref
# #endregion SupersetCompilationAdapter.create_sql_lab_session
# [/DEF:SupersetCompilationAdapter.create_sql_lab_session:Function]
# #region SupersetCompilationAdapter._request_superset_preview [C:4] [TYPE Function]
# @BRIEF Request preview compilation through explicit client support backed by real Superset endpoints only.
# @PRE payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [SupersetClient.compile_dataset_preview]
# [DEF:SupersetCompilationAdapter._request_superset_preview:Function]
# @COMPLEXITY: 4
# @PURPOSE: Request preview compilation through explicit client support backed by real Superset endpoints only.
# @RELATION: [CALLS] ->[SupersetClient.compile_dataset_preview]
# @PRE: payload contains a valid dataset identifier and deterministic execution inputs for one preview attempt.
# @POST: returns one normalized upstream compilation response including the chosen strategy metadata.
# @SIDE_EFFECT: issues one or more Superset preview requests through the client fallback chain.
# @DATA_CONTRACT: Input[PreviewCompilationPayload] -> Output[Dict[str,Any]]
def _request_superset_preview(
self, payload: PreviewCompilationPayload
) -> Dict[str, Any]:
@@ -384,15 +390,16 @@ class SupersetCompilationAdapter:
)
raise RuntimeError(str(exc)) from exc
# #endregion SupersetCompilationAdapter._request_superset_preview
# [/DEF:SupersetCompilationAdapter._request_superset_preview:Function]
# #region SupersetCompilationAdapter._request_sql_lab_session [C:4] [TYPE Function]
# @BRIEF Probe supported SQL Lab execution surfaces and return the first successful response.
# @PRE payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
# @RELATION CALLS -> [SupersetClient.get_dataset]
# [DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
# @COMPLEXITY: 4
# @PURPOSE: Probe supported SQL Lab execution surfaces and return the first successful response.
# @RELATION: [CALLS] ->[SupersetClient.get_dataset]
# @PRE: payload carries non-empty Superset-originated SQL and a preview identifier for the current launch.
# @POST: returns the first successful SQL Lab execution response from Superset.
# @SIDE_EFFECT: issues Superset dataset lookup and SQL Lab execution requests.
# @DATA_CONTRACT: Input[SqlLabLaunchPayload] -> Output[Dict[str,Any]]
def _request_sql_lab_session(self, payload: SqlLabLaunchPayload) -> Dict[str, Any]:
dataset_raw = self.client.get_dataset(payload.dataset_id)
dataset_record = (
@@ -444,11 +451,12 @@ class SupersetCompilationAdapter:
"; ".join(errors) or "No Superset SQL Lab surface accepted the request"
)
# #endregion SupersetCompilationAdapter._request_sql_lab_session
# [/DEF:SupersetCompilationAdapter._request_sql_lab_session:Function]
# #region SupersetCompilationAdapter._normalize_preview_response [C:3] [TYPE Function]
# @BRIEF Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION DEPENDS_ON -> [CompiledPreview]
# [DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
# @COMPLEXITY: 3
# @PURPOSE: Normalize candidate Superset preview responses into one compiled-sql structure.
# @RELATION: [DEPENDS_ON] ->[CompiledPreview]
def _normalize_preview_response(self, response: Any) -> Optional[Dict[str, Any]]:
if not isinstance(response, dict):
return None
@@ -477,16 +485,19 @@ class SupersetCompilationAdapter:
}
return None
# #endregion SupersetCompilationAdapter._normalize_preview_response
# [/DEF:SupersetCompilationAdapter._normalize_preview_response:Function]
# #region SupersetCompilationAdapter._dump_json [C:1] [TYPE Function]
# @BRIEF Serialize Superset request payload deterministically for network transport.
# [DEF:SupersetCompilationAdapter._dump_json:Function]
# @COMPLEXITY: 1
# @PURPOSE: Serialize Superset request payload deterministically for network transport.
def _dump_json(self, payload: Dict[str, Any]) -> str:
import json
return json.dumps(payload, sort_keys=True, default=str)
# #endregion SupersetCompilationAdapter._dump_json
# [/DEF:SupersetCompilationAdapter._dump_json:Function]
# #endregion SupersetCompilationAdapter
# #endregion SupersetCompilationAdapter

18
backend/src/core/utils/superset_context_extractor/__init__.py
View File

@@ -1,15 +1,15 @@
# #region SupersetContextExtractorPackage [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery]
# @LAYER: Infra
# @BRIEF Recover dataset and dashboard context from Superset links while preserving explicit partial-recovery markers.
# @LAYER Infra
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RATIONALE Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
#
# @RATIONALE: Decomposed from monolithic superset_context_extractor.py (1397 lines) into
# domain-scoped mixins to satisfy INV_7 (module < 400 lines). The composed class
# preserves the original public API surface — all consumers continue to import
# from `src.core.utils.superset_context_extractor` without changes.
@@ -26,15 +26,15 @@ from ._pii import mask_pii_value, sanitize_imported_filter_for_assistant, _mask_
# #region SupersetContextExtractor [C:4] [TYPE Class]
# @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
# @RELATION DEPENDS_ON -> [Environment]
# @RELATION INHERITS -> [SupersetContextExtractorBase]
# @RELATION INHERITS -> [SupersetContextParsingMixin]
# @RELATION INHERITS -> [SupersetContextRecoveryMixin]
# @RELATION INHERITS -> [SupersetContextTemplatesMixin]
# @RELATION INHERITS -> [SupersetContextFiltersExtractMixin]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractor(
SupersetContextFiltersExtractMixin,
SupersetContextRecoveryMixin,

81
backend/src/core/utils/superset_context_extractor/_base.py
View File

@@ -1,13 +1,13 @@
# #region SupersetContextExtractorBase [C:4] [TYPE Module] [SEMANTICS dataset_review, superset, link_parsing, context_recovery, partial_recovery]
# @LAYER: Infra
# @BRIEF Base class and helpers for recovering dataset/dashboard context from Superset links with explicit partial-recovery markers.
# @LAYER Infra
# @PRE Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT Performs upstream Superset API reads.
# @INVARIANT Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# @RELATION DEPENDS_ON -> [ImportedFilter]
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION CALLS -> [SupersetClient]
# @PRE: Superset link or dataset reference must be parseable enough to resolve an environment-scoped target resource.
# @POST: Returns the best available recovered context with explicit provenance and partial-recovery markers when necessary.
# @SIDE_EFFECT: Performs upstream Superset API reads.
# @INVARIANT: Partial recovery is surfaced explicitly and never misrepresented as fully confirmed context.
# #endregion SupersetContextExtractorBase
# #region _base_imports [TYPE Block]
@@ -50,23 +50,25 @@ class SupersetParsedContext:
# #region SupersetContextExtractorBase [C:4] [TYPE Class]
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
# @PRE constructor receives a configured environment with a usable Superset base URL.
# @POST extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT downstream parse operations may call Superset APIs through SupersetClient.
# @RELATION DEPENDS_ON -> [Environment]
# @PRE: constructor receives a configured environment with a usable Superset base URL.
# @POST: extractor instance is ready to parse links against one Superset environment.
# @SIDE_EFFECT: downstream parse operations may call Superset APIs through SupersetClient.
class SupersetContextExtractorBase:
# #region SupersetContextExtractorBase.__init__ [C:2] [TYPE Function]
# @BRIEF Bind extractor to one Superset environment and client instance.
# [DEF:SupersetContextExtractorBase.__init__:Function]
# @COMPLEXITY: 2
# @PURPOSE: Bind extractor to one Superset environment and client instance.
def __init__(
self, environment: Environment, client: Optional[SupersetClient] = None
) -> None:
self.environment = environment
self.client = client or SupersetClient(environment)
# #endregion SupersetContextExtractorBase.__init__
# [/DEF:SupersetContextExtractorBase.__init__:Function]
# #region SupersetContextExtractorBase.build_recovery_summary [C:2] [TYPE Function]
# @BRIEF Summarize recovered, partial, and unresolved context for session state and UX.
# [DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Summarize recovered, partial, and unresolved context for session state and UX.
def build_recovery_summary(
self, parsed_context: SupersetParsedContext
) -> Dict[str, Any]:
@@ -80,10 +82,11 @@ class SupersetContextExtractorBase:
"imported_filter_count": len(parsed_context.imported_filters),
}
# #endregion SupersetContextExtractorBase.build_recovery_summary
# [/DEF:SupersetContextExtractorBase.build_recovery_summary:Function]
# #region SupersetContextExtractorBase._extract_numeric_identifier [C:2] [TYPE Function]
# @BRIEF Extract a numeric identifier from a REST-like Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a numeric identifier from a REST-like Superset URL path.
def _extract_numeric_identifier(
self, path_parts: List[str], resource_name: str
) -> Optional[int]:
@@ -102,10 +105,11 @@ class SupersetContextExtractorBase:
return None
return int(candidate)
# #endregion SupersetContextExtractorBase._extract_numeric_identifier
# [/DEF:SupersetContextExtractorBase._extract_numeric_identifier:Function]
# #region SupersetContextExtractorBase._extract_dashboard_reference [C:2] [TYPE Function]
# @BRIEF Extract a dashboard id-or-slug reference from a Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard id-or-slug reference from a Superset URL path.
def _extract_dashboard_reference(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
@@ -122,10 +126,11 @@ class SupersetContextExtractorBase:
return None
return candidate
# #endregion SupersetContextExtractorBase._extract_dashboard_reference
# [/DEF:SupersetContextExtractorBase._extract_dashboard_reference:Function]
# #region SupersetContextExtractorBase._extract_dashboard_permalink_key [C:2] [TYPE Function]
# @BRIEF Extract a dashboard permalink key from a Superset URL path.
# [DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard permalink key from a Superset URL path.
def _extract_dashboard_permalink_key(self, path_parts: List[str]) -> Optional[str]:
if "dashboard" not in path_parts:
return None
@@ -143,31 +148,34 @@ class SupersetContextExtractorBase:
return None
return permalink_key
# #endregion SupersetContextExtractorBase._extract_dashboard_permalink_key
# [/DEF:SupersetContextExtractorBase._extract_dashboard_permalink_key:Function]
# #region SupersetContextExtractorBase._extract_dashboard_id_from_state [C:2] [TYPE Function]
# @BRIEF Extract a dashboard identifier from returned permalink state when present.
# [DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a dashboard identifier from returned permalink state when present.
def _extract_dashboard_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"dashboardId", "dashboard_id", "dashboard_id_value"},
)
# #endregion SupersetContextExtractorBase._extract_dashboard_id_from_state
# [/DEF:SupersetContextExtractorBase._extract_dashboard_id_from_state:Function]
# #region SupersetContextExtractorBase._extract_chart_id_from_state [C:2] [TYPE Function]
# @BRIEF Extract a chart identifier from returned permalink state when dashboard id is absent.
# [DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a chart identifier from returned permalink state when dashboard id is absent.
def _extract_chart_id_from_state(self, state: Dict[str, Any]) -> Optional[int]:
return self._search_nested_numeric_key(
payload=state,
candidate_keys={"slice_id", "sliceId", "chartId", "chart_id"},
)
# #endregion SupersetContextExtractorBase._extract_chart_id_from_state
# [/DEF:SupersetContextExtractorBase._extract_chart_id_from_state:Function]
# #region SupersetContextExtractorBase._search_nested_numeric_key [C:3] [TYPE Function]
# @BRIEF Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# [DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recursively search nested dict/list payloads for the first numeric value under a candidate key set.
# @RELATION: DEPENDS_ON -> [SupersetContextExtractorBase]
def _search_nested_numeric_key(
self, payload: Any, candidate_keys: Set[str]
) -> Optional[int]:
@@ -189,10 +197,11 @@ class SupersetContextExtractorBase:
return found
return None
# #endregion SupersetContextExtractorBase._search_nested_numeric_key
# [/DEF:SupersetContextExtractorBase._search_nested_numeric_key:Function]
# #region SupersetContextExtractorBase._decode_query_state [C:2] [TYPE Function]
# @BRIEF Decode query-string structures used by Superset URL state transport.
# [DEF:SupersetContextExtractorBase._decode_query_state:Function]
# @COMPLEXITY: 2
# @PURPOSE: Decode query-string structures used by Superset URL state transport.
def _decode_query_state(self, query_params: Dict[str, List[str]]) -> Dict[str, Any]:
query_state: Dict[str, Any] = {}
for key, values in query_params.items():
@@ -212,7 +221,7 @@ class SupersetContextExtractorBase:
query_state[key] = decoded_value
return query_state
# #endregion SupersetContextExtractorBase._decode_query_state
# [/DEF:SupersetContextExtractorBase._decode_query_state:Function]
# #endregion SupersetContextExtractorBase

9
backend/src/core/utils/superset_context_extractor/_filters.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Module] [SEMANTICS superset, filters, extraction, query_state, dataMask, native_filters]
# @LAYER: Infra
# @BRIEF Extract normalized imported filters from decoded Superset query state (native_filters, dataMask, native_filter_state, form_data).
# @LAYER Infra
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextFiltersExtractMixin
@@ -19,8 +19,9 @@ app_logger = cast(Any, app_logger)
# #region SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor.
class SupersetContextFiltersExtractMixin:
# #region SupersetContextFiltersExtractMixin._extract_imported_filters [C:2] [TYPE Function]
# @BRIEF Normalize imported filters from decoded query state without fabricating missing values.
# [DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize imported filters from decoded query state without fabricating missing values.
def _extract_imported_filters(
self, query_state: Dict[str, Any]
) -> List[Dict[str, Any]]:
@@ -251,7 +252,7 @@ class SupersetContextFiltersExtractMixin:
return imported_filters
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
# [/DEF:SupersetContextFiltersExtractMixin._extract_imported_filters:Function]
# #endregion SupersetContextFiltersExtractMixin

28
backend/src/core/utils/superset_context_extractor/_parsing.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextParsingMixin [C:4] [TYPE Module] [SEMANTICS superset, link_parsing, url, permalink, recovery]
# @LAYER: Infra
# @BRIEF Parse supported Superset URLs and recover canonical dataset/dashboard references for review-session intake.
# @LAYER Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION CALLS -> [SupersetContextExtractorBase]
# #endregion SupersetContextParsingMixin
@@ -21,13 +21,14 @@ logger = cast(Any, logger)
# #region SupersetContextParsingMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor.
class SupersetContextParsingMixin:
# #region SupersetContextParsingMixin.parse_superset_link [C:4] [TYPE Function]
# @BRIEF Extract candidate identifiers and query state from supported Superset URLs.
# @PRE link is a non-empty Superset URL compatible with the configured environment.
# @POST returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT Input[link:str] -> Output[SupersetParsedContext]
# @RELATION CALLS -> [SupersetClient.get_dashboard_detail]
# [DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# @COMPLEXITY: 4
# @PURPOSE: Extract candidate identifiers and query state from supported Superset URLs.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
# @PRE: link is a non-empty Superset URL compatible with the configured environment.
# @POST: returns resolved dataset/dashboard context, preserving explicit partial-recovery state if some identifiers cannot be confirmed.
# @SIDE_EFFECT: may issue Superset API reads to resolve dataset references from dashboard or chart URLs.
# @DATA_CONTRACT: Input[link:str] -> Output[SupersetParsedContext]
def parse_superset_link(self, link: str) -> SupersetParsedContext:
with belief_scope("SupersetContextExtractor.parse_superset_link"):
normalized_link = str(link or "").strip()
@@ -328,11 +329,12 @@ class SupersetContextParsingMixin:
)
return result
# #endregion SupersetContextParsingMixin.parse_superset_link
# [/DEF:SupersetContextParsingMixin.parse_superset_link:Function]
# #region SupersetContextParsingMixin._recover_dataset_binding_from_dashboard [C:3] [TYPE Function]
# @BRIEF Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION CALLS -> [SupersetClient.get_dashboard_detail]
# [DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# @COMPLEXITY: 3
# @PURPOSE: Recover a dataset binding from resolved dashboard context while preserving explicit unresolved markers.
# @RELATION: CALLS -> [SupersetClient.get_dashboard_detail]
def _recover_dataset_binding_from_dashboard(
self,
dashboard_id: int,
@@ -369,7 +371,7 @@ class SupersetContextParsingMixin:
unresolved_references.append("dashboard_dataset_binding_missing")
return None, unresolved_references
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
# [/DEF:SupersetContextParsingMixin._recover_dataset_binding_from_dashboard:Function]
# #endregion SupersetContextParsingMixin

2
backend/src/core/utils/superset_context_extractor/_pii.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextExtractorPII [C:3] [TYPE Module] [SEMANTICS superset, pii, masking, sanitization, privacy]
# @LAYER: Infra
# @BRIEF PII redaction helpers for assistant-facing dataset-review context — mask emails, UUIDs, long digit strings, and mixed identifiers.
# @LAYER Infra
# #endregion SupersetContextExtractorPII
# #region _pii_imports [TYPE Block]

26
backend/src/core/utils/superset_context_extractor/_recovery.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextRecoveryMixin [C:4] [TYPE Module] [SEMANTICS superset, filters, recovery, imported_filters]
# @LAYER: Infra
# @BRIEF Recover imported filters from Superset parsed context and dashboard metadata.
# @LAYER Infra
# @RELATION CALLS -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextRecoveryMixin
@@ -22,13 +22,14 @@ logger = cast(Any, logger)
# #region SupersetContextRecoveryMixin [C:4] [TYPE Class]
# @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor.
class SupersetContextRecoveryMixin:
# #region SupersetContextRecoveryMixin.recover_imported_filters [C:4] [TYPE Function]
# @BRIEF Build imported filter entries from URL state and Superset-side saved context.
# @PRE parsed_context comes from a successful Superset link parse for one environment.
# @POST returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
# @RELATION CALLS -> [SupersetClient.get_dashboard]
# [DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# @COMPLEXITY: 4
# @PURPOSE: Build imported filter entries from URL state and Superset-side saved context.
# @RELATION: CALLS -> [SupersetClient.get_dashboard]
# @PRE: parsed_context comes from a successful Superset link parse for one environment.
# @POST: returns explicit recovered and partial filter entries with preserved provenance and confirmation requirements.
# @SIDE_EFFECT: may issue Superset reads for dashboard metadata enrichment.
# @DATA_CONTRACT: Input[SupersetParsedContext] -> Output[List[Dict[str,Any]]]
def recover_imported_filters(
self, parsed_context: SupersetParsedContext
) -> List[Dict[str, Any]]:
@@ -259,10 +260,11 @@ class SupersetContextRecoveryMixin:
)
return recovered_filters
# #endregion SupersetContextRecoveryMixin.recover_imported_filters
# [/DEF:SupersetContextRecoveryMixin.recover_imported_filters:Function]
# #region SupersetContextRecoveryMixin._normalize_imported_filter_payload [C:2] [TYPE Function]
# @BRIEF Normalize one imported-filter payload with explicit provenance and confirmation state.
# [DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize one imported-filter payload with explicit provenance and confirmation state.
def _normalize_imported_filter_payload(
self,
payload: Dict[str, Any],
@@ -303,7 +305,7 @@ class SupersetContextRecoveryMixin:
"notes": str(payload.get("notes") or default_note),
}
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
# [/DEF:SupersetContextRecoveryMixin._normalize_imported_filter_payload:Function]
# #endregion SupersetContextRecoveryMixin

47
backend/src/core/utils/superset_context_extractor/_templates.py
View File

@@ -1,6 +1,6 @@
# #region SupersetContextTemplatesMixin [C:3] [TYPE Module] [SEMANTICS superset, template_variables, jinja, discovery, deterministic]
# @LAYER: Infra
# @BRIEF Deterministically detect runtime variables and Jinja references from dataset query-bearing fields without execution.
# @LAYER Infra
# @RELATION DEPENDS_ON -> [TemplateVariable]
# @RELATION DEPENDS_ON -> [SupersetContextExtractorBase]
# #endregion SupersetContextTemplatesMixin
@@ -20,12 +20,13 @@ logger = cast(Any, logger)
# #region SupersetContextTemplatesMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor.
class SupersetContextTemplatesMixin:
# #region SupersetContextTemplatesMixin.discover_template_variables [C:4] [TYPE Function]
# @BRIEF Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT none.
# @DATA_CONTRACT Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
# [DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# @COMPLEXITY: 4
# @PURPOSE: Detect runtime variables and Jinja references from dataset query-bearing fields.
# @PRE: dataset_payload is a Superset dataset-detail style payload with query-bearing fields when available.
# @POST: returns deduplicated explicit variable records without executing Jinja or fabricating runtime values.
# @SIDE_EFFECT: none.
# @DATA_CONTRACT: Input[dataset_payload:Dict[str,Any]] -> Output[List[Dict[str,Any]]]
def discover_template_variables(
self, dataset_payload: Dict[str, Any]
) -> List[Dict[str, Any]]:
@@ -117,11 +118,12 @@ class SupersetContextTemplatesMixin:
)
return discovered
# #endregion SupersetContextTemplatesMixin.discover_template_variables
# [/DEF:SupersetContextTemplatesMixin.discover_template_variables:Function]
# #region SupersetContextTemplatesMixin._collect_query_bearing_expressions [C:3] [TYPE Function]
# @BRIEF Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
# [DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# @COMPLEXITY: 3
# @PURPOSE: Collect SQL and expression-bearing dataset fields for deterministic template-variable discovery.
# @RELATION: DEPENDS_ON -> [SupersetContextTemplatesMixin.discover_template_variables]
def _collect_query_bearing_expressions(
self, dataset_payload: Dict[str, Any]
) -> List[str]:
@@ -160,10 +162,11 @@ class SupersetContextTemplatesMixin:
return expressions
# #endregion SupersetContextTemplatesMixin._collect_query_bearing_expressions
# [/DEF:SupersetContextTemplatesMixin._collect_query_bearing_expressions:Function]
# #region SupersetContextTemplatesMixin._append_template_variable [C:2] [TYPE Function]
# @BRIEF Append one deduplicated template-variable descriptor.
# [DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# @COMPLEXITY: 2
# @PURPOSE: Append one deduplicated template-variable descriptor.
def _append_template_variable(
self,
discovered: List[Dict[str, Any]],
@@ -192,10 +195,11 @@ class SupersetContextTemplatesMixin:
}
)
# #endregion SupersetContextTemplatesMixin._append_template_variable
# [/DEF:SupersetContextTemplatesMixin._append_template_variable:Function]
# #region SupersetContextTemplatesMixin._extract_primary_jinja_identifier [C:2] [TYPE Function]
# @BRIEF Extract a deterministic primary identifier from a Jinja expression without executing it.
# [DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# @COMPLEXITY: 2
# @PURPOSE: Extract a deterministic primary identifier from a Jinja expression without executing it.
def _extract_primary_jinja_identifier(self, expression: str) -> Optional[str]:
matched = re.match(r"([A-Za-z_][A-Za-z0-9_]*)", expression.strip())
if matched is None:
@@ -214,10 +218,11 @@ class SupersetContextTemplatesMixin:
return None
return candidate
# #endregion SupersetContextTemplatesMixin._extract_primary_jinja_identifier
# [/DEF:SupersetContextTemplatesMixin._extract_primary_jinja_identifier:Function]
# #region SupersetContextTemplatesMixin._normalize_default_literal [C:2] [TYPE Function]
# @BRIEF Normalize literal default fragments from template helper calls into JSON-safe values.
# [DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# @COMPLEXITY: 2
# @PURPOSE: Normalize literal default fragments from template helper calls into JSON-safe values.
def _normalize_default_literal(self, literal: Optional[str]) -> Any:
normalized_literal = str(literal or "").strip()
if not normalized_literal:
@@ -241,7 +246,7 @@ class SupersetContextTemplatesMixin:
except ValueError:
return normalized_literal
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
# [/DEF:SupersetContextTemplatesMixin._normalize_default_literal:Function]
# #endregion SupersetContextTemplatesMixin