busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics: complete DEF-to-region migration, fix regressions

Browse Source 
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
This commit is contained in:
busya
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
backend/src/core/mapping_service.py
View File

@@ -1,40 +1,35 @@
# #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS mapping, ids, synchronization, environments, cross-filters]
#
# @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
# @LAYER Core
# @PRE Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @LAYER: Core
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [LoggerModule]
#
# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
#
# @INVARIANT: sync_environment must handle remote API failures gracefully.
# [SECTION: IMPORTS]
from typing import Dict, List, Optional
from datetime import datetime, timezone
from sqlalchemy.orm import Session
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from src.models.mapping import ResourceMapping, ResourceType
from src.core.cot_logger import MarkerLogger
from src.core.logger import logger, belief_scope
log = MarkerLogger("IdMapping")
# [/SECTION]
# #region IdMappingService [C:5] [TYPE Class]
# @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs.
# @PRE db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @SIDE_EFFECT Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService]
# @INVARIANT self.db remains the authoritative session for all mapping operations.
# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [LoggerModule]
# @INVARIANT: self.db remains the authoritative session for all mapping operations.
# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]
#
# @TEST_CONTRACT: IdMappingServiceModel ->
# {
@@ -51,17 +46,17 @@ log = MarkerLogger("IdMapping")
# @TEST_EDGE: get_batch_empty_list -> returns empty dict
# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]
class IdMappingService:
# #region __init__ [TYPE Function]
# @BRIEF Initializes the mapping service.
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapping service.
def __init__(self, db_session: Session):
self.db = db_session
self.scheduler = BackgroundScheduler()
self._sync_job = None
# #endregion __init__
# [/DEF:__init__:Function]
# #region start_scheduler [TYPE Function]
# @BRIEF Starts the background scheduler with a given cron string.
# [DEF:start_scheduler:Function]
# @PURPOSE: Starts the background scheduler with a given cron string.
# @PARAM: cron_string (str) - Cron expression for the sync interval.
# @PARAM: environments (List[str]) - List of environment IDs to sync.
# @PARAM: superset_client_factory - Function to get a client for an environment.
@@ -71,7 +66,9 @@ class IdMappingService:
with belief_scope("IdMappingService.start_scheduler"):
if self._sync_job:
self.scheduler.remove_job(self._sync_job.id)
log.reflect("Removed existing sync job.")
logger.info(
"[IdMappingService.start_scheduler][Reflect] Removed existing sync job."
)
def sync_all():
for env_id in environments:
@@ -88,14 +85,18 @@ class IdMappingService:
if not self.scheduler.running:
self.scheduler.start()
log.reason(f"Started background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}"
)
else:
log.reason(f"Updated background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}"
)
# #endregion start_scheduler
# [/DEF:start_scheduler:Function]
# #region sync_environment [TYPE Function]
# @BRIEF Fully synchronizes mapping for a specific environment.
# [DEF:sync_environment:Function]
# @PURPOSE: Fully synchronizes mapping for a specific environment.
# @PARAM: environment_id (str) - Target environment ID.
# @PARAM: superset_client - Instance capable of hitting the Superset API.
# @PRE: environment_id exists in the database.
@@ -108,7 +109,9 @@ class IdMappingService:
If incremental=True, only fetches items changed since the max last_synced_at date.
"""
with belief_scope("IdMappingService.sync_environment"):
log.reason(f"Starting sync for environment {environment_id} (incremental={incremental})")
logger.info(
f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})"
)
# Implementation Note: In a real scenario, superset_client needs to be an instance
# capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/
@@ -128,7 +131,9 @@ class IdMappingService:
total_deleted = 0
try:
for res_enum, endpoint, name_field in types_to_poll:
log.reason(f"Polling {endpoint} endpoint")
logger.debug(
f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint"
)
# Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... )
# This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] }
@@ -153,8 +158,8 @@ class IdMappingService:
from datetime import timedelta
since_dttm = max_date - timedelta(minutes=5)
log.reason(
f"Incremental sync since {since_dttm}"
logger.debug(
f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}"
)
resources = superset_client.get_all_resources(
@@ -218,24 +223,32 @@ class IdMappingService:
deleted = stale_query.delete(synchronize_session="fetch")
if deleted:
total_deleted += deleted
log.reason(f"Removed {deleted} stale {endpoint} mapping(s) for {environment_id}")
logger.info(
f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}"
)
except Exception as loop_e:
log.explore(f"Error polling {endpoint}", error=str(loop_e))
logger.error(
f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}"
)
# Continue to next resource type instead of blowing up the whole sync
self.db.commit()
log.reflect(f"Successfully synced {total_synced} items and deleted {total_deleted} stale items.")
logger.info(
f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items."
)
except Exception as e:
self.db.rollback()
log.explore("Critical sync failure", error=str(e))
logger.error(
f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}"
)
raise
# #endregion sync_environment
# [/DEF:sync_environment:Function]
# #region get_remote_id [TYPE Function]
# @BRIEF Retrieves the remote integer ID for a given universal UUID.
# [DEF:get_remote_id:Function]
# @PURPOSE: Retrieves the remote integer ID for a given universal UUID.
# @PARAM: environment_id (str)
# @PARAM: resource_type (ResourceType)
# @PARAM: uuid (str)
@@ -258,10 +271,10 @@ class IdMappingService:
return None
return None
# #endregion get_remote_id
# [/DEF:get_remote_id:Function]
# #region get_remote_ids_batch [TYPE Function]
# @BRIEF Retrieves remote integer IDs for a list of universal UUIDs efficiently.
# [DEF:get_remote_ids_batch:Function]
# @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.
# @PARAM: environment_id (str)
# @PARAM: resource_type (ResourceType)
# @PARAM: uuids (List[str])
@@ -291,7 +304,7 @@ class IdMappingService:
return result
# #endregion get_remote_ids_batch
# [/DEF:get_remote_ids_batch:Function]
# #endregion IdMappingService