# #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, mapping, sync, superset, resource] # @defgroup Core Module group. # # @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID) # @LAYER Core # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [LoggerModule] # @RELATION DEPENDS_ON -> [AsyncJobRunner] # @PRE Database session is valid and Superset client factory returns authenticated clients for requested environments. # @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution. # @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs. # @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None] # @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]} # # @INVARIANT sync_environment must handle remote API failures gracefully. # @RATIONALE Centralizes UUID-to-integer ID resolution for Superset resources because the Superset API uses different ID schemes across endpoints (UUIDs for import/export, integer IDs for CRUD operations), enabling cross-environment migration with consistent resource references. # @REJECTED BackgroundScheduler — was never started; replaced by AsyncJobRunner for async/sync bridge. from datetime import UTC, datetime from sqlalchemy.orm import Session from src.core.cot_logger import seed_trace_id from src.core.logger import belief_scope, logger from src.models.mapping import ResourceMapping, ResourceType # #region IdMappingService [C:5] [TYPE Class] # @defgroup Core Module group. # @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs. # @PRE db_session is an active SQLAlchemy Session bound to mapping tables. # @POST Service instance provides environment-scoped mapping synchronization APIs. # @RELATION DEPENDS_ON -> [MappingModels] # @RELATION DEPENDS_ON -> [LoggerModule] # @RELATION DEPENDS_ON -> [AsyncJobRunner] # @INVARIANT self.db remains the authoritative session for all mapping operations. # @SIDE_EFFECT Performs database writes during sync cycles. # @DATA_CONTRACT Input[db_session] -> Output[IdMappingService] # # @TEST_CONTRACT IdMappingServiceModel -> # { # required_fields: {db_session: Session}, # invariants: [ # "sync_environment correctly creates or updates ResourceMapping records", # "get_remote_id returns an integer or None", # "get_remote_ids_batch returns a dictionary of valid UUIDs to integers" # ] # } # @TEST_FIXTURE valid_mapping_service -> {"db_session": "MockSession()"} # @TEST_EDGE sync_api_failure -> handles exception gracefully # @TEST_EDGE get_remote_id_not_found -> returns None # @TEST_EDGE get_batch_empty_list -> returns empty dict # @TEST_INVARIANT resilient_fetching -> verifies: [sync_api_failure] class IdMappingService: # #region __init__ [TYPE Function] # @PURPOSE: Initializes the mapping service. def __init__(self, db_session: Session): self.db = db_session # #endregion __init__ # #region sync_environment [TYPE Function] # @ingroup Core # @PURPOSE: Fully synchronizes mapping for a specific environment. # @PARAM environment_id (str) - Target environment ID. # @PARAM superset_client - Instance capable of hitting the Superset API. # @PRE environment_id exists in the database. # @POST ResourceMapping records for the environment are created or updated. async def sync_environment( self, environment_id: str, superset_client, incremental: bool = False ) -> None: """ Polls the Superset APIs for the target environment and updates the local mapping table. If incremental=True, only fetches items changed since the max last_synced_at date. """ seed_trace_id() with belief_scope("IdMappingService.sync_environment"): logger.reason( f"Starting sync for environment {environment_id}", extra={"src": "IdMappingService.sync_environment", "payload": {"environment_id": environment_id, "incremental": incremental}}, ) # Implementation Note: In a real scenario, superset_client needs to be an instance # capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/ # Here we structure the logic according to the spec. types_to_poll = [ (ResourceType.CHART, "chart", "slice_name"), (ResourceType.DATASET, "dataset", "table_name"), ( ResourceType.DASHBOARD, "dashboard", "slug", ), # Note: dashboard slug or dashboard_title ] total_synced = 0 total_deleted = 0 try: for res_enum, endpoint, name_field in types_to_poll: logger.debug( "Polling %s endpoint", endpoint, ) # Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... ) # This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] } # We assume superset_client provides a generic method to fetch all pages. try: since_dttm = None if incremental: from sqlalchemy.sql import func max_date = ( self.db.query(func.max(ResourceMapping.last_synced_at)) .filter( ResourceMapping.environment_id == environment_id, ResourceMapping.resource_type == res_enum, ) .scalar() ) if max_date: # We subtract a bit for safety overlap from datetime import timedelta since_dttm = max_date - timedelta(minutes=5) logger.debug( "Incremental sync since %s", since_dttm, ) resources = await superset_client.get_all_resources( endpoint, since_dttm=since_dttm ) # Track which UUIDs we see in this sync cycle synced_uuids = set() for res in resources: res_uuid = res.get("uuid") raw_id = res.get("id") res_name = res.get(name_field) if not res_uuid or raw_id is None: continue synced_uuids.add(res_uuid) res_id = str(raw_id) # Store as string # Upsert Logic mapping = ( self.db.query(ResourceMapping) .filter_by( environment_id=environment_id, resource_type=res_enum, uuid=res_uuid, ) .first() ) if mapping: mapping.remote_integer_id = res_id mapping.resource_name = res_name mapping.last_synced_at = datetime.now(UTC) else: new_mapping = ResourceMapping( environment_id=environment_id, resource_type=res_enum, uuid=res_uuid, remote_integer_id=res_id, resource_name=res_name, last_synced_at=datetime.now(UTC), ) self.db.add(new_mapping) total_synced += 1 # Delete stale mappings: rows for this env+type whose UUID # was NOT returned by the API (resource was deleted remotely) # We only do this on full syncs, because incremental syncs don't return all UUIDs if not incremental: stale_query = self.db.query(ResourceMapping).filter( ResourceMapping.environment_id == environment_id, ResourceMapping.resource_type == res_enum, ) if synced_uuids: stale_query = stale_query.filter( ResourceMapping.uuid.notin_(synced_uuids) ) deleted = stale_query.delete(synchronize_session="fetch") if deleted: total_deleted += deleted logger.reason( "Removed stale mappings", payload={"deleted": deleted, "endpoint": endpoint, "environment_id": environment_id}, ) except Exception as loop_e: logger.explore( "Error polling endpoint", error=str(loop_e), payload={"endpoint": endpoint}, ) # Continue to next resource type instead of blowing up the whole sync self.db.commit() logger.reflect( "Successfully synced environment", payload={"synced": total_synced, "deleted_stale": total_deleted}, ) except Exception as e: self.db.rollback() logger.explore( "Critical sync failure", error=str(e), ) raise # #endregion sync_environment # #region get_remote_id [TYPE Function] # @ingroup Core # @PURPOSE: Retrieves the remote integer ID for a given universal UUID. # @PARAM environment_id (str) # @PARAM resource_type (ResourceType) # @PARAM uuid (str) # @RETURN Optional[int] def get_remote_id( self, environment_id: str, resource_type: ResourceType, uuid: str ) -> int | None: mapping = ( self.db.query(ResourceMapping) .filter_by( environment_id=environment_id, resource_type=resource_type, uuid=uuid ) .first() ) if mapping: try: return int(mapping.remote_integer_id) except ValueError: return None return None # #endregion get_remote_id # #region get_remote_ids_batch [TYPE Function] # @ingroup Core # @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently. # @PARAM environment_id (str) # @PARAM resource_type (ResourceType) # @PARAM uuids (List[str]) # @RETURN Dict[str, int] - Mapping of UUID -> Integer ID def get_remote_ids_batch( self, environment_id: str, resource_type: ResourceType, uuids: list[str] ) -> dict[str, int]: if not uuids: return {} mappings = ( self.db.query(ResourceMapping) .filter( ResourceMapping.environment_id == environment_id, ResourceMapping.resource_type == resource_type, ResourceMapping.uuid.in_(uuids), ) .all() ) result = {} for m in mappings: try: result[m.uuid] = int(m.remote_integer_id) except ValueError: logger.debug("Could not parse remote_integer_id for mapping %s (uuid=%s)", m.id, m.uuid) return result # #endregion get_remote_ids_batch # #endregion IdMappingService # #endregion IdMappingServiceModule