busya
8c10632494
- Add @RATIONALE/@REJECTED to 103+ C4/C5 contracts across backend core, services, API routes, and frontend models - Fix 109 unresolved @RELATION edges (Auth.*, SupersetClient.*, AgentChat.*, ADR cross-refs) - Add 13 @ingroup tags for DSA/HCA attention grouping - Repair 29 stale graph edges via index rebuild - Update .kilo agent prompts and skills for GRACE-Poly v2.6 compliance - Git integration: merge routes, branch lifecycle, remote providers, UX components - 0 broken anchor pairs, index rebuilt with 0 parse warnings
252 lines
12 KiB
Python
252 lines
12 KiB
Python
# #region IdMappingServiceModule [C:5] [TYPE Module] [SEMANTICS sqlalchemy, mapping, sync, superset, resource]
|
|
# @defgroup Core Module group.
|
|
#
|
|
# @BRIEF Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
|
|
# @LAYER Core
|
|
# @RELATION DEPENDS_ON -> [MappingModels]
|
|
# @RELATION DEPENDS_ON -> [LoggerModule]
|
|
# @RELATION DEPENDS_ON -> [AsyncJobRunner]
|
|
# @PRE Database session is valid and Superset client factory returns authenticated clients for requested environments.
|
|
# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
|
|
# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs.
|
|
# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
|
|
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
|
|
#
|
|
# @INVARIANT sync_environment must handle remote API failures gracefully.
|
|
# @RATIONALE Centralizes UUID-to-integer ID resolution for Superset resources because the Superset API uses different ID schemes across endpoints (UUIDs for import/export, integer IDs for CRUD operations), enabling cross-environment migration with consistent resource references.
|
|
# @REJECTED BackgroundScheduler — was never started; replaced by AsyncJobRunner for async/sync bridge.
|
|
from datetime import UTC, datetime
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from src.core.cot_logger import seed_trace_id
|
|
from src.core.logger import belief_scope, logger
|
|
from src.models.mapping import ResourceMapping, ResourceType
|
|
|
|
|
|
# #region IdMappingService [C:5] [TYPE Class]
|
|
# @defgroup Core Module group.
|
|
# @BRIEF Service handling the cataloging and retrieval of remote Superset Integer IDs.
|
|
# @PRE db_session is an active SQLAlchemy Session bound to mapping tables.
|
|
# @POST Service instance provides environment-scoped mapping synchronization APIs.
|
|
# @RELATION DEPENDS_ON -> [MappingModels]
|
|
# @RELATION DEPENDS_ON -> [LoggerModule]
|
|
# @RELATION DEPENDS_ON -> [AsyncJobRunner]
|
|
# @INVARIANT self.db remains the authoritative session for all mapping operations.
|
|
# @SIDE_EFFECT Performs database writes during sync cycles.
|
|
# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService]
|
|
#
|
|
# @TEST_CONTRACT IdMappingServiceModel ->
|
|
# {
|
|
# required_fields: {db_session: Session},
|
|
# invariants: [
|
|
# "sync_environment correctly creates or updates ResourceMapping records",
|
|
# "get_remote_id returns an integer or None",
|
|
# "get_remote_ids_batch returns a dictionary of valid UUIDs to integers"
|
|
# ]
|
|
# }
|
|
# @TEST_FIXTURE valid_mapping_service -> {"db_session": "MockSession()"}
|
|
# @TEST_EDGE sync_api_failure -> handles exception gracefully
|
|
# @TEST_EDGE get_remote_id_not_found -> returns None
|
|
# @TEST_EDGE get_batch_empty_list -> returns empty dict
|
|
# @TEST_INVARIANT resilient_fetching -> verifies: [sync_api_failure]
|
|
class IdMappingService:
|
|
# #region __init__ [TYPE Function]
|
|
# @PURPOSE: Initializes the mapping service.
|
|
def __init__(self, db_session: Session):
|
|
self.db = db_session
|
|
# #endregion __init__
|
|
# #region sync_environment [TYPE Function]
|
|
# @ingroup Core
|
|
# @PURPOSE: Fully synchronizes mapping for a specific environment.
|
|
# @PARAM environment_id (str) - Target environment ID.
|
|
# @PARAM superset_client - Instance capable of hitting the Superset API.
|
|
# @PRE environment_id exists in the database.
|
|
# @POST ResourceMapping records for the environment are created or updated.
|
|
async def sync_environment(
|
|
self, environment_id: str, superset_client, incremental: bool = False
|
|
) -> None:
|
|
"""
|
|
Polls the Superset APIs for the target environment and updates the local mapping table.
|
|
If incremental=True, only fetches items changed since the max last_synced_at date.
|
|
"""
|
|
seed_trace_id()
|
|
with belief_scope("IdMappingService.sync_environment"):
|
|
logger.reason(
|
|
f"Starting sync for environment {environment_id}",
|
|
extra={"src": "IdMappingService.sync_environment", "payload": {"environment_id": environment_id, "incremental": incremental}},
|
|
)
|
|
# Implementation Note: In a real scenario, superset_client needs to be an instance
|
|
# capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/
|
|
# Here we structure the logic according to the spec.
|
|
types_to_poll = [
|
|
(ResourceType.CHART, "chart", "slice_name"),
|
|
(ResourceType.DATASET, "dataset", "table_name"),
|
|
(
|
|
ResourceType.DASHBOARD,
|
|
"dashboard",
|
|
"slug",
|
|
), # Note: dashboard slug or dashboard_title
|
|
]
|
|
total_synced = 0
|
|
total_deleted = 0
|
|
try:
|
|
for res_enum, endpoint, name_field in types_to_poll:
|
|
logger.debug(
|
|
"Polling %s endpoint", endpoint,
|
|
)
|
|
# Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... )
|
|
# This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] }
|
|
# We assume superset_client provides a generic method to fetch all pages.
|
|
try:
|
|
since_dttm = None
|
|
if incremental:
|
|
from sqlalchemy.sql import func
|
|
max_date = (
|
|
self.db.query(func.max(ResourceMapping.last_synced_at))
|
|
.filter(
|
|
ResourceMapping.environment_id == environment_id,
|
|
ResourceMapping.resource_type == res_enum,
|
|
)
|
|
.scalar()
|
|
)
|
|
if max_date:
|
|
# We subtract a bit for safety overlap
|
|
from datetime import timedelta
|
|
since_dttm = max_date - timedelta(minutes=5)
|
|
logger.debug(
|
|
"Incremental sync since %s", since_dttm,
|
|
)
|
|
resources = await superset_client.get_all_resources(
|
|
endpoint, since_dttm=since_dttm
|
|
)
|
|
# Track which UUIDs we see in this sync cycle
|
|
synced_uuids = set()
|
|
for res in resources:
|
|
res_uuid = res.get("uuid")
|
|
raw_id = res.get("id")
|
|
res_name = res.get(name_field)
|
|
if not res_uuid or raw_id is None:
|
|
continue
|
|
synced_uuids.add(res_uuid)
|
|
res_id = str(raw_id) # Store as string
|
|
# Upsert Logic
|
|
mapping = (
|
|
self.db.query(ResourceMapping)
|
|
.filter_by(
|
|
environment_id=environment_id,
|
|
resource_type=res_enum,
|
|
uuid=res_uuid,
|
|
)
|
|
.first()
|
|
)
|
|
if mapping:
|
|
mapping.remote_integer_id = res_id
|
|
mapping.resource_name = res_name
|
|
mapping.last_synced_at = datetime.now(UTC)
|
|
else:
|
|
new_mapping = ResourceMapping(
|
|
environment_id=environment_id,
|
|
resource_type=res_enum,
|
|
uuid=res_uuid,
|
|
remote_integer_id=res_id,
|
|
resource_name=res_name,
|
|
last_synced_at=datetime.now(UTC),
|
|
)
|
|
self.db.add(new_mapping)
|
|
total_synced += 1
|
|
# Delete stale mappings: rows for this env+type whose UUID
|
|
# was NOT returned by the API (resource was deleted remotely)
|
|
# We only do this on full syncs, because incremental syncs don't return all UUIDs
|
|
if not incremental:
|
|
stale_query = self.db.query(ResourceMapping).filter(
|
|
ResourceMapping.environment_id == environment_id,
|
|
ResourceMapping.resource_type == res_enum,
|
|
)
|
|
if synced_uuids:
|
|
stale_query = stale_query.filter(
|
|
ResourceMapping.uuid.notin_(synced_uuids)
|
|
)
|
|
deleted = stale_query.delete(synchronize_session="fetch")
|
|
if deleted:
|
|
total_deleted += deleted
|
|
logger.reason(
|
|
"Removed stale mappings",
|
|
payload={"deleted": deleted, "endpoint": endpoint, "environment_id": environment_id},
|
|
)
|
|
except Exception as loop_e:
|
|
logger.explore(
|
|
"Error polling endpoint",
|
|
error=str(loop_e),
|
|
payload={"endpoint": endpoint},
|
|
)
|
|
# Continue to next resource type instead of blowing up the whole sync
|
|
self.db.commit()
|
|
logger.reflect(
|
|
"Successfully synced environment",
|
|
payload={"synced": total_synced, "deleted_stale": total_deleted},
|
|
)
|
|
except Exception as e:
|
|
self.db.rollback()
|
|
logger.explore(
|
|
"Critical sync failure",
|
|
error=str(e),
|
|
)
|
|
raise
|
|
# #endregion sync_environment
|
|
# #region get_remote_id [TYPE Function]
|
|
# @ingroup Core
|
|
# @PURPOSE: Retrieves the remote integer ID for a given universal UUID.
|
|
# @PARAM environment_id (str)
|
|
# @PARAM resource_type (ResourceType)
|
|
# @PARAM uuid (str)
|
|
# @RETURN Optional[int]
|
|
def get_remote_id(
|
|
self, environment_id: str, resource_type: ResourceType, uuid: str
|
|
) -> int | None:
|
|
mapping = (
|
|
self.db.query(ResourceMapping)
|
|
.filter_by(
|
|
environment_id=environment_id, resource_type=resource_type, uuid=uuid
|
|
)
|
|
.first()
|
|
)
|
|
if mapping:
|
|
try:
|
|
return int(mapping.remote_integer_id)
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
# #endregion get_remote_id
|
|
# #region get_remote_ids_batch [TYPE Function]
|
|
# @ingroup Core
|
|
# @PURPOSE: Retrieves remote integer IDs for a list of universal UUIDs efficiently.
|
|
# @PARAM environment_id (str)
|
|
# @PARAM resource_type (ResourceType)
|
|
# @PARAM uuids (List[str])
|
|
# @RETURN Dict[str, int] - Mapping of UUID -> Integer ID
|
|
def get_remote_ids_batch(
|
|
self, environment_id: str, resource_type: ResourceType, uuids: list[str]
|
|
) -> dict[str, int]:
|
|
if not uuids:
|
|
return {}
|
|
mappings = (
|
|
self.db.query(ResourceMapping)
|
|
.filter(
|
|
ResourceMapping.environment_id == environment_id,
|
|
ResourceMapping.resource_type == resource_type,
|
|
ResourceMapping.uuid.in_(uuids),
|
|
)
|
|
.all()
|
|
)
|
|
result = {}
|
|
for m in mappings:
|
|
try:
|
|
result[m.uuid] = int(m.remote_integer_id)
|
|
except ValueError:
|
|
logger.debug("Could not parse remote_integer_id for mapping %s (uuid=%s)", m.id, m.uuid)
|
|
return result
|
|
# #endregion get_remote_ids_batch
|
|
# #endregion IdMappingService
|
|
# #endregion IdMappingServiceModule
|