semantics: complete DEF-to-region migration, fix regressions

- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files - Reinstate _normalize_timestamp_value in sql_generator.py - Fix MarkerLogger→logger migration in events.py (molecular CoT markers) - Fix dataset_review orchestrator dependencies (_build_execution_snapshot) - Fix config_manager stale-record deletion (moved to save path only) - Add 77 missing [/DEF:] closers in 5 unbalanced test files - Update assistant_chat.integration.test.js for #region format - Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
2026-05-12 23:54:55 +03:00
parent fe8978f716
commit 306c5ae742
331 changed files with 9630 additions and 10312 deletions
--- a/backend/src/core/migration/archive_parser.py
+++ b/backend/src/core/migration/archive_parser.py
@@ -1,8 +1,8 @@
 # #region MigrationArchiveParserModule [C:3] [TYPE Module] [SEMANTICS migration, zip, parser, yaml, metadata]
 # @BRIEF Parse Superset export ZIP archives into normalized object catalogs for diffing.
-# @LAYER Core
-# @INVARIANT Parsing is read-only and never mutates archive files.
+# @LAYER: Core
 # @RELATION DEPENDS_ON -> [LoggerModule]
+# @INVARIANT: Parsing is read-only and never mutates archive files.

 import json
 import tempfile
@@ -21,12 +21,12 @@ from ..logger import logger, belief_scope
 # @RELATION CONTAINS -> [_collect_yaml_objects]
 # @RELATION CONTAINS -> [_normalize_object_payload]
 class MigrationArchiveParser:
-    # #region extract_objects_from_zip [TYPE Function]
-    # @BRIEF Extract object catalogs from Superset archive.
-    # @PRE zip_path points to a valid readable ZIP.
-    # @POST Returns object lists grouped by resource type.
-    # @RETURN Dict[str, List[Dict[str, Any]]]
-    # @RELATION DEPENDS_ON -> [_collect_yaml_objects]
+    # [DEF:extract_objects_from_zip:Function]
+    # @PURPOSE: Extract object catalogs from Superset archive.
+    # @RELATION: DEPENDS_ON -> [_collect_yaml_objects]
+    # @PRE: zip_path points to a valid readable ZIP.
+    # @POST: Returns object lists grouped by resource type.
+    # @RETURN: Dict[str, List[Dict[str, Any]]]
    def extract_objects_from_zip(
        self, zip_path: str
    ) -> Dict[str, List[Dict[str, Any]]]:
@@ -49,13 +49,13 @@ class MigrationArchiveParser:

            return result

-    # #endregion extract_objects_from_zip
+    # [/DEF:extract_objects_from_zip:Function]

-    # #region _collect_yaml_objects [TYPE Function]
-    # @BRIEF Read and normalize YAML manifests for one object type.
-    # @PRE object_type is one of dashboards/charts/datasets.
-    # @POST Returns only valid normalized objects.
-    # @RELATION DEPENDS_ON -> [_normalize_object_payload]
+    # [DEF:_collect_yaml_objects:Function]
+    # @PURPOSE: Read and normalize YAML manifests for one object type.
+    # @RELATION: DEPENDS_ON -> [_normalize_object_payload]
+    # @PRE: object_type is one of dashboards/charts/datasets.
+    # @POST: Returns only valid normalized objects.
    def _collect_yaml_objects(
        self, root_dir: Path, object_type: str
    ) -> List[Dict[str, Any]]:
@@ -79,12 +79,12 @@ class MigrationArchiveParser:
                    )
            return objects

-    # #endregion _collect_yaml_objects
+    # [/DEF:_collect_yaml_objects:Function]

-    # #region _normalize_object_payload [TYPE Function]
-    # @BRIEF Convert raw YAML payload to stable diff signature shape.
-    # @PRE payload is parsed YAML mapping.
-    # @POST Returns normalized descriptor with `uuid`, `title`, and `signature`.
+    # [DEF:_normalize_object_payload:Function]
+    # @PURPOSE: Convert raw YAML payload to stable diff signature shape.
+    # @PRE: payload is parsed YAML mapping.
+    # @POST: Returns normalized descriptor with `uuid`, `title`, and `signature`.
    def _normalize_object_payload(
        self, payload: Dict[str, Any], object_type: str
    ) -> Optional[Dict[str, Any]]:
@@ -149,7 +149,7 @@ class MigrationArchiveParser:

            return None

-    # #endregion _normalize_object_payload
+    # [/DEF:_normalize_object_payload:Function]


 # #endregion MigrationArchiveParser
--- a/backend/src/core/migration/dry_run_orchestrator.py
+++ b/backend/src/core/migration/dry_run_orchestrator.py
@@ -1,11 +1,11 @@
 # #region MigrationDryRunOrchestratorModule [C:3] [TYPE Module] [SEMANTICS migration, dry_run, diff, risk, superset]
 # @BRIEF Compute pre-flight migration diff and risk scoring without apply.
-# @LAYER Core
-# @INVARIANT Dry run is informative only and must not mutate target environment.
+# @LAYER: Core
 # @RELATION DEPENDS_ON -> [SupersetClient]
 # @RELATION DEPENDS_ON -> [MigrationEngine]
 # @RELATION DEPENDS_ON -> [MigrationArchiveParser]
 # @RELATION DEPENDS_ON -> [RiskAssessorModule]
+# @INVARIANT: Dry run is informative only and must not mutate target environment.

 from datetime import datetime, timezone
 import json
@@ -34,25 +34,25 @@ from ..utils.fileio import create_temp_file
 # @RELATION CONTAINS -> [_build_target_signatures]
 # @RELATION CONTAINS -> [_build_risks]
 class MigrationDryRunService:
-    # #region __init__ [TYPE Function]
-    # @BRIEF Wire parser dependency for archive object extraction.
-    # @PRE parser can be omitted to use default implementation.
-    # @POST Service is ready to calculate dry-run payload.
+    # [DEF:__init__:Function]
+    # @PURPOSE: Wire parser dependency for archive object extraction.
+    # @PRE: parser can be omitted to use default implementation.
+    # @POST: Service is ready to calculate dry-run payload.
    def __init__(self, parser: MigrationArchiveParser | None = None):
        self.parser = parser or MigrationArchiveParser()

-    # #endregion __init__
+    # [/DEF:__init__:Function]

-    # #region run [TYPE Function]
-    # @BRIEF Execute full dry-run computation for selected dashboards.
-    # @PRE source/target clients are authenticated and selection validated by caller.
-    # @POST Returns JSON-serializable pre-flight payload with summary, diff and risk.
-    # @SIDE_EFFECT Reads source export archives and target metadata via network.
-    # @RELATION DEPENDS_ON -> [_load_db_mapping]
-    # @RELATION DEPENDS_ON -> [_accumulate_objects]
-    # @RELATION DEPENDS_ON -> [_build_target_signatures]
-    # @RELATION DEPENDS_ON -> [_build_object_diff]
-    # @RELATION DEPENDS_ON -> [_build_risks]
+    # [DEF:run:Function]
+    # @PURPOSE: Execute full dry-run computation for selected dashboards.
+    # @RELATION: DEPENDS_ON -> [_load_db_mapping]
+    # @RELATION: DEPENDS_ON -> [_accumulate_objects]
+    # @RELATION: DEPENDS_ON -> [_build_target_signatures]
+    # @RELATION: DEPENDS_ON -> [_build_object_diff]
+    # @RELATION: DEPENDS_ON -> [_build_risks]
+    # @PRE: source/target clients are authenticated and selection validated by caller.
+    # @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk.
+    # @SIDE_EFFECT: Reads source export archives and target metadata via network.
    def run(
        self,
        selection: DashboardSelection,
@@ -154,10 +154,10 @@ class MigrationDryRunService:
                "risk": score_risks(risk),
            }

-    # #endregion run
+    # [/DEF:run:Function]

-    # #region _load_db_mapping [TYPE Function]
-    # @BRIEF Resolve UUID mapping for optional DB config replacement.
+    # [DEF:_load_db_mapping:Function]
+    # @PURPOSE: Resolve UUID mapping for optional DB config replacement.
    def _load_db_mapping(
        self, db: Session, selection: DashboardSelection
    ) -> Dict[str, str]:
@@ -171,10 +171,10 @@ class MigrationDryRunService:
        )
        return {row.source_db_uuid: row.target_db_uuid for row in rows}

-    # #endregion _load_db_mapping
+    # [/DEF:_load_db_mapping:Function]

-    # #region _accumulate_objects [TYPE Function]
-    # @BRIEF Merge extracted resources by UUID to avoid duplicates.
+    # [DEF:_accumulate_objects:Function]
+    # @PURPOSE: Merge extracted resources by UUID to avoid duplicates.
    def _accumulate_objects(
        self,
        target: Dict[str, Dict[str, Dict[str, Any]]],
@@ -186,10 +186,10 @@ class MigrationDryRunService:
                if uuid:
                    target[object_type][str(uuid)] = item

-    # #endregion _accumulate_objects
+    # [/DEF:_accumulate_objects:Function]

-    # #region _index_by_uuid [TYPE Function]
-    # @BRIEF Build UUID-index map for normalized resources.
+    # [DEF:_index_by_uuid:Function]
+    # @PURPOSE: Build UUID-index map for normalized resources.
    def _index_by_uuid(
        self, objects: List[Dict[str, Any]]
    ) -> Dict[str, Dict[str, Any]]:
@@ -200,11 +200,11 @@ class MigrationDryRunService:
                indexed[str(uuid)] = obj
        return indexed

-    # #endregion _index_by_uuid
+    # [/DEF:_index_by_uuid:Function]

-    # #region _build_object_diff [TYPE Function]
-    # @BRIEF Compute create/update/delete buckets by UUID+signature.
-    # @RELATION DEPENDS_ON -> [_index_by_uuid]
+    # [DEF:_build_object_diff:Function]
+    # @PURPOSE: Compute create/update/delete buckets by UUID+signature.
+    # @RELATION: DEPENDS_ON -> [_index_by_uuid]
    def _build_object_diff(
        self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]]
    ) -> Dict[str, List[Dict[str, Any]]]:
@@ -228,10 +228,10 @@ class MigrationDryRunService:
                )
        return {"create": created, "update": updated, "delete": deleted}

-    # #endregion _build_object_diff
+    # [/DEF:_build_object_diff:Function]

-    # #region _build_target_signatures [TYPE Function]
-    # @BRIEF Pull target metadata and normalize it into comparable signatures.
+    # [DEF:_build_target_signatures:Function]
+    # @PURPOSE: Pull target metadata and normalize it into comparable signatures.
    def _build_target_signatures(
        self, client: SupersetClient
    ) -> Dict[str, List[Dict[str, Any]]]:
@@ -341,10 +341,10 @@ class MigrationDryRunService:
            ],
        }

-    # #endregion _build_target_signatures
+    # [/DEF:_build_target_signatures:Function]

-    # #region _build_risks [TYPE Function]
-    # @BRIEF Build risk items for missing datasource, broken refs, overwrite, owner mismatch.
+    # [DEF:_build_risks:Function]
+    # @PURPOSE: Build risk items for missing datasource, broken refs, overwrite, owner mismatch.
    def _build_risks(
        self,
        source_objects: Dict[str, List[Dict[str, Any]]],
@@ -354,7 +354,7 @@ class MigrationDryRunService:
    ) -> List[Dict[str, Any]]:
        return build_risks(source_objects, target_objects, diff, target_client)

-    # #endregion _build_risks
+    # [/DEF:_build_risks:Function]


 # #endregion MigrationDryRunService
--- a/backend/src/core/migration/risk_assessor.py
+++ b/backend/src/core/migration/risk_assessor.py
@@ -1,16 +1,16 @@
 # #region RiskAssessorModule [C:5] [TYPE Module] [SEMANTICS migration, dry_run, risk, scoring, preflight]
 # @BRIEF Compute deterministic migration risk items and aggregate score for dry-run reporting.
-# @LAYER Domain
-# @PRE Risk assessor functions receive normalized migration object collections from dry-run orchestration.
-# @POST Risk scoring output preserves item list and provides bounded score with derived level.
-# @SIDE_EFFECT Emits diagnostic logs and performs read-only metadata requests via Superset client.
-# @DATA_CONTRACT Module[build_risks, score_risks]
-# @INVARIANT Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
+# @LAYER: Domain
 # @RELATION DEPENDS_ON -> [SupersetClient]
 # @RELATION CONTAINS -> [index_by_uuid]
 # @RELATION CONTAINS -> [extract_owner_identifiers]
 # @RELATION CONTAINS -> [build_risks]
 # @RELATION CONTAINS -> [score_risks]
+# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
+# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.
+# @POST: Risk scoring output preserves item list and provides bounded score with derived level.
+# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.
+# @DATA_CONTRACT: Module[build_risks, score_risks]
 # @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
 # @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]
 # @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
@@ -26,28 +26,25 @@

 from typing import Any, Dict, List

-from ..logger import belief_scope
-from ..cot_logger import MarkerLogger
+from ..logger import logger, belief_scope
 from ..superset_client import SupersetClient

-log = MarkerLogger("RiskAssessor")
-

 # #region index_by_uuid [TYPE Function]
 # @BRIEF Build UUID-index from normalized objects.
-# @PRE Input list items are dict-like payloads potentially containing "uuid".
-# @POST Returns mapping keyed by string uuid; only truthy uuid values are included.
-# @SIDE_EFFECT Emits reasoning/reflective logs only.
-# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]
+# @PRE: Input list items are dict-like payloads potentially containing "uuid".
+# @POST: Returns mapping keyed by string uuid; only truthy uuid values are included.
+# @SIDE_EFFECT: Emits reasoning/reflective logs only.
+# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Dict[str, Any]]
 def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
    with belief_scope("risk_assessor.index_by_uuid"):
-        log.reason("Building UUID index", payload={"objects_count": len(objects)})
+        logger.reason("Building UUID index", extra={"objects_count": len(objects)})
        indexed: Dict[str, Dict[str, Any]] = {}
        for obj in objects:
            uuid = obj.get("uuid")
            if uuid:
                indexed[str(uuid)] = obj
-        log.reflect("UUID index built", payload={"indexed_count": len(indexed)})
+        logger.reflect("UUID index built", extra={"indexed_count": len(indexed)})
        return indexed


@@ -56,15 +53,15 @@ def index_by_uuid(objects: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:

 # #region extract_owner_identifiers [TYPE Function]
 # @BRIEF Normalize owner payloads for stable comparison.
-# @PRE Owners may be list payload, scalar values, or None.
-# @POST Returns sorted unique owner identifiers as strings.
-# @SIDE_EFFECT Emits reasoning/reflective logs only.
-# @DATA_CONTRACT Any -> List[str]
+# @PRE: Owners may be list payload, scalar values, or None.
+# @POST: Returns sorted unique owner identifiers as strings.
+# @SIDE_EFFECT: Emits reasoning/reflective logs only.
+# @DATA_CONTRACT: Any -> List[str]
 def extract_owner_identifiers(owners: Any) -> List[str]:
    with belief_scope("risk_assessor.extract_owner_identifiers"):
-        log.reason("Normalizing owner identifiers")
+        logger.reason("Normalizing owner identifiers")
        if not isinstance(owners, list):
-            log.reflect("Owners payload is not list; returning empty identifiers")
+            logger.reflect("Owners payload is not list; returning empty identifiers")
            return []
        ids: List[str] = []
        for owner in owners:
@@ -76,8 +73,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]:
            elif owner is not None:
                ids.append(str(owner))
        normalized_ids = sorted(set(ids))
-        log.reflect(
-            "Owner identifiers normalized", payload={"owner_count": len(normalized_ids)}
+        logger.reflect(
+            "Owner identifiers normalized", extra={"owner_count": len(normalized_ids)}
        )
        return normalized_ids

@@ -87,18 +84,18 @@ def extract_owner_identifiers(owners: Any) -> List[str]:

 # #region build_risks [TYPE Function]
 # @BRIEF Build risk list from computed diffs and target catalog state.
-# @PRE source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
-# @PRE target_client is authenticated/usable for database list retrieval.
-# @POST Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
-# @SIDE_EFFECT Calls target Superset API for databases metadata and emits logs.
-# @DATA_CONTRACT (
-# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]],
-# @DATA_CONTRACT Dict[str, List[Dict[str, Any]]],
-# @DATA_CONTRACT Dict[str, Dict[str, List[Dict[str, Any]]]],
-# @DATA_CONTRACT SupersetClient
-# @DATA_CONTRACT ) -> List[Dict[str, Any]]
 # @RELATION DEPENDS_ON -> [index_by_uuid]
 # @RELATION DEPENDS_ON -> [extract_owner_identifiers]
+# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
+# @PRE: target_client is authenticated/usable for database list retrieval.
+# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
+# @SIDE_EFFECT: Calls target Superset API for databases metadata and emits logs.
+# @DATA_CONTRACT: (
+# @DATA_CONTRACT:   Dict[str, List[Dict[str, Any]]],
+# @DATA_CONTRACT:   Dict[str, List[Dict[str, Any]]],
+# @DATA_CONTRACT:   Dict[str, Dict[str, List[Dict[str, Any]]]],
+# @DATA_CONTRACT:   SupersetClient
+# @DATA_CONTRACT: ) -> List[Dict[str, Any]]
 def build_risks(
    source_objects: Dict[str, List[Dict[str, Any]]],
    target_objects: Dict[str, List[Dict[str, Any]]],
@@ -106,7 +103,7 @@ def build_risks(
    target_client: SupersetClient,
 ) -> List[Dict[str, Any]]:
    with belief_scope("risk_assessor.build_risks"):
-        log.reason("Building migration risks from diff payload")
+        logger.reason("Building migration risks from diff payload")
        risks: List[Dict[str, Any]] = []
        for object_type in ("dashboards", "charts", "datasets"):
            for item in diff[object_type]["update"]:
@@ -171,7 +168,7 @@ def build_risks(
                        "message": f"Owner mismatch for dashboard {item.get('title') or item['uuid']}",
                    }
                )
-        log.reflect("Risk list assembled", payload={"risk_count": len(risks)})
+        logger.reflect("Risk list assembled", extra={"risk_count": len(risks)})
        return risks


@@ -180,20 +177,20 @@ def build_risks(

 # #region score_risks [TYPE Function]
 # @BRIEF Aggregate risk list into score and level.
-# @PRE risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.
-# @POST Returns dict with score in [0,100], derived level, and original items.
-# @SIDE_EFFECT Emits reasoning/reflective logs only.
-# @DATA_CONTRACT List[Dict[str, Any]] -> Dict[str, Any]
+# @PRE: risk_items contains optional severity fields expected in {high,medium,low} or defaults to low weight.
+# @POST: Returns dict with score in [0,100], derived level, and original items.
+# @SIDE_EFFECT: Emits reasoning/reflective logs only.
+# @DATA_CONTRACT: List[Dict[str, Any]] -> Dict[str, Any]
 def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:
    with belief_scope("risk_assessor.score_risks"):
-        log.reason("Scoring risk items", payload={"risk_items_count": len(risk_items)})
+        logger.reason("Scoring risk items", extra={"risk_items_count": len(risk_items)})
        weights = {"high": 25, "medium": 10, "low": 5}
        score = min(
            100, sum(weights.get(item.get("severity", "low"), 5) for item in risk_items)
        )
        level = "low" if score < 25 else "medium" if score < 60 else "high"
        result = {"score": score, "level": level, "items": risk_items}
-        log.reflect("Risk score computed", payload={"score": score, "level": level})
+        logger.reflect("Risk score computed", extra={"score": score, "level": level})
        return result