# #region SemanticSourceResolver [C:4] [TYPE Module] [SEMANTICS pydantic, dataset, semantic, resolver, mapping] # @BRIEF Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback. # @LAYER Domain # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [SemanticSource] # @RELATION DEPENDS_ON -> [SemanticFieldEntry] # @RELATION DEPENDS_ON -> [SemanticCandidate] # @PRE selected source and target field set must be known. # @POST candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable. # @SIDE_EFFECT may create conflict findings and semantic candidate records. # @INVARIANT Manual overrides are never silently replaced by imported, inferred, or AI-generated values. from __future__ import annotations from collections.abc import Iterable, Mapping # #region imports [TYPE Block] from dataclasses import dataclass, field from difflib import SequenceMatcher from typing import Any from src.core.logger import belief_scope, logger from src.models.dataset_review import ( CandidateMatchType, CandidateStatus, FieldProvenance, SemanticSource, ) # #endregion imports # #region DictionaryResolutionResult [C:2] [TYPE Class] # @BRIEF Carries field-level dictionary resolution output with explicit review and partial-recovery state. @dataclass class DictionaryResolutionResult: source_ref: str resolved_fields: list[dict[str, Any]] = field(default_factory=list) unresolved_fields: list[str] = field(default_factory=list) partial_recovery: bool = False # #endregion DictionaryResolutionResult # #region SemanticSourceResolver [C:4] [TYPE Class] # @BRIEF Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering. # @RELATION DEPENDS_ON -> [SemanticFieldEntry] # @RELATION DEPENDS_ON -> [SemanticCandidate] # @PRE source payload and target field collection are provided by the caller. # @POST result contains confidence-ranked candidates and does not overwrite manual locks implicitly. # @SIDE_EFFECT emits semantic trace logs for ranking and fallback decisions. class SemanticSourceResolver: # region resolve_from_file [TYPE Function] # @PURPOSE: Normalize uploaded semantic file records into field-level candidates. def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult: return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "uploaded_file")) # endregion resolve_from_file # region resolve_from_dictionary [TYPE Function] # @PURPOSE: Resolve candidates from connected tabular dictionary sources. # @RELATION DEPENDS_ON ->[SemanticFieldEntry] # @RELATION DEPENDS_ON ->[SemanticCandidate] # @PRE dictionary source exists and fields contain stable field_name values. # @POST returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit. # @SIDE_EFFECT emits belief-state logs describing trusted-match and partial-recovery outcomes. # @DATA_CONTRACT Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult] def resolve_from_dictionary( self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]], ) -> DictionaryResolutionResult: with belief_scope("SemanticSourceResolver.resolve_from_dictionary"): source_ref = str(source_payload.get("source_ref") or "").strip() dictionary_rows = source_payload.get("rows") if not source_ref: logger.explore("Dictionary semantic source is missing source_ref") raise ValueError("Dictionary semantic source must include source_ref") if not isinstance(dictionary_rows, list) or not dictionary_rows: logger.explore( "Dictionary semantic source has no usable rows", extra={"source_ref": source_ref}, ) raise ValueError("Dictionary semantic source must include non-empty rows") logger.reason( "Resolving semantics from trusted dictionary source", extra={"source_ref": source_ref, "row_count": len(dictionary_rows)}, ) normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)] row_index = { row["field_key"]: row for row in normalized_rows if row.get("field_key") } resolved_fields: list[dict[str, Any]] = [] unresolved_fields: list[str] = [] for raw_field in fields: field_name = str(raw_field.get("field_name") or "").strip() if not field_name: continue is_locked = bool(raw_field.get("is_locked")) if is_locked: logger.reason( "Preserving manual lock during dictionary resolution", extra={"field_name": field_name}, ) resolved_fields.append( { "field_name": field_name, "applied_candidate": None, "candidates": [], "provenance": FieldProvenance.MANUAL_OVERRIDE.value, "needs_review": False, "has_conflict": False, "is_locked": True, "status": "preserved_manual", } ) continue exact_match = row_index.get(self._normalize_key(field_name)) candidates: list[dict[str, Any]] = [] if exact_match is not None: logger.reason( "Resolved exact dictionary match", extra={"field_name": field_name, "source_ref": source_ref}, ) candidates.append( self._build_candidate_payload( rank=1, match_type=CandidateMatchType.EXACT, confidence_score=1.0, row=exact_match, ) ) else: fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows) for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1): candidates.append( self._build_candidate_payload( rank=rank_offset, match_type=CandidateMatchType.FUZZY, confidence_score=float(fuzzy_match["score"]), row=fuzzy_match["row"], ) ) if not candidates: unresolved_fields.append(field_name) resolved_fields.append( { "field_name": field_name, "applied_candidate": None, "candidates": [], "provenance": FieldProvenance.UNRESOLVED.value, "needs_review": True, "has_conflict": False, "is_locked": False, "status": "unresolved", } ) logger.explore( "No trusted dictionary match found for field", extra={"field_name": field_name, "source_ref": source_ref}, ) continue ranked_candidates = self.rank_candidates(candidates) applied_candidate = ranked_candidates[0] has_conflict = len(ranked_candidates) > 1 provenance = ( FieldProvenance.DICTIONARY_EXACT.value if applied_candidate["match_type"] == CandidateMatchType.EXACT.value else FieldProvenance.FUZZY_INFERRED.value ) needs_review = applied_candidate["match_type"] != CandidateMatchType.EXACT.value resolved_fields.append( { "field_name": field_name, "applied_candidate": applied_candidate, "candidates": ranked_candidates, "provenance": provenance, "needs_review": needs_review, "has_conflict": has_conflict, "is_locked": False, "status": "resolved", } ) result = DictionaryResolutionResult( source_ref=source_ref, resolved_fields=resolved_fields, unresolved_fields=unresolved_fields, partial_recovery=bool(unresolved_fields), ) logger.reflect( "Dictionary resolution completed", extra={ "source_ref": source_ref, "resolved_fields": len(resolved_fields), "unresolved_fields": len(unresolved_fields), "partial_recovery": result.partial_recovery, }, ) return result # endregion resolve_from_dictionary # region resolve_from_reference_dataset [TYPE Function] # @PURPOSE: Reuse semantic metadata from trusted Superset datasets. def resolve_from_reference_dataset( self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]], ) -> DictionaryResolutionResult: return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "reference_dataset")) # endregion resolve_from_reference_dataset # region rank_candidates [TYPE Function] # @PURPOSE: Apply confidence ordering and determine best candidate per field. # @RELATION DEPENDS_ON ->[SemanticCandidate] def rank_candidates(self, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: ranked = sorted( candidates, key=lambda candidate: ( self._match_priority(candidate.get("match_type")), -float(candidate.get("confidence_score", 0.0)), int(candidate.get("candidate_rank", 999)), ), ) for index, candidate in enumerate(ranked, start=1): candidate["candidate_rank"] = index return ranked # endregion rank_candidates # region detect_conflicts [TYPE Function] # @PURPOSE: Mark competing candidate sets that require explicit user review. def detect_conflicts(self, candidates: list[dict[str, Any]]) -> bool: return len(candidates) > 1 # endregion detect_conflicts # region apply_field_decision [TYPE Function] # @PURPOSE: Accept, reject, or manually override a field-level semantic value. def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> dict[str, Any]: merged = dict(field_state) merged.update(decision) return merged # endregion apply_field_decision # region propagate_source_version_update [TYPE Function] # @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values. # @RELATION DEPENDS_ON ->[SemanticSource] # @RELATION DEPENDS_ON ->[SemanticFieldEntry] # @PRE source is persisted and fields belong to the same session aggregate. # @POST unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched. # @SIDE_EFFECT mutates in-memory field state for the caller to persist. # @DATA_CONTRACT Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]] def propagate_source_version_update( self, source: SemanticSource, fields: Iterable[Any], ) -> dict[str, int]: with belief_scope("SemanticSourceResolver.propagate_source_version_update"): source_id = str(source.source_id or "").strip() source_version = str(source.source_version or "").strip() if not source_id or not source_version: logger.explore( "Semantic source version propagation rejected due to incomplete source metadata", extra={"source_id": source_id, "source_version": source_version}, ) raise ValueError("Semantic source must provide source_id and source_version") propagated = 0 preserved_locked = 0 untouched = 0 for field in fields: if str(getattr(field, "source_id", "") or "").strip() != source_id: untouched += 1 continue if bool(getattr(field, "is_locked", False)) or getattr(field, "provenance", None) == FieldProvenance.MANUAL_OVERRIDE: preserved_locked += 1 continue field.source_version = source_version field.needs_review = True field.has_conflict = bool(getattr(field, "has_conflict", False)) propagated += 1 logger.reflect( "Semantic source version propagation completed", extra={ "source_id": source_id, "source_version": source_version, "propagated": propagated, "preserved_locked": preserved_locked, "untouched": untouched, }, ) return { "propagated": propagated, "preserved_locked": preserved_locked, "untouched": untouched, } # endregion propagate_source_version_update # region _normalize_dictionary_row [TYPE Function] # @PURPOSE: Normalize one dictionary row into a consistent lookup structure. def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> dict[str, Any]: field_name = ( row.get("field_name") or row.get("column_name") or row.get("name") or row.get("field") ) normalized_name = str(field_name or "").strip() return { "field_name": normalized_name, "field_key": self._normalize_key(normalized_name), "verbose_name": row.get("verbose_name") or row.get("label"), "description": row.get("description"), "display_format": row.get("display_format") or row.get("format"), } # endregion _normalize_dictionary_row # region _find_fuzzy_matches [TYPE Function] # @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable. def _find_fuzzy_matches(self, field_name: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: normalized_target = self._normalize_key(field_name) fuzzy_matches: list[dict[str, Any]] = [] for row in rows: candidate_key = str(row.get("field_key") or "") if not candidate_key: continue score = SequenceMatcher(None, normalized_target, candidate_key).ratio() if score < 0.72: continue fuzzy_matches.append({"row": row, "score": round(score, 3)}) fuzzy_matches.sort(key=lambda item: item["score"], reverse=True) return fuzzy_matches[:3] # endregion _find_fuzzy_matches # region _build_candidate_payload [TYPE Function] # @PURPOSE: Project normalized dictionary rows into semantic candidate payloads. def _build_candidate_payload( self, rank: int, match_type: CandidateMatchType, confidence_score: float, row: Mapping[str, Any], ) -> dict[str, Any]: return { "candidate_rank": rank, "match_type": match_type.value, "confidence_score": confidence_score, "proposed_verbose_name": row.get("verbose_name"), "proposed_description": row.get("description"), "proposed_display_format": row.get("display_format"), "status": CandidateStatus.PROPOSED.value, } # endregion _build_candidate_payload # region _match_priority [TYPE Function] # @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention. def _match_priority(self, match_type: str | None) -> int: priority = { CandidateMatchType.EXACT.value: 0, CandidateMatchType.REFERENCE.value: 1, CandidateMatchType.FUZZY.value: 2, CandidateMatchType.GENERATED.value: 3, } return priority.get(str(match_type or ""), 99) # endregion _match_priority # region _normalize_key [TYPE Function] # @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons. def _normalize_key(self, value: str) -> str: return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch == "_") # endregion _normalize_key # #endregion SemanticSourceResolver # #endregion SemanticSourceResolver