388 lines
17 KiB
Python
388 lines
17 KiB
Python
# #region SemanticSourceResolver [C:4] [TYPE Module] [SEMANTICS pydantic, dataset, semantic, resolver, mapping]
|
|
# @BRIEF Resolve and rank semantic candidates from trusted dictionary-like sources before any inferred fallback.
|
|
# @LAYER Domain
|
|
# @RELATION DEPENDS_ON -> [LLMProviderService]
|
|
# @RELATION DEPENDS_ON -> [SemanticSource]
|
|
# @RELATION DEPENDS_ON -> [SemanticFieldEntry]
|
|
# @RELATION DEPENDS_ON -> [SemanticCandidate]
|
|
# @PRE selected source and target field set must be known.
|
|
# @POST candidate ranking follows the configured confidence hierarchy and unresolved fuzzy matches remain reviewable.
|
|
# @SIDE_EFFECT may create conflict findings and semantic candidate records.
|
|
# @INVARIANT Manual overrides are never silently replaced by imported, inferred, or AI-generated values.
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Iterable, Mapping
|
|
|
|
# #region imports [TYPE Block]
|
|
from dataclasses import dataclass, field
|
|
from difflib import SequenceMatcher
|
|
from typing import Any
|
|
|
|
from src.core.logger import belief_scope, logger
|
|
from src.models.dataset_review import (
|
|
CandidateMatchType,
|
|
CandidateStatus,
|
|
FieldProvenance,
|
|
SemanticSource,
|
|
)
|
|
|
|
# #endregion imports
|
|
|
|
|
|
# #region DictionaryResolutionResult [C:2] [TYPE Class]
|
|
# @BRIEF Carries field-level dictionary resolution output with explicit review and partial-recovery state.
|
|
@dataclass
|
|
class DictionaryResolutionResult:
|
|
source_ref: str
|
|
resolved_fields: list[dict[str, Any]] = field(default_factory=list)
|
|
unresolved_fields: list[str] = field(default_factory=list)
|
|
partial_recovery: bool = False
|
|
# #endregion DictionaryResolutionResult
|
|
|
|
|
|
# #region SemanticSourceResolver [C:4] [TYPE Class]
|
|
# @BRIEF Resolve semantic candidates from trusted sources while preserving manual locks and confidence ordering.
|
|
# @RELATION DEPENDS_ON -> [SemanticFieldEntry]
|
|
# @RELATION DEPENDS_ON -> [SemanticCandidate]
|
|
# @PRE source payload and target field collection are provided by the caller.
|
|
# @POST result contains confidence-ranked candidates and does not overwrite manual locks implicitly.
|
|
# @SIDE_EFFECT emits semantic trace logs for ranking and fallback decisions.
|
|
class SemanticSourceResolver:
|
|
# region resolve_from_file [TYPE Function]
|
|
# @PURPOSE: Normalize uploaded semantic file records into field-level candidates.
|
|
def resolve_from_file(self, source_payload: Mapping[str, Any], fields: Iterable[Mapping[str, Any]]) -> DictionaryResolutionResult:
|
|
return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "uploaded_file"))
|
|
# endregion resolve_from_file
|
|
|
|
# region resolve_from_dictionary [TYPE Function]
|
|
# @PURPOSE: Resolve candidates from connected tabular dictionary sources.
|
|
# @RELATION DEPENDS_ON ->[SemanticFieldEntry]
|
|
# @RELATION DEPENDS_ON ->[SemanticCandidate]
|
|
# @PRE dictionary source exists and fields contain stable field_name values.
|
|
# @POST returns confidence-ranked candidates where exact dictionary matches outrank fuzzy matches and unresolved fields stay explicit.
|
|
# @SIDE_EFFECT emits belief-state logs describing trusted-match and partial-recovery outcomes.
|
|
# @DATA_CONTRACT Input[source_payload:Mapping,fields:Iterable] -> Output[DictionaryResolutionResult]
|
|
def resolve_from_dictionary(
|
|
self,
|
|
source_payload: Mapping[str, Any],
|
|
fields: Iterable[Mapping[str, Any]],
|
|
) -> DictionaryResolutionResult:
|
|
with belief_scope("SemanticSourceResolver.resolve_from_dictionary"):
|
|
source_ref = str(source_payload.get("source_ref") or "").strip()
|
|
dictionary_rows = source_payload.get("rows")
|
|
|
|
if not source_ref:
|
|
logger.explore("Dictionary semantic source is missing source_ref")
|
|
raise ValueError("Dictionary semantic source must include source_ref")
|
|
|
|
if not isinstance(dictionary_rows, list) or not dictionary_rows:
|
|
logger.explore(
|
|
"Dictionary semantic source has no usable rows",
|
|
extra={"source_ref": source_ref},
|
|
)
|
|
raise ValueError("Dictionary semantic source must include non-empty rows")
|
|
|
|
logger.reason(
|
|
"Resolving semantics from trusted dictionary source",
|
|
extra={"source_ref": source_ref, "row_count": len(dictionary_rows)},
|
|
)
|
|
|
|
normalized_rows = [self._normalize_dictionary_row(row) for row in dictionary_rows if isinstance(row, Mapping)]
|
|
row_index = {
|
|
row["field_key"]: row
|
|
for row in normalized_rows
|
|
if row.get("field_key")
|
|
}
|
|
|
|
resolved_fields: list[dict[str, Any]] = []
|
|
unresolved_fields: list[str] = []
|
|
|
|
for raw_field in fields:
|
|
field_name = str(raw_field.get("field_name") or "").strip()
|
|
if not field_name:
|
|
continue
|
|
|
|
is_locked = bool(raw_field.get("is_locked"))
|
|
if is_locked:
|
|
logger.reason(
|
|
"Preserving manual lock during dictionary resolution",
|
|
extra={"field_name": field_name},
|
|
)
|
|
resolved_fields.append(
|
|
{
|
|
"field_name": field_name,
|
|
"applied_candidate": None,
|
|
"candidates": [],
|
|
"provenance": FieldProvenance.MANUAL_OVERRIDE.value,
|
|
"needs_review": False,
|
|
"has_conflict": False,
|
|
"is_locked": True,
|
|
"status": "preserved_manual",
|
|
}
|
|
)
|
|
continue
|
|
|
|
exact_match = row_index.get(self._normalize_key(field_name))
|
|
candidates: list[dict[str, Any]] = []
|
|
|
|
if exact_match is not None:
|
|
logger.reason(
|
|
"Resolved exact dictionary match",
|
|
extra={"field_name": field_name, "source_ref": source_ref},
|
|
)
|
|
candidates.append(
|
|
self._build_candidate_payload(
|
|
rank=1,
|
|
match_type=CandidateMatchType.EXACT,
|
|
confidence_score=1.0,
|
|
row=exact_match,
|
|
)
|
|
)
|
|
else:
|
|
fuzzy_matches = self._find_fuzzy_matches(field_name, normalized_rows)
|
|
for rank_offset, fuzzy_match in enumerate(fuzzy_matches, start=1):
|
|
candidates.append(
|
|
self._build_candidate_payload(
|
|
rank=rank_offset,
|
|
match_type=CandidateMatchType.FUZZY,
|
|
confidence_score=float(fuzzy_match["score"]),
|
|
row=fuzzy_match["row"],
|
|
)
|
|
)
|
|
|
|
if not candidates:
|
|
unresolved_fields.append(field_name)
|
|
resolved_fields.append(
|
|
{
|
|
"field_name": field_name,
|
|
"applied_candidate": None,
|
|
"candidates": [],
|
|
"provenance": FieldProvenance.UNRESOLVED.value,
|
|
"needs_review": True,
|
|
"has_conflict": False,
|
|
"is_locked": False,
|
|
"status": "unresolved",
|
|
}
|
|
)
|
|
logger.explore(
|
|
"No trusted dictionary match found for field",
|
|
extra={"field_name": field_name, "source_ref": source_ref},
|
|
)
|
|
continue
|
|
|
|
ranked_candidates = self.rank_candidates(candidates)
|
|
applied_candidate = ranked_candidates[0]
|
|
has_conflict = len(ranked_candidates) > 1
|
|
provenance = (
|
|
FieldProvenance.DICTIONARY_EXACT.value
|
|
if applied_candidate["match_type"] == CandidateMatchType.EXACT.value
|
|
else FieldProvenance.FUZZY_INFERRED.value
|
|
)
|
|
needs_review = applied_candidate["match_type"] != CandidateMatchType.EXACT.value
|
|
|
|
resolved_fields.append(
|
|
{
|
|
"field_name": field_name,
|
|
"applied_candidate": applied_candidate,
|
|
"candidates": ranked_candidates,
|
|
"provenance": provenance,
|
|
"needs_review": needs_review,
|
|
"has_conflict": has_conflict,
|
|
"is_locked": False,
|
|
"status": "resolved",
|
|
}
|
|
)
|
|
|
|
result = DictionaryResolutionResult(
|
|
source_ref=source_ref,
|
|
resolved_fields=resolved_fields,
|
|
unresolved_fields=unresolved_fields,
|
|
partial_recovery=bool(unresolved_fields),
|
|
)
|
|
logger.reflect(
|
|
"Dictionary resolution completed",
|
|
extra={
|
|
"source_ref": source_ref,
|
|
"resolved_fields": len(resolved_fields),
|
|
"unresolved_fields": len(unresolved_fields),
|
|
"partial_recovery": result.partial_recovery,
|
|
},
|
|
)
|
|
return result
|
|
# endregion resolve_from_dictionary
|
|
|
|
# region resolve_from_reference_dataset [TYPE Function]
|
|
# @PURPOSE: Reuse semantic metadata from trusted Superset datasets.
|
|
def resolve_from_reference_dataset(
|
|
self,
|
|
source_payload: Mapping[str, Any],
|
|
fields: Iterable[Mapping[str, Any]],
|
|
) -> DictionaryResolutionResult:
|
|
return DictionaryResolutionResult(source_ref=str(source_payload.get("source_ref") or "reference_dataset"))
|
|
# endregion resolve_from_reference_dataset
|
|
|
|
# region rank_candidates [TYPE Function]
|
|
# @PURPOSE: Apply confidence ordering and determine best candidate per field.
|
|
# @RELATION DEPENDS_ON ->[SemanticCandidate]
|
|
def rank_candidates(self, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
ranked = sorted(
|
|
candidates,
|
|
key=lambda candidate: (
|
|
self._match_priority(candidate.get("match_type")),
|
|
-float(candidate.get("confidence_score", 0.0)),
|
|
int(candidate.get("candidate_rank", 999)),
|
|
),
|
|
)
|
|
for index, candidate in enumerate(ranked, start=1):
|
|
candidate["candidate_rank"] = index
|
|
return ranked
|
|
# endregion rank_candidates
|
|
|
|
# region detect_conflicts [TYPE Function]
|
|
# @PURPOSE: Mark competing candidate sets that require explicit user review.
|
|
def detect_conflicts(self, candidates: list[dict[str, Any]]) -> bool:
|
|
return len(candidates) > 1
|
|
# endregion detect_conflicts
|
|
|
|
# region apply_field_decision [TYPE Function]
|
|
# @PURPOSE: Accept, reject, or manually override a field-level semantic value.
|
|
def apply_field_decision(self, field_state: Mapping[str, Any], decision: Mapping[str, Any]) -> dict[str, Any]:
|
|
merged = dict(field_state)
|
|
merged.update(decision)
|
|
return merged
|
|
# endregion apply_field_decision
|
|
|
|
# region propagate_source_version_update [TYPE Function]
|
|
# @PURPOSE: Propagate a semantic source version change to unlocked field entries without silently overwriting manual or locked values.
|
|
# @RELATION DEPENDS_ON ->[SemanticSource]
|
|
# @RELATION DEPENDS_ON ->[SemanticFieldEntry]
|
|
# @PRE source is persisted and fields belong to the same session aggregate.
|
|
# @POST unlocked fields linked to the source carry the new source version and are marked reviewable; manual or locked fields keep their active values untouched.
|
|
# @SIDE_EFFECT mutates in-memory field state for the caller to persist.
|
|
# @DATA_CONTRACT Input[SemanticSource,List[SemanticFieldEntry]] -> Output[Dict[str,int]]
|
|
def propagate_source_version_update(
|
|
self,
|
|
source: SemanticSource,
|
|
fields: Iterable[Any],
|
|
) -> dict[str, int]:
|
|
with belief_scope("SemanticSourceResolver.propagate_source_version_update"):
|
|
source_id = str(source.source_id or "").strip()
|
|
source_version = str(source.source_version or "").strip()
|
|
if not source_id or not source_version:
|
|
logger.explore(
|
|
"Semantic source version propagation rejected due to incomplete source metadata",
|
|
extra={"source_id": source_id, "source_version": source_version},
|
|
)
|
|
raise ValueError("Semantic source must provide source_id and source_version")
|
|
|
|
propagated = 0
|
|
preserved_locked = 0
|
|
untouched = 0
|
|
for field in fields:
|
|
if str(getattr(field, "source_id", "") or "").strip() != source_id:
|
|
untouched += 1
|
|
continue
|
|
if bool(getattr(field, "is_locked", False)) or getattr(field, "provenance", None) == FieldProvenance.MANUAL_OVERRIDE:
|
|
preserved_locked += 1
|
|
continue
|
|
|
|
field.source_version = source_version
|
|
field.needs_review = True
|
|
field.has_conflict = bool(getattr(field, "has_conflict", False))
|
|
propagated += 1
|
|
|
|
logger.reflect(
|
|
"Semantic source version propagation completed",
|
|
extra={
|
|
"source_id": source_id,
|
|
"source_version": source_version,
|
|
"propagated": propagated,
|
|
"preserved_locked": preserved_locked,
|
|
"untouched": untouched,
|
|
},
|
|
)
|
|
return {
|
|
"propagated": propagated,
|
|
"preserved_locked": preserved_locked,
|
|
"untouched": untouched,
|
|
}
|
|
# endregion propagate_source_version_update
|
|
|
|
# region _normalize_dictionary_row [TYPE Function]
|
|
# @PURPOSE: Normalize one dictionary row into a consistent lookup structure.
|
|
def _normalize_dictionary_row(self, row: Mapping[str, Any]) -> dict[str, Any]:
|
|
field_name = (
|
|
row.get("field_name")
|
|
or row.get("column_name")
|
|
or row.get("name")
|
|
or row.get("field")
|
|
)
|
|
normalized_name = str(field_name or "").strip()
|
|
return {
|
|
"field_name": normalized_name,
|
|
"field_key": self._normalize_key(normalized_name),
|
|
"verbose_name": row.get("verbose_name") or row.get("label"),
|
|
"description": row.get("description"),
|
|
"display_format": row.get("display_format") or row.get("format"),
|
|
}
|
|
# endregion _normalize_dictionary_row
|
|
|
|
# region _find_fuzzy_matches [TYPE Function]
|
|
# @PURPOSE: Produce confidence-scored fuzzy matches while keeping them reviewable.
|
|
def _find_fuzzy_matches(self, field_name: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
normalized_target = self._normalize_key(field_name)
|
|
fuzzy_matches: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
candidate_key = str(row.get("field_key") or "")
|
|
if not candidate_key:
|
|
continue
|
|
score = SequenceMatcher(None, normalized_target, candidate_key).ratio()
|
|
if score < 0.72:
|
|
continue
|
|
fuzzy_matches.append({"row": row, "score": round(score, 3)})
|
|
fuzzy_matches.sort(key=lambda item: item["score"], reverse=True)
|
|
return fuzzy_matches[:3]
|
|
# endregion _find_fuzzy_matches
|
|
|
|
# region _build_candidate_payload [TYPE Function]
|
|
# @PURPOSE: Project normalized dictionary rows into semantic candidate payloads.
|
|
def _build_candidate_payload(
|
|
self,
|
|
rank: int,
|
|
match_type: CandidateMatchType,
|
|
confidence_score: float,
|
|
row: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"candidate_rank": rank,
|
|
"match_type": match_type.value,
|
|
"confidence_score": confidence_score,
|
|
"proposed_verbose_name": row.get("verbose_name"),
|
|
"proposed_description": row.get("description"),
|
|
"proposed_display_format": row.get("display_format"),
|
|
"status": CandidateStatus.PROPOSED.value,
|
|
}
|
|
# endregion _build_candidate_payload
|
|
|
|
# region _match_priority [TYPE Function]
|
|
# @PURPOSE: Encode trusted-confidence ordering so exact dictionary reuse beats fuzzy invention.
|
|
def _match_priority(self, match_type: str | None) -> int:
|
|
priority = {
|
|
CandidateMatchType.EXACT.value: 0,
|
|
CandidateMatchType.REFERENCE.value: 1,
|
|
CandidateMatchType.FUZZY.value: 2,
|
|
CandidateMatchType.GENERATED.value: 3,
|
|
}
|
|
return priority.get(str(match_type or ""), 99)
|
|
# endregion _match_priority
|
|
|
|
# region _normalize_key [TYPE Function]
|
|
# @PURPOSE: Normalize field identifiers for stable exact/fuzzy comparisons.
|
|
def _normalize_key(self, value: str) -> str:
|
|
return "".join(ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch == "_")
|
|
# endregion _normalize_key
|
|
# #endregion SemanticSourceResolver
|
|
|
|
# #endregion SemanticSourceResolver
|