380 lines
16 KiB
Python
380 lines
16 KiB
Python
# #region Test.SemanticSourceResolver [C:2] [TYPE Module] [SEMANTICS test,dataset,semantic,resolver,mapping]
|
|
# @BRIEF Tests for semantic_resolver.py — semantic candidate resolution, ranking, and conflict detection.
|
|
# @RELATION BINDS_TO -> [SemanticSourceResolver]
|
|
# @TEST_EDGE: resolve_from_file -> Returns basic resolution result
|
|
# @TEST_EDGE: resolve_from_dictionary -> Exact, fuzzy, and unresolved field matching
|
|
# @TEST_EDGE: resolve_from_dictionary_validation -> Missing source_ref or rows raises ValueError
|
|
# @TEST_EDGE: resolve_from_reference_dataset -> Returns basic result
|
|
# @TEST_EDGE: rank_candidates -> Confidence ordering
|
|
# @TEST_EDGE: detect_conflicts -> Multi-candidate detection
|
|
# @TEST_EDGE: apply_field_decision -> Field merging
|
|
# @TEST_EDGE: propagate_source_version_update -> Version propagation with locks
|
|
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
import pytest
|
|
|
|
|
|
class TestSemanticSourceResolverResolveFromFile:
|
|
"""SemanticSourceResolver.resolve_from_file — basic file resolution."""
|
|
|
|
def test_resolve_from_file(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
result = resolver.resolve_from_file(
|
|
{"source_ref": "upload.csv"},
|
|
[{"field_name": "region"}],
|
|
)
|
|
# Source returns the source_ref value from the payload, defaulting to "uploaded_file"
|
|
assert result.source_ref == "upload.csv"
|
|
|
|
|
|
class TestSemanticSourceResolverResolveFromDictionary:
|
|
"""SemanticSourceResolver.resolve_from_dictionary — dictionary resolution."""
|
|
|
|
def test_exact_match(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [
|
|
{"column_name": "region", "verbose_name": "Region", "description": "Geographic region"},
|
|
{"column_name": "product", "verbose_name": "Product", "description": "Product name"},
|
|
],
|
|
}
|
|
fields = [{"field_name": "region"}]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 1
|
|
assert result.resolved_fields[0]["field_name"] == "region"
|
|
assert result.resolved_fields[0]["provenance"] == "dictionary_exact"
|
|
assert result.resolved_fields[0]["needs_review"] is False
|
|
|
|
def test_fuzzy_match(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [
|
|
{"column_name": "region_name", "verbose_name": "Region Name", "description": "Region name"},
|
|
],
|
|
}
|
|
fields = [{"field_name": "region"}]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 1
|
|
# region vs region_name has SequenceMatcher ratio > 0.72 -> fuzzy match
|
|
assert result.resolved_fields[0]["provenance"] == "fuzzy_inferred"
|
|
assert result.resolved_fields[0]["needs_review"] is True
|
|
|
|
def test_unresolved_field(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [
|
|
{"column_name": "product", "verbose_name": "Product"},
|
|
],
|
|
}
|
|
fields = [{"field_name": "zzz_nonexistent"}]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 1
|
|
assert result.resolved_fields[0]["status"] == "unresolved"
|
|
assert result.resolved_fields[0]["provenance"] == "unresolved"
|
|
assert result.unresolved_fields == ["zzz_nonexistent"]
|
|
assert result.partial_recovery is True
|
|
|
|
def test_missing_source_ref_raises_value_error(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
with pytest.raises(ValueError, match="source_ref"):
|
|
resolver.resolve_from_dictionary({"rows": [{"column_name": "x"}]}, [{"field_name": "x"}])
|
|
|
|
def test_empty_rows_raises_value_error(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
with pytest.raises(ValueError, match="rows"):
|
|
resolver.resolve_from_dictionary({"source_ref": "dict_1", "rows": []}, [{"field_name": "x"}])
|
|
|
|
def test_non_list_rows_raises_value_error(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
with pytest.raises(ValueError, match="rows"):
|
|
resolver.resolve_from_dictionary({"source_ref": "dict_1", "rows": "not_a_list"}, [{"field_name": "x"}])
|
|
|
|
def test_locked_field_preserved(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [{"column_name": "region", "verbose_name": "Region"}],
|
|
}
|
|
fields = [{"field_name": "region", "is_locked": True}]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 1
|
|
assert result.resolved_fields[0]["status"] == "preserved_manual"
|
|
assert result.resolved_fields[0]["is_locked"] is True
|
|
|
|
def test_skip_empty_field_name(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [{"column_name": "region", "verbose_name": "Region"}],
|
|
}
|
|
fields = [{"field_name": ""}, {"field_name": "region"}]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 1
|
|
assert result.resolved_fields[0]["field_name"] == "region"
|
|
|
|
def test_mixed_resolved_and_unresolved(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source_payload = {
|
|
"source_ref": "dict_1",
|
|
"rows": [{"column_name": "region", "verbose_name": "Region"}],
|
|
}
|
|
fields = [
|
|
{"field_name": "region"},
|
|
{"field_name": "product"},
|
|
{"field_name": "sales_amount"},
|
|
]
|
|
result = resolver.resolve_from_dictionary(source_payload, fields)
|
|
assert len(result.resolved_fields) == 3
|
|
region_field = next(f for f in result.resolved_fields if f["field_name"] == "region")
|
|
assert region_field["provenance"] == "dictionary_exact"
|
|
product_field = next(f for f in result.resolved_fields if f["field_name"] == "product")
|
|
assert product_field["status"] == "unresolved"
|
|
assert len(result.unresolved_fields) == 2
|
|
|
|
|
|
class TestSemanticSourceResolverResolveFromReference:
|
|
"""SemanticSourceResolver.resolve_from_reference_dataset — reference dataset."""
|
|
|
|
def test_resolve_from_reference(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
result = resolver.resolve_from_reference_dataset(
|
|
{"source_ref": "ref_1"},
|
|
[{"field_name": "region"}],
|
|
)
|
|
# Source returns the source_ref value from the payload, defaulting to "reference_dataset"
|
|
assert result.source_ref == "ref_1"
|
|
|
|
|
|
class TestSemanticSourceResolverRankCandidates:
|
|
"""SemanticSourceResolver.rank_candidates — confidence ordering."""
|
|
|
|
def test_exact_before_fuzzy(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import CandidateMatchType
|
|
|
|
resolver = SemanticSourceResolver()
|
|
candidates = [
|
|
{"match_type": CandidateMatchType.FUZZY.value, "confidence_score": 0.95, "candidate_rank": 999},
|
|
{"match_type": CandidateMatchType.EXACT.value, "confidence_score": 1.0, "candidate_rank": 999},
|
|
]
|
|
ranked = resolver.rank_candidates(candidates)
|
|
assert ranked[0]["match_type"] == CandidateMatchType.EXACT.value
|
|
assert ranked[1]["match_type"] == CandidateMatchType.FUZZY.value
|
|
|
|
def test_higher_confidence_first(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import CandidateMatchType
|
|
|
|
resolver = SemanticSourceResolver()
|
|
candidates = [
|
|
{"match_type": CandidateMatchType.FUZZY.value, "confidence_score": 0.8, "candidate_rank": 999},
|
|
{"match_type": CandidateMatchType.FUZZY.value, "confidence_score": 0.9, "candidate_rank": 999},
|
|
]
|
|
ranked = resolver.rank_candidates(candidates)
|
|
assert ranked[0]["confidence_score"] == 0.9
|
|
assert ranked[1]["confidence_score"] == 0.8
|
|
|
|
def test_ranks_assigned(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import CandidateMatchType
|
|
|
|
resolver = SemanticSourceResolver()
|
|
candidates = [
|
|
{"match_type": CandidateMatchType.EXACT.value, "confidence_score": 1.0, "candidate_rank": 999},
|
|
{"match_type": CandidateMatchType.FUZZY.value, "confidence_score": 0.9, "candidate_rank": 999},
|
|
{"match_type": CandidateMatchType.FUZZY.value, "confidence_score": 0.8, "candidate_rank": 999},
|
|
]
|
|
ranked = resolver.rank_candidates(candidates)
|
|
assert ranked[0]["candidate_rank"] == 1
|
|
assert ranked[1]["candidate_rank"] == 2
|
|
assert ranked[2]["candidate_rank"] == 3
|
|
|
|
|
|
class TestSemanticSourceResolverDetectConflicts:
|
|
"""SemanticSourceResolver.detect_conflicts — multi-candidate detection."""
|
|
|
|
def test_single_candidate_no_conflict(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
assert resolver.detect_conflicts([{"rank": 1}]) is False
|
|
|
|
def test_multiple_candidates_conflict(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
assert resolver.detect_conflicts([{"rank": 1}, {"rank": 2}]) is True
|
|
|
|
|
|
class TestSemanticSourceResolverApplyFieldDecision:
|
|
"""SemanticSourceResolver.apply_field_decision — field merging."""
|
|
|
|
def test_merge_fields(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
field_state = {"field_name": "region", "status": "unresolved"}
|
|
decision = {"status": "resolved", "verbose_name": "Region"}
|
|
merged = resolver.apply_field_decision(field_state, decision)
|
|
assert merged["field_name"] == "region"
|
|
assert merged["status"] == "resolved"
|
|
assert merged["verbose_name"] == "Region"
|
|
|
|
|
|
class TestSemanticSourceResolverPropagateSourceVersion:
|
|
"""SemanticSourceResolver.propagate_source_version_update — version propagation."""
|
|
|
|
def test_propagate_success(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import SemanticSource
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source = SemanticSource(source_id="src-1", source_version="v2")
|
|
|
|
field_mock = MagicMock()
|
|
field_mock.source_id = "src-1"
|
|
field_mock.is_locked = False
|
|
field_mock.provenance = "dictionary_exact"
|
|
field_mock.source_version = "v1"
|
|
field_mock.needs_review = False
|
|
field_mock.has_conflict = False
|
|
|
|
result = resolver.propagate_source_version_update(source, [field_mock])
|
|
assert result["propagated"] == 1
|
|
assert result["preserved_locked"] == 0
|
|
assert result["untouched"] == 0
|
|
assert field_mock.source_version == "v2"
|
|
assert field_mock.needs_review is True
|
|
|
|
def test_preserve_locked(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import SemanticSource
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source = SemanticSource(source_id="src-1", source_version="v2")
|
|
|
|
field_mock = MagicMock()
|
|
field_mock.source_id = "src-1"
|
|
field_mock.is_locked = True
|
|
field_mock.provenance = "manual_override"
|
|
field_mock.source_version = "v1"
|
|
|
|
result = resolver.propagate_source_version_update(source, [field_mock])
|
|
assert result["propagated"] == 0
|
|
assert result["preserved_locked"] == 1
|
|
|
|
def test_untouched_different_source(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import SemanticSource
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source = SemanticSource(source_id="src-1", source_version="v2")
|
|
|
|
field_mock = MagicMock()
|
|
field_mock.source_id = "src-2"
|
|
field_mock.is_locked = False
|
|
field_mock.provenance = "dictionary_exact"
|
|
|
|
result = resolver.propagate_source_version_update(source, [field_mock])
|
|
assert result["propagated"] == 0
|
|
assert result["untouched"] == 1
|
|
|
|
def test_missing_source_metadata_raises(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import SemanticSource
|
|
|
|
resolver = SemanticSourceResolver()
|
|
source = SemanticSource(source_id="src-1", source_version="")
|
|
with pytest.raises(ValueError, match="source_version"):
|
|
resolver.propagate_source_version_update(source, [])
|
|
|
|
|
|
class TestNormalizeDictionaryRow:
|
|
"""SemanticSourceResolver._normalize_dictionary_row — row normalization."""
|
|
|
|
def test_field_name_variants(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
r1 = resolver._normalize_dictionary_row({"column_name": "col1", "verbose_name": "Col 1"})
|
|
assert r1["field_name"] == "col1"
|
|
r2 = resolver._normalize_dictionary_row({"name": "named_field"})
|
|
assert r2["field_name"] == "named_field"
|
|
r3 = resolver._normalize_dictionary_row({"field": "field_val"})
|
|
assert r3["field_name"] == "field_val"
|
|
r4 = resolver._normalize_dictionary_row({"field_name": "direct"})
|
|
assert r4["field_name"] == "direct"
|
|
|
|
def test_verbose_name_fallback(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
r1 = resolver._normalize_dictionary_row({"field_name": "a", "label": "Label A"})
|
|
assert r1["verbose_name"] == "Label A"
|
|
r2 = resolver._normalize_dictionary_row({"field_name": "b"})
|
|
assert r2.get("verbose_name") is None
|
|
|
|
|
|
class TestMatchPriority:
|
|
"""SemanticSourceResolver._match_priority — priority encoding."""
|
|
|
|
def test_priority_ordering(self):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
from src.models.dataset_review import CandidateMatchType
|
|
|
|
resolver = SemanticSourceResolver()
|
|
assert resolver._match_priority(CandidateMatchType.EXACT.value) == 0
|
|
assert resolver._match_priority(CandidateMatchType.FUZZY.value) == 2
|
|
assert resolver._match_priority(CandidateMatchType.GENERATED.value) == 3
|
|
assert resolver._match_priority(None) == 99
|
|
assert resolver._match_priority("UNKNOWN") == 99
|
|
|
|
|
|
class TestNormalizeKey:
|
|
"""SemanticSourceResolver._normalize_key — key normalization."""
|
|
|
|
@pytest.mark.parametrize("input_val,expected", [
|
|
("Hello World", "helloworld"),
|
|
("region_code", "region_code"),
|
|
("UPPER_CASE", "upper_case"), # _normalize_key preserves underscores
|
|
(" with spaces ", "withspaces"),
|
|
("special!@#chars", "specialchars"),
|
|
("", ""),
|
|
])
|
|
def test_normalize_key(self, input_val, expected):
|
|
from src.services.dataset_review.semantic_resolver import SemanticSourceResolver
|
|
|
|
resolver = SemanticSourceResolver()
|
|
assert resolver._normalize_key(input_val) == expected
|
|
# #endregion Test.SemanticSourceResolver
|