refactor
This commit is contained in:
File diff suppressed because it is too large
Load Diff
122
backend/src/models/dataset_review_pkg/__init__.py
Normal file
122
backend/src/models/dataset_review_pkg/__init__.py
Normal file
@@ -0,0 +1,122 @@
|
||||
# [DEF:DatasetReviewModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: dataset_review, session, profile, findings, semantics, clarification, execution, sqlalchemy
|
||||
# @PURPOSE: Re-export all dataset review domain models from decomposed sub-modules for backward-compatible imports.
|
||||
# @LAYER: Domain
|
||||
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
SessionStatus,
|
||||
SessionPhase,
|
||||
ReadinessState,
|
||||
RecommendedAction,
|
||||
SessionCollaboratorRole,
|
||||
BusinessSummarySource,
|
||||
ConfidenceState,
|
||||
FindingArea,
|
||||
FindingSeverity,
|
||||
ResolutionState,
|
||||
SemanticSourceType,
|
||||
TrustLevel,
|
||||
SemanticSourceStatus,
|
||||
FieldKind,
|
||||
FieldProvenance,
|
||||
CandidateMatchType,
|
||||
CandidateStatus,
|
||||
FilterSource,
|
||||
FilterConfidenceState,
|
||||
FilterRecoveryStatus,
|
||||
VariableKind,
|
||||
MappingStatus,
|
||||
MappingMethod,
|
||||
MappingWarningLevel,
|
||||
ApprovalState,
|
||||
ClarificationStatus,
|
||||
QuestionState,
|
||||
AnswerKind,
|
||||
PreviewStatus,
|
||||
LaunchStatus,
|
||||
ArtifactType,
|
||||
ArtifactFormat,
|
||||
)
|
||||
from src.models.dataset_review_pkg._session_models import (
|
||||
SessionCollaborator,
|
||||
DatasetReviewSession,
|
||||
)
|
||||
from src.models.dataset_review_pkg._profile_models import DatasetProfile
|
||||
from src.models.dataset_review_pkg._finding_models import ValidationFinding
|
||||
from src.models.dataset_review_pkg._semantic_models import (
|
||||
SemanticSource,
|
||||
SemanticFieldEntry,
|
||||
SemanticCandidate,
|
||||
)
|
||||
from src.models.dataset_review_pkg._filter_models import (
|
||||
ImportedFilter,
|
||||
TemplateVariable,
|
||||
)
|
||||
from src.models.dataset_review_pkg._mapping_models import ExecutionMapping
|
||||
from src.models.dataset_review_pkg._clarification_models import (
|
||||
ClarificationSession,
|
||||
ClarificationQuestion,
|
||||
ClarificationOption,
|
||||
ClarificationAnswer,
|
||||
)
|
||||
from src.models.dataset_review_pkg._execution_models import (
|
||||
CompiledPreview,
|
||||
DatasetRunContext,
|
||||
SessionEvent,
|
||||
ExportArtifact,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SessionStatus",
|
||||
"SessionPhase",
|
||||
"ReadinessState",
|
||||
"RecommendedAction",
|
||||
"SessionCollaboratorRole",
|
||||
"BusinessSummarySource",
|
||||
"ConfidenceState",
|
||||
"FindingArea",
|
||||
"FindingSeverity",
|
||||
"ResolutionState",
|
||||
"SemanticSourceType",
|
||||
"TrustLevel",
|
||||
"SemanticSourceStatus",
|
||||
"FieldKind",
|
||||
"FieldProvenance",
|
||||
"CandidateMatchType",
|
||||
"CandidateStatus",
|
||||
"FilterSource",
|
||||
"FilterConfidenceState",
|
||||
"FilterRecoveryStatus",
|
||||
"VariableKind",
|
||||
"MappingStatus",
|
||||
"MappingMethod",
|
||||
"MappingWarningLevel",
|
||||
"ApprovalState",
|
||||
"ClarificationStatus",
|
||||
"QuestionState",
|
||||
"AnswerKind",
|
||||
"PreviewStatus",
|
||||
"LaunchStatus",
|
||||
"ArtifactType",
|
||||
"ArtifactFormat",
|
||||
"SessionCollaborator",
|
||||
"DatasetReviewSession",
|
||||
"DatasetProfile",
|
||||
"ValidationFinding",
|
||||
"SemanticSource",
|
||||
"SemanticFieldEntry",
|
||||
"SemanticCandidate",
|
||||
"ImportedFilter",
|
||||
"TemplateVariable",
|
||||
"ExecutionMapping",
|
||||
"ClarificationSession",
|
||||
"ClarificationQuestion",
|
||||
"ClarificationOption",
|
||||
"ClarificationAnswer",
|
||||
"CompiledPreview",
|
||||
"DatasetRunContext",
|
||||
"SessionEvent",
|
||||
"ExportArtifact",
|
||||
]
|
||||
# [/DEF:DatasetReviewModels:Module]
|
||||
125
backend/src/models/dataset_review_pkg/_clarification_models.py
Normal file
125
backend/src/models/dataset_review_pkg/_clarification_models.py
Normal file
@@ -0,0 +1,125 @@
|
||||
# [DEF:DatasetReviewClarificationModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Clarification session, question, option, and answer models for guided review flow.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
# @INVARIANT: Only one active clarification question may exist at a time per session.
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Column,
|
||||
String,
|
||||
Integer,
|
||||
Text,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
ClarificationStatus,
|
||||
QuestionState,
|
||||
AnswerKind,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:ClarificationSession:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One clarification session aggregate owning questions and tracking resolution progress.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class ClarificationSession(Base):
|
||||
__tablename__ = "clarification_sessions"
|
||||
|
||||
clarification_session_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(String, ForeignKey("dataset_review_sessions.session_id"), nullable=False)
|
||||
status = Column(SQLEnum(ClarificationStatus), nullable=False, default=ClarificationStatus.PENDING)
|
||||
current_question_id = Column(String, nullable=True)
|
||||
resolved_count = Column(Integer, nullable=False, default=0)
|
||||
remaining_count = Column(Integer, nullable=False, default=0)
|
||||
summary_delta = Column(Text, nullable=True)
|
||||
started_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="clarification_sessions")
|
||||
questions = relationship("ClarificationQuestion", back_populates="clarification_session", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
# [/DEF:ClarificationSession:Class]
|
||||
|
||||
|
||||
# [DEF:ClarificationQuestion:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One clarification question with priority ordering, options, and state machine.
|
||||
# @RELATION: DEPENDS_ON -> [ClarificationSession]
|
||||
class ClarificationQuestion(Base):
|
||||
__tablename__ = "clarification_questions"
|
||||
|
||||
question_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
clarification_session_id = Column(String, ForeignKey("clarification_sessions.clarification_session_id"), nullable=False)
|
||||
topic_ref = Column(String, nullable=False)
|
||||
question_text = Column(Text, nullable=False)
|
||||
why_it_matters = Column(Text, nullable=False)
|
||||
current_guess = Column(Text, nullable=True)
|
||||
priority = Column(Integer, nullable=False, default=0)
|
||||
state = Column(SQLEnum(QuestionState), nullable=False, default=QuestionState.OPEN)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
clarification_session = relationship("ClarificationSession", back_populates="questions")
|
||||
options = relationship("ClarificationOption", back_populates="question", cascade="all, delete-orphan")
|
||||
answer = relationship("ClarificationAnswer", back_populates="question", uselist=False, cascade="all, delete-orphan")
|
||||
|
||||
|
||||
# [/DEF:ClarificationQuestion:Class]
|
||||
|
||||
|
||||
# [DEF:ClarificationOption:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: One selectable option for a clarification question with recommendation flag.
|
||||
# @RELATION: DEPENDS_ON -> [ClarificationQuestion]
|
||||
class ClarificationOption(Base):
|
||||
__tablename__ = "clarification_options"
|
||||
|
||||
option_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
question_id = Column(String, ForeignKey("clarification_questions.question_id"), nullable=False)
|
||||
label = Column(String, nullable=False)
|
||||
value = Column(String, nullable=False)
|
||||
is_recommended = Column(Boolean, nullable=False, default=False)
|
||||
display_order = Column(Integer, nullable=False, default=0)
|
||||
|
||||
question = relationship("ClarificationQuestion", back_populates="options")
|
||||
|
||||
|
||||
# [/DEF:ClarificationOption:Class]
|
||||
|
||||
|
||||
# [DEF:ClarificationAnswer:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One persisted clarification answer with impact summary and feedback tracking.
|
||||
# @RELATION: DEPENDS_ON -> [ClarificationQuestion]
|
||||
class ClarificationAnswer(Base):
|
||||
__tablename__ = "clarification_answers"
|
||||
|
||||
answer_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
question_id = Column(String, ForeignKey("clarification_questions.question_id"), nullable=False, unique=True)
|
||||
answer_kind = Column(SQLEnum(AnswerKind), nullable=False)
|
||||
answer_value = Column(Text, nullable=True)
|
||||
answered_by_user_id = Column(String, nullable=False)
|
||||
impact_summary = Column(Text, nullable=True)
|
||||
user_feedback = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
question = relationship("ClarificationQuestion", back_populates="answer")
|
||||
|
||||
|
||||
# [/DEF:ClarificationAnswer:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewClarificationModels:Module]
|
||||
463
backend/src/models/dataset_review_pkg/_enums.py
Normal file
463
backend/src/models/dataset_review_pkg/_enums.py
Normal file
@@ -0,0 +1,463 @@
|
||||
# [DEF:DatasetReviewEnums:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: All enumeration types for the dataset review domain, grouped for stable cross-module reuse.
|
||||
# @LAYER: Domain
|
||||
# @INVARIANT: Enum values are string-based for JSON serialization compatibility.
|
||||
|
||||
import enum
|
||||
|
||||
|
||||
# [DEF:SessionStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status of a dataset review session.
|
||||
class SessionStatus(str, enum.Enum):
|
||||
ACTIVE = "active"
|
||||
PAUSED = "paused"
|
||||
COMPLETED = "completed"
|
||||
ARCHIVED = "archived"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
# [/DEF:SessionStatus:Class]
|
||||
|
||||
|
||||
# [DEF:SessionPhase:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Ordered phase progression for dataset review orchestration.
|
||||
class SessionPhase(str, enum.Enum):
|
||||
INTAKE = "intake"
|
||||
RECOVERY = "recovery"
|
||||
REVIEW = "review"
|
||||
SEMANTIC_REVIEW = "semantic_review"
|
||||
CLARIFICATION = "clarification"
|
||||
MAPPING_REVIEW = "mapping_review"
|
||||
PREVIEW = "preview"
|
||||
LAUNCH = "launch"
|
||||
POST_RUN = "post_run"
|
||||
|
||||
|
||||
# [/DEF:SessionPhase:Class]
|
||||
|
||||
|
||||
# [DEF:ReadinessState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Granular readiness indicator driving the recommended-action UX flow.
|
||||
class ReadinessState(str, enum.Enum):
|
||||
EMPTY = "empty"
|
||||
IMPORTING = "importing"
|
||||
REVIEW_READY = "review_ready"
|
||||
SEMANTIC_SOURCE_REVIEW_NEEDED = "semantic_source_review_needed"
|
||||
CLARIFICATION_NEEDED = "clarification_needed"
|
||||
CLARIFICATION_ACTIVE = "clarification_active"
|
||||
MAPPING_REVIEW_NEEDED = "mapping_review_needed"
|
||||
COMPILED_PREVIEW_READY = "compiled_preview_ready"
|
||||
PARTIALLY_READY = "partially_ready"
|
||||
RUN_READY = "run_ready"
|
||||
RUN_IN_PROGRESS = "run_in_progress"
|
||||
COMPLETED = "completed"
|
||||
RECOVERY_REQUIRED = "recovery_required"
|
||||
|
||||
|
||||
# [/DEF:ReadinessState:Class]
|
||||
|
||||
|
||||
# [DEF:RecommendedAction:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Next-action guidance derived from the current readiness state.
|
||||
class RecommendedAction(str, enum.Enum):
|
||||
IMPORT_FROM_SUPERSET = "import_from_superset"
|
||||
REVIEW_DOCUMENTATION = "review_documentation"
|
||||
APPLY_SEMANTIC_SOURCE = "apply_semantic_source"
|
||||
START_CLARIFICATION = "start_clarification"
|
||||
ANSWER_NEXT_QUESTION = "answer_next_question"
|
||||
APPROVE_MAPPING = "approve_mapping"
|
||||
GENERATE_SQL_PREVIEW = "generate_sql_preview"
|
||||
COMPLETE_REQUIRED_VALUES = "complete_required_values"
|
||||
LAUNCH_DATASET = "launch_dataset"
|
||||
RESUME_SESSION = "resume_session"
|
||||
EXPORT_OUTPUTS = "export_outputs"
|
||||
|
||||
|
||||
# [/DEF:RecommendedAction:Class]
|
||||
|
||||
|
||||
# [DEF:SessionCollaboratorRole:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: RBAC role for session collaborators.
|
||||
class SessionCollaboratorRole(str, enum.Enum):
|
||||
VIEWER = "viewer"
|
||||
REVIEWER = "reviewer"
|
||||
APPROVER = "approver"
|
||||
|
||||
|
||||
# [/DEF:SessionCollaboratorRole:Class]
|
||||
|
||||
|
||||
# [DEF:BusinessSummarySource:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Provenance of the dataset business summary text.
|
||||
class BusinessSummarySource(str, enum.Enum):
|
||||
CONFIRMED = "confirmed"
|
||||
IMPORTED = "imported"
|
||||
INFERRED = "inferred"
|
||||
AI_DRAFT = "ai_draft"
|
||||
MANUAL_OVERRIDE = "manual_override"
|
||||
|
||||
|
||||
# [/DEF:BusinessSummarySource:Class]
|
||||
|
||||
|
||||
# [DEF:ConfidenceState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Confidence level for dataset profile completeness.
|
||||
class ConfidenceState(str, enum.Enum):
|
||||
CONFIRMED = "confirmed"
|
||||
MOSTLY_CONFIRMED = "mostly_confirmed"
|
||||
MIXED = "mixed"
|
||||
LOW_CONFIDENCE = "low_confidence"
|
||||
UNRESOLVED = "unresolved"
|
||||
|
||||
|
||||
# [/DEF:ConfidenceState:Class]
|
||||
|
||||
|
||||
# [DEF:FindingArea:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Domain area classification for validation findings.
|
||||
class FindingArea(str, enum.Enum):
|
||||
SOURCE_INTAKE = "source_intake"
|
||||
DATASET_PROFILE = "dataset_profile"
|
||||
SEMANTIC_ENRICHMENT = "semantic_enrichment"
|
||||
CLARIFICATION = "clarification"
|
||||
FILTER_RECOVERY = "filter_recovery"
|
||||
TEMPLATE_MAPPING = "template_mapping"
|
||||
COMPILED_PREVIEW = "compiled_preview"
|
||||
LAUNCH = "launch"
|
||||
AUDIT = "audit"
|
||||
|
||||
|
||||
# [/DEF:FindingArea:Class]
|
||||
|
||||
|
||||
# [DEF:FindingSeverity:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Severity classification for validation findings.
|
||||
class FindingSeverity(str, enum.Enum):
|
||||
BLOCKING = "blocking"
|
||||
WARNING = "warning"
|
||||
INFORMATIONAL = "informational"
|
||||
|
||||
|
||||
# [/DEF:FindingSeverity:Class]
|
||||
|
||||
|
||||
# [DEF:ResolutionState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Resolution status for validation findings and clarification items.
|
||||
class ResolutionState(str, enum.Enum):
|
||||
OPEN = "open"
|
||||
RESOLVED = "resolved"
|
||||
APPROVED = "approved"
|
||||
SKIPPED = "skipped"
|
||||
DEFERRED = "deferred"
|
||||
EXPERT_REVIEW = "expert_review"
|
||||
|
||||
|
||||
# [/DEF:ResolutionState:Class]
|
||||
|
||||
|
||||
# [DEF:SemanticSourceType:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Classification of semantic enrichment source origins.
|
||||
class SemanticSourceType(str, enum.Enum):
|
||||
UPLOADED_FILE = "uploaded_file"
|
||||
CONNECTED_DICTIONARY = "connected_dictionary"
|
||||
REFERENCE_DATASET = "reference_dataset"
|
||||
NEIGHBOR_DATASET = "neighbor_dataset"
|
||||
AI_GENERATED = "ai_generated"
|
||||
|
||||
|
||||
# [/DEF:SemanticSourceType:Class]
|
||||
|
||||
|
||||
# [DEF:TrustLevel:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Trust classification for semantic source reliability.
|
||||
class TrustLevel(str, enum.Enum):
|
||||
TRUSTED = "trusted"
|
||||
RECOMMENDED = "recommended"
|
||||
CANDIDATE = "candidate"
|
||||
GENERATED = "generated"
|
||||
|
||||
|
||||
# [/DEF:TrustLevel:Class]
|
||||
|
||||
|
||||
# [DEF:SemanticSourceStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status for semantic source application.
|
||||
class SemanticSourceStatus(str, enum.Enum):
|
||||
AVAILABLE = "available"
|
||||
SELECTED = "selected"
|
||||
APPLIED = "applied"
|
||||
REJECTED = "rejected"
|
||||
PARTIAL = "partial"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
# [/DEF:SemanticSourceStatus:Class]
|
||||
|
||||
|
||||
# [DEF:FieldKind:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Kind classification for semantic field entries.
|
||||
class FieldKind(str, enum.Enum):
|
||||
COLUMN = "column"
|
||||
METRIC = "metric"
|
||||
FILTER_DIMENSION = "filter_dimension"
|
||||
PARAMETER = "parameter"
|
||||
|
||||
|
||||
# [/DEF:FieldKind:Class]
|
||||
|
||||
|
||||
# [DEF:FieldProvenance:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Provenance tracking for semantic field value origin.
|
||||
class FieldProvenance(str, enum.Enum):
|
||||
DICTIONARY_EXACT = "dictionary_exact"
|
||||
REFERENCE_IMPORTED = "reference_imported"
|
||||
FUZZY_INFERRED = "fuzzy_inferred"
|
||||
AI_GENERATED = "ai_generated"
|
||||
MANUAL_OVERRIDE = "manual_override"
|
||||
UNRESOLVED = "unresolved"
|
||||
|
||||
|
||||
# [/DEF:FieldProvenance:Class]
|
||||
|
||||
|
||||
# [DEF:CandidateMatchType:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Match type classification for semantic candidates.
|
||||
class CandidateMatchType(str, enum.Enum):
|
||||
EXACT = "exact"
|
||||
REFERENCE = "reference"
|
||||
FUZZY = "fuzzy"
|
||||
GENERATED = "generated"
|
||||
|
||||
|
||||
# [/DEF:CandidateMatchType:Class]
|
||||
|
||||
|
||||
# [DEF:CandidateStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status for semantic candidate proposals.
|
||||
class CandidateStatus(str, enum.Enum):
|
||||
PROPOSED = "proposed"
|
||||
ACCEPTED = "accepted"
|
||||
REJECTED = "rejected"
|
||||
SUPERSEDED = "superseded"
|
||||
|
||||
|
||||
# [/DEF:CandidateStatus:Class]
|
||||
|
||||
|
||||
# [DEF:FilterSource:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Origin classification for imported filters.
|
||||
class FilterSource(str, enum.Enum):
|
||||
SUPERSET_NATIVE = "superset_native"
|
||||
SUPERSET_URL = "superset_url"
|
||||
SUPERSET_PERMALINK = "superset_permalink"
|
||||
SUPERSET_NATIVE_FILTERS_KEY = "superset_native_filters_key"
|
||||
MANUAL = "manual"
|
||||
INFERRED = "inferred"
|
||||
|
||||
|
||||
# [/DEF:FilterSource:Class]
|
||||
|
||||
|
||||
# [DEF:FilterConfidenceState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Confidence classification for imported filter values.
|
||||
class FilterConfidenceState(str, enum.Enum):
|
||||
CONFIRMED = "confirmed"
|
||||
IMPORTED = "imported"
|
||||
INFERRED = "inferred"
|
||||
AI_DRAFT = "ai_draft"
|
||||
UNRESOLVED = "unresolved"
|
||||
|
||||
|
||||
# [/DEF:FilterConfidenceState:Class]
|
||||
|
||||
|
||||
# [DEF:FilterRecoveryStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Recovery quality status for imported filters.
|
||||
class FilterRecoveryStatus(str, enum.Enum):
|
||||
RECOVERED = "recovered"
|
||||
PARTIAL = "partial"
|
||||
MISSING = "missing"
|
||||
CONFLICTED = "conflicted"
|
||||
|
||||
|
||||
# [/DEF:FilterRecoveryStatus:Class]
|
||||
|
||||
|
||||
# [DEF:VariableKind:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Kind classification for template variables.
|
||||
class VariableKind(str, enum.Enum):
|
||||
NATIVE_FILTER = "native_filter"
|
||||
PARAMETER = "parameter"
|
||||
DERIVED = "derived"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
# [/DEF:VariableKind:Class]
|
||||
|
||||
|
||||
# [DEF:MappingStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status for template variable mapping.
|
||||
class MappingStatus(str, enum.Enum):
|
||||
UNMAPPED = "unmapped"
|
||||
PROPOSED = "proposed"
|
||||
APPROVED = "approved"
|
||||
OVERRIDDEN = "overridden"
|
||||
INVALID = "invalid"
|
||||
|
||||
|
||||
# [/DEF:MappingStatus:Class]
|
||||
|
||||
|
||||
# [DEF:MappingMethod:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Method classification for execution mapping creation.
|
||||
class MappingMethod(str, enum.Enum):
|
||||
DIRECT_MATCH = "direct_match"
|
||||
HEURISTIC_MATCH = "heuristic_match"
|
||||
SEMANTIC_MATCH = "semantic_match"
|
||||
MANUAL_OVERRIDE = "manual_override"
|
||||
|
||||
|
||||
# [/DEF:MappingMethod:Class]
|
||||
|
||||
|
||||
# [DEF:MappingWarningLevel:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Warning severity for execution mapping quality indicators.
|
||||
class MappingWarningLevel(str, enum.Enum):
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
|
||||
|
||||
# [/DEF:MappingWarningLevel:Class]
|
||||
|
||||
|
||||
# [DEF:ApprovalState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Approval lifecycle for execution mapping gate checks.
|
||||
class ApprovalState(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
APPROVED = "approved"
|
||||
REJECTED = "rejected"
|
||||
NOT_REQUIRED = "not_required"
|
||||
|
||||
|
||||
# [/DEF:ApprovalState:Class]
|
||||
|
||||
|
||||
# [DEF:ClarificationStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status for clarification sessions.
|
||||
class ClarificationStatus(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
ACTIVE = "active"
|
||||
PAUSED = "paused"
|
||||
COMPLETED = "completed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
# [/DEF:ClarificationStatus:Class]
|
||||
|
||||
|
||||
# [DEF:QuestionState:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: State machine for individual clarification questions.
|
||||
class QuestionState(str, enum.Enum):
|
||||
OPEN = "open"
|
||||
ANSWERED = "answered"
|
||||
SKIPPED = "skipped"
|
||||
EXPERT_REVIEW = "expert_review"
|
||||
SUPERSEDED = "superseded"
|
||||
|
||||
|
||||
# [/DEF:QuestionState:Class]
|
||||
|
||||
|
||||
# [DEF:AnswerKind:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Classification of clarification answer types.
|
||||
class AnswerKind(str, enum.Enum):
|
||||
SELECTED = "selected"
|
||||
CUSTOM = "custom"
|
||||
SKIPPED = "skipped"
|
||||
EXPERT_REVIEW = "expert_review"
|
||||
|
||||
|
||||
# [/DEF:AnswerKind:Class]
|
||||
|
||||
|
||||
# [DEF:PreviewStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Lifecycle status for compiled SQL previews.
|
||||
class PreviewStatus(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
READY = "ready"
|
||||
FAILED = "failed"
|
||||
STALE = "stale"
|
||||
|
||||
|
||||
# [/DEF:PreviewStatus:Class]
|
||||
|
||||
|
||||
# [DEF:LaunchStatus:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Outcome status for dataset launch handoff.
|
||||
class LaunchStatus(str, enum.Enum):
|
||||
STARTED = "started"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
# [/DEF:LaunchStatus:Class]
|
||||
|
||||
|
||||
# [DEF:ArtifactType:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Type classification for export artifacts.
|
||||
class ArtifactType(str, enum.Enum):
|
||||
DOCUMENTATION = "documentation"
|
||||
VALIDATION_REPORT = "validation_report"
|
||||
RUN_SUMMARY = "run_summary"
|
||||
|
||||
|
||||
# [/DEF:ArtifactType:Class]
|
||||
|
||||
|
||||
# [DEF:ArtifactFormat:Class]
|
||||
# @COMPLEXITY: 1
|
||||
# @PURPOSE: Format classification for export artifact output.
|
||||
class ArtifactFormat(str, enum.Enum):
|
||||
JSON = "json"
|
||||
MARKDOWN = "markdown"
|
||||
CSV = "csv"
|
||||
PDF = "pdf"
|
||||
|
||||
|
||||
# [/DEF:ArtifactFormat:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewEnums:Module]
|
||||
140
backend/src/models/dataset_review_pkg/_execution_models.py
Normal file
140
backend/src/models/dataset_review_pkg/_execution_models.py
Normal file
@@ -0,0 +1,140 @@
|
||||
# [DEF:DatasetReviewExecutionModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Compiled preview, run context, session event, and export artifact models for execution and audit.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Text,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
JSON,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
PreviewStatus,
|
||||
LaunchStatus,
|
||||
ArtifactType,
|
||||
ArtifactFormat,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:CompiledPreview:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One compiled SQL preview snapshot with fingerprint for staleness detection.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class CompiledPreview(Base):
|
||||
__tablename__ = "compiled_previews"
|
||||
|
||||
preview_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
preview_status = Column(
|
||||
SQLEnum(PreviewStatus), nullable=False, default=PreviewStatus.PENDING
|
||||
)
|
||||
compiled_sql = Column(Text, nullable=True)
|
||||
preview_fingerprint = Column(String, nullable=False)
|
||||
compiled_by = Column(String, nullable=False, default="superset")
|
||||
error_code = Column(String, nullable=True)
|
||||
error_details = Column(Text, nullable=True)
|
||||
compiled_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="previews")
|
||||
|
||||
|
||||
# [/DEF:CompiledPreview:Class]
|
||||
|
||||
|
||||
# [DEF:DatasetRunContext:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Immutable launch audit snapshot capturing effective filters, template params, and approval state at launch time.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class DatasetRunContext(Base):
|
||||
__tablename__ = "dataset_run_contexts"
|
||||
|
||||
run_context_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
dataset_ref = Column(String, nullable=False)
|
||||
environment_id = Column(String, nullable=False)
|
||||
preview_id = Column(String, nullable=False)
|
||||
sql_lab_session_ref = Column(String, nullable=False)
|
||||
effective_filters = Column(JSON, nullable=False)
|
||||
template_params = Column(JSON, nullable=False)
|
||||
approved_mapping_ids = Column(JSON, nullable=False)
|
||||
semantic_decision_refs = Column(JSON, nullable=False)
|
||||
open_warning_refs = Column(JSON, nullable=False)
|
||||
launch_status = Column(SQLEnum(LaunchStatus), nullable=False)
|
||||
launch_error = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="run_contexts")
|
||||
|
||||
|
||||
# [/DEF:DatasetRunContext:Class]
|
||||
|
||||
|
||||
# [DEF:SessionEvent:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One persisted audit event for dataset review session mutations.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class SessionEvent(Base):
|
||||
__tablename__ = "session_events"
|
||||
|
||||
session_event_id = Column(
|
||||
String, primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
)
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
actor_user_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
event_type = Column(String, nullable=False)
|
||||
event_summary = Column(Text, nullable=False)
|
||||
current_phase = Column(String, nullable=True)
|
||||
readiness_state = Column(String, nullable=True)
|
||||
event_details = Column(JSON, nullable=False, default=dict)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="events")
|
||||
actor = relationship("User")
|
||||
|
||||
|
||||
# [/DEF:SessionEvent:Class]
|
||||
|
||||
|
||||
# [DEF:ExportArtifact:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One persisted export artifact reference for documentation and validation outputs.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class ExportArtifact(Base):
|
||||
__tablename__ = "export_artifacts"
|
||||
|
||||
artifact_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
artifact_type = Column(SQLEnum(ArtifactType), nullable=False)
|
||||
format = Column(SQLEnum(ArtifactFormat), nullable=False)
|
||||
storage_ref = Column(String, nullable=False)
|
||||
created_by_user_id = Column(String, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="export_artifacts")
|
||||
|
||||
|
||||
# [/DEF:ExportArtifact:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewExecutionModels:Module]
|
||||
95
backend/src/models/dataset_review_pkg/_filter_models.py
Normal file
95
backend/src/models/dataset_review_pkg/_filter_models.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# [DEF:DatasetReviewFilterModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Imported filter and template variable models for Superset context recovery and execution mapping.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Text,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
JSON,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
FilterSource,
|
||||
FilterConfidenceState,
|
||||
FilterRecoveryStatus,
|
||||
VariableKind,
|
||||
MappingStatus,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:ImportedFilter:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Recovered Superset filter with confidence and recovery status tracking.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class ImportedFilter(Base):
|
||||
__tablename__ = "imported_filters"
|
||||
|
||||
filter_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
filter_name = Column(String, nullable=False)
|
||||
display_name = Column(String, nullable=True)
|
||||
raw_value = Column(JSON, nullable=False)
|
||||
raw_value_masked = Column(Boolean, nullable=False, default=False)
|
||||
normalized_value = Column(JSON, nullable=True)
|
||||
source = Column(SQLEnum(FilterSource), nullable=False)
|
||||
confidence_state = Column(SQLEnum(FilterConfidenceState), nullable=False)
|
||||
requires_confirmation = Column(Boolean, nullable=False, default=False)
|
||||
recovery_status = Column(SQLEnum(FilterRecoveryStatus), nullable=False)
|
||||
notes = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(
|
||||
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="imported_filters")
|
||||
|
||||
|
||||
# [/DEF:ImportedFilter:Class]
|
||||
|
||||
|
||||
# [DEF:TemplateVariable:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Discovered template variable from dataset SQL with mapping status tracking.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class TemplateVariable(Base):
|
||||
__tablename__ = "template_variables"
|
||||
|
||||
variable_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
variable_name = Column(String, nullable=False)
|
||||
expression_source = Column(Text, nullable=False)
|
||||
variable_kind = Column(SQLEnum(VariableKind), nullable=False)
|
||||
is_required = Column(Boolean, nullable=False, default=True)
|
||||
default_value = Column(JSON, nullable=True)
|
||||
mapping_status = Column(
|
||||
SQLEnum(MappingStatus), nullable=False, default=MappingStatus.UNMAPPED
|
||||
)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(
|
||||
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="template_variables")
|
||||
|
||||
|
||||
# [/DEF:TemplateVariable:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewFilterModels:Module]
|
||||
59
backend/src/models/dataset_review_pkg/_finding_models.py
Normal file
59
backend/src/models/dataset_review_pkg/_finding_models.py
Normal file
@@ -0,0 +1,59 @@
|
||||
# [DEF:DatasetReviewFindingModels:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Validation finding model for tracking blocking, warning, and informational issues during review.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Text,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
FindingArea,
|
||||
FindingSeverity,
|
||||
ResolutionState,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:ValidationFinding:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Structured finding record for dataset review validation issues with resolution tracking.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class ValidationFinding(Base):
|
||||
__tablename__ = "validation_findings"
|
||||
|
||||
finding_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
area = Column(SQLEnum(FindingArea), nullable=False)
|
||||
severity = Column(SQLEnum(FindingSeverity), nullable=False)
|
||||
code = Column(String, nullable=False)
|
||||
title = Column(String, nullable=False)
|
||||
message = Column(Text, nullable=False)
|
||||
resolution_state = Column(
|
||||
SQLEnum(ResolutionState), nullable=False, default=ResolutionState.OPEN
|
||||
)
|
||||
resolution_note = Column(Text, nullable=True)
|
||||
caused_by_ref = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="findings")
|
||||
|
||||
|
||||
# [/DEF:ValidationFinding:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewFindingModels:Module]
|
||||
61
backend/src/models/dataset_review_pkg/_mapping_models.py
Normal file
61
backend/src/models/dataset_review_pkg/_mapping_models.py
Normal file
@@ -0,0 +1,61 @@
|
||||
# [DEF:DatasetReviewMappingModels:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Execution mapping model linking imported filters to template variables with approval gates.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Text,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
JSON,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
MappingMethod,
|
||||
MappingWarningLevel,
|
||||
ApprovalState,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:ExecutionMapping:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One filter-to-variable mapping with approval gate, effective value, and transformation metadata.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
# @INVARIANT: Explicit approval is required before launch when requires_explicit_approval is true.
|
||||
class ExecutionMapping(Base):
|
||||
__tablename__ = "execution_mappings"
|
||||
|
||||
mapping_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(String, ForeignKey("dataset_review_sessions.session_id"), nullable=False)
|
||||
filter_id = Column(String, nullable=False)
|
||||
variable_id = Column(String, nullable=False)
|
||||
mapping_method = Column(SQLEnum(MappingMethod), nullable=False)
|
||||
raw_input_value = Column(JSON, nullable=False)
|
||||
effective_value = Column(JSON, nullable=True)
|
||||
transformation_note = Column(Text, nullable=True)
|
||||
warning_level = Column(SQLEnum(MappingWarningLevel), nullable=True)
|
||||
requires_explicit_approval = Column(Boolean, nullable=False, default=False)
|
||||
approval_state = Column(SQLEnum(ApprovalState), nullable=False, default=ApprovalState.NOT_REQUIRED)
|
||||
approved_by_user_id = Column(String, nullable=True)
|
||||
approved_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="execution_mappings")
|
||||
|
||||
|
||||
# [/DEF:ExecutionMapping:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewMappingModels:Module]
|
||||
68
backend/src/models/dataset_review_pkg/_profile_models.py
Normal file
68
backend/src/models/dataset_review_pkg/_profile_models.py
Normal file
@@ -0,0 +1,68 @@
|
||||
# [DEF:DatasetReviewProfileModels:Module]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Dataset profile model capturing business summary, confidence, and completeness metadata.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Text,
|
||||
Float,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
BusinessSummarySource,
|
||||
ConfidenceState,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:DatasetProfile:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One-to-one profile snapshot for a dataset review session, tracking business summary provenance and completeness.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class DatasetProfile(Base):
|
||||
__tablename__ = "dataset_profiles"
|
||||
|
||||
profile_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String,
|
||||
ForeignKey("dataset_review_sessions.session_id"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
dataset_name = Column(String, nullable=False)
|
||||
schema_name = Column(String, nullable=True)
|
||||
database_name = Column(String, nullable=True)
|
||||
business_summary = Column(Text, nullable=False)
|
||||
business_summary_source = Column(SQLEnum(BusinessSummarySource), nullable=False)
|
||||
description = Column(Text, nullable=True)
|
||||
dataset_type = Column(String, nullable=True)
|
||||
is_sqllab_view = Column(Boolean, nullable=False, default=False)
|
||||
completeness_score = Column(Float, nullable=True)
|
||||
confidence_state = Column(SQLEnum(ConfidenceState), nullable=False)
|
||||
has_blocking_findings = Column(Boolean, nullable=False, default=False)
|
||||
has_warning_findings = Column(Boolean, nullable=False, default=False)
|
||||
manual_summary_locked = Column(Boolean, nullable=False, default=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(
|
||||
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="profile")
|
||||
|
||||
|
||||
# [/DEF:DatasetProfile:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewProfileModels:Module]
|
||||
139
backend/src/models/dataset_review_pkg/_semantic_models.py
Normal file
139
backend/src/models/dataset_review_pkg/_semantic_models.py
Normal file
@@ -0,0 +1,139 @@
|
||||
# [DEF:DatasetReviewSemanticModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Semantic source, field entry, and candidate models for dictionary-driven semantic enrichment.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values.
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Integer,
|
||||
Text,
|
||||
Float,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
SemanticSourceType,
|
||||
TrustLevel,
|
||||
SemanticSourceStatus,
|
||||
FieldKind,
|
||||
FieldProvenance,
|
||||
CandidateMatchType,
|
||||
CandidateStatus,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:SemanticSource:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: Registered semantic enrichment source with trust level and application status.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class SemanticSource(Base):
|
||||
__tablename__ = "semantic_sources"
|
||||
|
||||
source_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
source_type = Column(SQLEnum(SemanticSourceType), nullable=False)
|
||||
source_ref = Column(String, nullable=False)
|
||||
source_version = Column(String, nullable=False)
|
||||
display_name = Column(String, nullable=False)
|
||||
trust_level = Column(SQLEnum(TrustLevel), nullable=False)
|
||||
schema_overlap_score = Column(Float, nullable=True)
|
||||
status = Column(
|
||||
SQLEnum(SemanticSourceStatus),
|
||||
nullable=False,
|
||||
default=SemanticSourceStatus.AVAILABLE,
|
||||
)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="semantic_sources")
|
||||
|
||||
|
||||
# [/DEF:SemanticSource:Class]
|
||||
|
||||
|
||||
# [DEF:SemanticFieldEntry:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Per-field semantic metadata entry with provenance tracking, lock state, and candidate set.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
# @RELATION: DEPENDS_ON -> [SemanticCandidate]
|
||||
# @INVARIANT: Locked fields preserve their active value regardless of later candidate proposals.
|
||||
class SemanticFieldEntry(Base):
|
||||
__tablename__ = "semantic_field_entries"
|
||||
|
||||
field_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
field_name = Column(String, nullable=False)
|
||||
field_kind = Column(SQLEnum(FieldKind), nullable=False)
|
||||
verbose_name = Column(String, nullable=True)
|
||||
description = Column(Text, nullable=True)
|
||||
display_format = Column(String, nullable=True)
|
||||
provenance = Column(
|
||||
SQLEnum(FieldProvenance), nullable=False, default=FieldProvenance.UNRESOLVED
|
||||
)
|
||||
source_id = Column(String, nullable=True)
|
||||
source_version = Column(String, nullable=True)
|
||||
confidence_rank = Column(Integer, nullable=True)
|
||||
is_locked = Column(Boolean, nullable=False, default=False)
|
||||
has_conflict = Column(Boolean, nullable=False, default=False)
|
||||
needs_review = Column(Boolean, nullable=False, default=True)
|
||||
last_changed_by = Column(String, nullable=False)
|
||||
user_feedback = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(
|
||||
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="semantic_fields")
|
||||
candidates = relationship(
|
||||
"SemanticCandidate", back_populates="field", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
# [/DEF:SemanticFieldEntry:Class]
|
||||
|
||||
|
||||
# [DEF:SemanticCandidate:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: One proposed semantic value for a field entry, ranked by match type and confidence.
|
||||
# @RELATION: DEPENDS_ON -> [SemanticFieldEntry]
|
||||
class SemanticCandidate(Base):
|
||||
__tablename__ = "semantic_candidates"
|
||||
|
||||
candidate_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
field_id = Column(
|
||||
String, ForeignKey("semantic_field_entries.field_id"), nullable=False
|
||||
)
|
||||
source_id = Column(String, nullable=True)
|
||||
candidate_rank = Column(Integer, nullable=False)
|
||||
match_type = Column(SQLEnum(CandidateMatchType), nullable=False)
|
||||
confidence_score = Column(Float, nullable=False)
|
||||
proposed_verbose_name = Column(String, nullable=True)
|
||||
proposed_description = Column(Text, nullable=True)
|
||||
proposed_display_format = Column(String, nullable=True)
|
||||
status = Column(
|
||||
SQLEnum(CandidateStatus), nullable=False, default=CandidateStatus.PROPOSED
|
||||
)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
field = relationship("SemanticFieldEntry", back_populates="candidates")
|
||||
|
||||
|
||||
# [/DEF:SemanticCandidate:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewSemanticModels:Module]
|
||||
156
backend/src/models/dataset_review_pkg/_session_models.py
Normal file
156
backend/src/models/dataset_review_pkg/_session_models.py
Normal file
@@ -0,0 +1,156 @@
|
||||
# [DEF:DatasetReviewSessionModels:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Session aggregate root and collaborator models for dataset review orchestration.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
|
||||
# @RELATION: DEPENDS_ON -> [MappingModels]
|
||||
# @INVARIANT: Session and profile entities are strictly scoped to an authenticated user.
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
String,
|
||||
Integer,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Enum as SQLEnum,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from src.models.mapping import Base
|
||||
from src.models.dataset_review_pkg._enums import (
|
||||
SessionStatus,
|
||||
SessionPhase,
|
||||
ReadinessState,
|
||||
RecommendedAction,
|
||||
SessionCollaboratorRole,
|
||||
)
|
||||
|
||||
|
||||
# [DEF:SessionCollaborator:Class]
|
||||
# @COMPLEXITY: 2
|
||||
# @PURPOSE: RBAC collaborator record linking a user to a dataset review session with a specific role.
|
||||
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
|
||||
class SessionCollaborator(Base):
|
||||
__tablename__ = "session_collaborators"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
session_id = Column(
|
||||
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
|
||||
)
|
||||
user_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
role = Column(SQLEnum(SessionCollaboratorRole), nullable=False)
|
||||
added_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
session = relationship("DatasetReviewSession", back_populates="collaborators")
|
||||
user = relationship("User")
|
||||
|
||||
|
||||
# [/DEF:SessionCollaborator:Class]
|
||||
|
||||
|
||||
# [DEF:DatasetReviewSession:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Aggregate root for the dataset review lifecycle, owning all child entities and driving readiness transitions.
|
||||
# @RELATION: DEPENDS_ON -> [SessionCollaborator]
|
||||
# @RELATION: DEPENDS_ON -> [DatasetProfile]
|
||||
# @RELATION: DEPENDS_ON -> [ValidationFinding]
|
||||
# @RELATION: DEPENDS_ON -> [SemanticSource]
|
||||
# @RELATION: DEPENDS_ON -> [SemanticFieldEntry]
|
||||
# @RELATION: DEPENDS_ON -> [ImportedFilter]
|
||||
# @RELATION: DEPENDS_ON -> [TemplateVariable]
|
||||
# @RELATION: DEPENDS_ON -> [ExecutionMapping]
|
||||
# @RELATION: DEPENDS_ON -> [ClarificationSession]
|
||||
# @RELATION: DEPENDS_ON -> [CompiledPreview]
|
||||
# @RELATION: DEPENDS_ON -> [DatasetRunContext]
|
||||
# @RELATION: DEPENDS_ON -> [ExportArtifact]
|
||||
# @RELATION: DEPENDS_ON -> [SessionEvent]
|
||||
# @INVARIANT: Optimistic-lock version column prevents lost-update races on concurrent mutations.
|
||||
class DatasetReviewSession(Base):
|
||||
__tablename__ = "dataset_review_sessions"
|
||||
|
||||
session_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
user_id = Column(String, ForeignKey("users.id"), nullable=False)
|
||||
environment_id = Column(String, ForeignKey("environments.id"), nullable=False)
|
||||
source_kind = Column(String, nullable=False)
|
||||
source_input = Column(String, nullable=False)
|
||||
dataset_ref = Column(String, nullable=False)
|
||||
dataset_id = Column(Integer, nullable=True)
|
||||
dashboard_id = Column(Integer, nullable=True)
|
||||
readiness_state = Column(
|
||||
SQLEnum(ReadinessState), nullable=False, default=ReadinessState.EMPTY
|
||||
)
|
||||
recommended_action = Column(
|
||||
SQLEnum(RecommendedAction),
|
||||
nullable=False,
|
||||
default=RecommendedAction.IMPORT_FROM_SUPERSET,
|
||||
)
|
||||
version = Column(Integer, nullable=False, default=0)
|
||||
__mapper_args__ = {"version_id_col": version, "version_id_generator": False}
|
||||
status = Column(
|
||||
SQLEnum(SessionStatus), nullable=False, default=SessionStatus.ACTIVE
|
||||
)
|
||||
current_phase = Column(
|
||||
SQLEnum(SessionPhase), nullable=False, default=SessionPhase.INTAKE
|
||||
)
|
||||
active_task_id = Column(String, nullable=True)
|
||||
last_preview_id = Column(String, nullable=True)
|
||||
last_run_context_id = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(
|
||||
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
|
||||
)
|
||||
last_activity_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
closed_at = Column(DateTime, nullable=True)
|
||||
|
||||
owner = relationship("User")
|
||||
collaborators = relationship(
|
||||
"SessionCollaborator", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
profile = relationship(
|
||||
"DatasetProfile",
|
||||
back_populates="session",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
findings = relationship(
|
||||
"ValidationFinding", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
semantic_sources = relationship(
|
||||
"SemanticSource", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
semantic_fields = relationship(
|
||||
"SemanticFieldEntry", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
imported_filters = relationship(
|
||||
"ImportedFilter", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
template_variables = relationship(
|
||||
"TemplateVariable", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
execution_mappings = relationship(
|
||||
"ExecutionMapping", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
clarification_sessions = relationship(
|
||||
"ClarificationSession", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
previews = relationship(
|
||||
"CompiledPreview", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
run_contexts = relationship(
|
||||
"DatasetRunContext", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
export_artifacts = relationship(
|
||||
"ExportArtifact", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
events = relationship(
|
||||
"SessionEvent", back_populates="session", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewSession:Class]
|
||||
|
||||
|
||||
# [/DEF:DatasetReviewSessionModels:Module]
|
||||
Reference in New Issue
Block a user