This commit is contained in:
2026-04-24 17:10:02 +03:00
parent b7a9ef71b2
commit 0b227a307c
34 changed files with 5220 additions and 5446 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,122 @@
# [DEF:DatasetReviewModels:Module]
# @COMPLEXITY: 3
# @SEMANTICS: dataset_review, session, profile, findings, semantics, clarification, execution, sqlalchemy
# @PURPOSE: Re-export all dataset review domain models from decomposed sub-modules for backward-compatible imports.
# @LAYER: Domain
from src.models.dataset_review_pkg._enums import (
SessionStatus,
SessionPhase,
ReadinessState,
RecommendedAction,
SessionCollaboratorRole,
BusinessSummarySource,
ConfidenceState,
FindingArea,
FindingSeverity,
ResolutionState,
SemanticSourceType,
TrustLevel,
SemanticSourceStatus,
FieldKind,
FieldProvenance,
CandidateMatchType,
CandidateStatus,
FilterSource,
FilterConfidenceState,
FilterRecoveryStatus,
VariableKind,
MappingStatus,
MappingMethod,
MappingWarningLevel,
ApprovalState,
ClarificationStatus,
QuestionState,
AnswerKind,
PreviewStatus,
LaunchStatus,
ArtifactType,
ArtifactFormat,
)
from src.models.dataset_review_pkg._session_models import (
SessionCollaborator,
DatasetReviewSession,
)
from src.models.dataset_review_pkg._profile_models import DatasetProfile
from src.models.dataset_review_pkg._finding_models import ValidationFinding
from src.models.dataset_review_pkg._semantic_models import (
SemanticSource,
SemanticFieldEntry,
SemanticCandidate,
)
from src.models.dataset_review_pkg._filter_models import (
ImportedFilter,
TemplateVariable,
)
from src.models.dataset_review_pkg._mapping_models import ExecutionMapping
from src.models.dataset_review_pkg._clarification_models import (
ClarificationSession,
ClarificationQuestion,
ClarificationOption,
ClarificationAnswer,
)
from src.models.dataset_review_pkg._execution_models import (
CompiledPreview,
DatasetRunContext,
SessionEvent,
ExportArtifact,
)
__all__ = [
"SessionStatus",
"SessionPhase",
"ReadinessState",
"RecommendedAction",
"SessionCollaboratorRole",
"BusinessSummarySource",
"ConfidenceState",
"FindingArea",
"FindingSeverity",
"ResolutionState",
"SemanticSourceType",
"TrustLevel",
"SemanticSourceStatus",
"FieldKind",
"FieldProvenance",
"CandidateMatchType",
"CandidateStatus",
"FilterSource",
"FilterConfidenceState",
"FilterRecoveryStatus",
"VariableKind",
"MappingStatus",
"MappingMethod",
"MappingWarningLevel",
"ApprovalState",
"ClarificationStatus",
"QuestionState",
"AnswerKind",
"PreviewStatus",
"LaunchStatus",
"ArtifactType",
"ArtifactFormat",
"SessionCollaborator",
"DatasetReviewSession",
"DatasetProfile",
"ValidationFinding",
"SemanticSource",
"SemanticFieldEntry",
"SemanticCandidate",
"ImportedFilter",
"TemplateVariable",
"ExecutionMapping",
"ClarificationSession",
"ClarificationQuestion",
"ClarificationOption",
"ClarificationAnswer",
"CompiledPreview",
"DatasetRunContext",
"SessionEvent",
"ExportArtifact",
]
# [/DEF:DatasetReviewModels:Module]

View File

@@ -0,0 +1,125 @@
# [DEF:DatasetReviewClarificationModels:Module]
# @COMPLEXITY: 3
# @PURPOSE: Clarification session, question, option, and answer models for guided review flow.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
# @INVARIANT: Only one active clarification question may exist at a time per session.
import uuid
from datetime import datetime
from sqlalchemy import (
Boolean,
Column,
String,
Integer,
Text,
DateTime,
ForeignKey,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
ClarificationStatus,
QuestionState,
AnswerKind,
)
# [DEF:ClarificationSession:Class]
# @COMPLEXITY: 2
# @PURPOSE: One clarification session aggregate owning questions and tracking resolution progress.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class ClarificationSession(Base):
__tablename__ = "clarification_sessions"
clarification_session_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(String, ForeignKey("dataset_review_sessions.session_id"), nullable=False)
status = Column(SQLEnum(ClarificationStatus), nullable=False, default=ClarificationStatus.PENDING)
current_question_id = Column(String, nullable=True)
resolved_count = Column(Integer, nullable=False, default=0)
remaining_count = Column(Integer, nullable=False, default=0)
summary_delta = Column(Text, nullable=True)
started_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
completed_at = Column(DateTime, nullable=True)
session = relationship("DatasetReviewSession", back_populates="clarification_sessions")
questions = relationship("ClarificationQuestion", back_populates="clarification_session", cascade="all, delete-orphan")
# [/DEF:ClarificationSession:Class]
# [DEF:ClarificationQuestion:Class]
# @COMPLEXITY: 2
# @PURPOSE: One clarification question with priority ordering, options, and state machine.
# @RELATION: DEPENDS_ON -> [ClarificationSession]
class ClarificationQuestion(Base):
__tablename__ = "clarification_questions"
question_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
clarification_session_id = Column(String, ForeignKey("clarification_sessions.clarification_session_id"), nullable=False)
topic_ref = Column(String, nullable=False)
question_text = Column(Text, nullable=False)
why_it_matters = Column(Text, nullable=False)
current_guess = Column(Text, nullable=True)
priority = Column(Integer, nullable=False, default=0)
state = Column(SQLEnum(QuestionState), nullable=False, default=QuestionState.OPEN)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
clarification_session = relationship("ClarificationSession", back_populates="questions")
options = relationship("ClarificationOption", back_populates="question", cascade="all, delete-orphan")
answer = relationship("ClarificationAnswer", back_populates="question", uselist=False, cascade="all, delete-orphan")
# [/DEF:ClarificationQuestion:Class]
# [DEF:ClarificationOption:Class]
# @COMPLEXITY: 1
# @PURPOSE: One selectable option for a clarification question with recommendation flag.
# @RELATION: DEPENDS_ON -> [ClarificationQuestion]
class ClarificationOption(Base):
__tablename__ = "clarification_options"
option_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
question_id = Column(String, ForeignKey("clarification_questions.question_id"), nullable=False)
label = Column(String, nullable=False)
value = Column(String, nullable=False)
is_recommended = Column(Boolean, nullable=False, default=False)
display_order = Column(Integer, nullable=False, default=0)
question = relationship("ClarificationQuestion", back_populates="options")
# [/DEF:ClarificationOption:Class]
# [DEF:ClarificationAnswer:Class]
# @COMPLEXITY: 2
# @PURPOSE: One persisted clarification answer with impact summary and feedback tracking.
# @RELATION: DEPENDS_ON -> [ClarificationQuestion]
class ClarificationAnswer(Base):
__tablename__ = "clarification_answers"
answer_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
question_id = Column(String, ForeignKey("clarification_questions.question_id"), nullable=False, unique=True)
answer_kind = Column(SQLEnum(AnswerKind), nullable=False)
answer_value = Column(Text, nullable=True)
answered_by_user_id = Column(String, nullable=False)
impact_summary = Column(Text, nullable=True)
user_feedback = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
question = relationship("ClarificationQuestion", back_populates="answer")
# [/DEF:ClarificationAnswer:Class]
# [/DEF:DatasetReviewClarificationModels:Module]

View File

@@ -0,0 +1,463 @@
# [DEF:DatasetReviewEnums:Module]
# @COMPLEXITY: 2
# @PURPOSE: All enumeration types for the dataset review domain, grouped for stable cross-module reuse.
# @LAYER: Domain
# @INVARIANT: Enum values are string-based for JSON serialization compatibility.
import enum
# [DEF:SessionStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status of a dataset review session.
class SessionStatus(str, enum.Enum):
ACTIVE = "active"
PAUSED = "paused"
COMPLETED = "completed"
ARCHIVED = "archived"
CANCELLED = "cancelled"
# [/DEF:SessionStatus:Class]
# [DEF:SessionPhase:Class]
# @COMPLEXITY: 1
# @PURPOSE: Ordered phase progression for dataset review orchestration.
class SessionPhase(str, enum.Enum):
INTAKE = "intake"
RECOVERY = "recovery"
REVIEW = "review"
SEMANTIC_REVIEW = "semantic_review"
CLARIFICATION = "clarification"
MAPPING_REVIEW = "mapping_review"
PREVIEW = "preview"
LAUNCH = "launch"
POST_RUN = "post_run"
# [/DEF:SessionPhase:Class]
# [DEF:ReadinessState:Class]
# @COMPLEXITY: 1
# @PURPOSE: Granular readiness indicator driving the recommended-action UX flow.
class ReadinessState(str, enum.Enum):
EMPTY = "empty"
IMPORTING = "importing"
REVIEW_READY = "review_ready"
SEMANTIC_SOURCE_REVIEW_NEEDED = "semantic_source_review_needed"
CLARIFICATION_NEEDED = "clarification_needed"
CLARIFICATION_ACTIVE = "clarification_active"
MAPPING_REVIEW_NEEDED = "mapping_review_needed"
COMPILED_PREVIEW_READY = "compiled_preview_ready"
PARTIALLY_READY = "partially_ready"
RUN_READY = "run_ready"
RUN_IN_PROGRESS = "run_in_progress"
COMPLETED = "completed"
RECOVERY_REQUIRED = "recovery_required"
# [/DEF:ReadinessState:Class]
# [DEF:RecommendedAction:Class]
# @COMPLEXITY: 1
# @PURPOSE: Next-action guidance derived from the current readiness state.
class RecommendedAction(str, enum.Enum):
IMPORT_FROM_SUPERSET = "import_from_superset"
REVIEW_DOCUMENTATION = "review_documentation"
APPLY_SEMANTIC_SOURCE = "apply_semantic_source"
START_CLARIFICATION = "start_clarification"
ANSWER_NEXT_QUESTION = "answer_next_question"
APPROVE_MAPPING = "approve_mapping"
GENERATE_SQL_PREVIEW = "generate_sql_preview"
COMPLETE_REQUIRED_VALUES = "complete_required_values"
LAUNCH_DATASET = "launch_dataset"
RESUME_SESSION = "resume_session"
EXPORT_OUTPUTS = "export_outputs"
# [/DEF:RecommendedAction:Class]
# [DEF:SessionCollaboratorRole:Class]
# @COMPLEXITY: 1
# @PURPOSE: RBAC role for session collaborators.
class SessionCollaboratorRole(str, enum.Enum):
VIEWER = "viewer"
REVIEWER = "reviewer"
APPROVER = "approver"
# [/DEF:SessionCollaboratorRole:Class]
# [DEF:BusinessSummarySource:Class]
# @COMPLEXITY: 1
# @PURPOSE: Provenance of the dataset business summary text.
class BusinessSummarySource(str, enum.Enum):
CONFIRMED = "confirmed"
IMPORTED = "imported"
INFERRED = "inferred"
AI_DRAFT = "ai_draft"
MANUAL_OVERRIDE = "manual_override"
# [/DEF:BusinessSummarySource:Class]
# [DEF:ConfidenceState:Class]
# @COMPLEXITY: 1
# @PURPOSE: Confidence level for dataset profile completeness.
class ConfidenceState(str, enum.Enum):
CONFIRMED = "confirmed"
MOSTLY_CONFIRMED = "mostly_confirmed"
MIXED = "mixed"
LOW_CONFIDENCE = "low_confidence"
UNRESOLVED = "unresolved"
# [/DEF:ConfidenceState:Class]
# [DEF:FindingArea:Class]
# @COMPLEXITY: 1
# @PURPOSE: Domain area classification for validation findings.
class FindingArea(str, enum.Enum):
SOURCE_INTAKE = "source_intake"
DATASET_PROFILE = "dataset_profile"
SEMANTIC_ENRICHMENT = "semantic_enrichment"
CLARIFICATION = "clarification"
FILTER_RECOVERY = "filter_recovery"
TEMPLATE_MAPPING = "template_mapping"
COMPILED_PREVIEW = "compiled_preview"
LAUNCH = "launch"
AUDIT = "audit"
# [/DEF:FindingArea:Class]
# [DEF:FindingSeverity:Class]
# @COMPLEXITY: 1
# @PURPOSE: Severity classification for validation findings.
class FindingSeverity(str, enum.Enum):
BLOCKING = "blocking"
WARNING = "warning"
INFORMATIONAL = "informational"
# [/DEF:FindingSeverity:Class]
# [DEF:ResolutionState:Class]
# @COMPLEXITY: 1
# @PURPOSE: Resolution status for validation findings and clarification items.
class ResolutionState(str, enum.Enum):
OPEN = "open"
RESOLVED = "resolved"
APPROVED = "approved"
SKIPPED = "skipped"
DEFERRED = "deferred"
EXPERT_REVIEW = "expert_review"
# [/DEF:ResolutionState:Class]
# [DEF:SemanticSourceType:Class]
# @COMPLEXITY: 1
# @PURPOSE: Classification of semantic enrichment source origins.
class SemanticSourceType(str, enum.Enum):
UPLOADED_FILE = "uploaded_file"
CONNECTED_DICTIONARY = "connected_dictionary"
REFERENCE_DATASET = "reference_dataset"
NEIGHBOR_DATASET = "neighbor_dataset"
AI_GENERATED = "ai_generated"
# [/DEF:SemanticSourceType:Class]
# [DEF:TrustLevel:Class]
# @COMPLEXITY: 1
# @PURPOSE: Trust classification for semantic source reliability.
class TrustLevel(str, enum.Enum):
TRUSTED = "trusted"
RECOMMENDED = "recommended"
CANDIDATE = "candidate"
GENERATED = "generated"
# [/DEF:TrustLevel:Class]
# [DEF:SemanticSourceStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status for semantic source application.
class SemanticSourceStatus(str, enum.Enum):
AVAILABLE = "available"
SELECTED = "selected"
APPLIED = "applied"
REJECTED = "rejected"
PARTIAL = "partial"
FAILED = "failed"
# [/DEF:SemanticSourceStatus:Class]
# [DEF:FieldKind:Class]
# @COMPLEXITY: 1
# @PURPOSE: Kind classification for semantic field entries.
class FieldKind(str, enum.Enum):
COLUMN = "column"
METRIC = "metric"
FILTER_DIMENSION = "filter_dimension"
PARAMETER = "parameter"
# [/DEF:FieldKind:Class]
# [DEF:FieldProvenance:Class]
# @COMPLEXITY: 1
# @PURPOSE: Provenance tracking for semantic field value origin.
class FieldProvenance(str, enum.Enum):
DICTIONARY_EXACT = "dictionary_exact"
REFERENCE_IMPORTED = "reference_imported"
FUZZY_INFERRED = "fuzzy_inferred"
AI_GENERATED = "ai_generated"
MANUAL_OVERRIDE = "manual_override"
UNRESOLVED = "unresolved"
# [/DEF:FieldProvenance:Class]
# [DEF:CandidateMatchType:Class]
# @COMPLEXITY: 1
# @PURPOSE: Match type classification for semantic candidates.
class CandidateMatchType(str, enum.Enum):
EXACT = "exact"
REFERENCE = "reference"
FUZZY = "fuzzy"
GENERATED = "generated"
# [/DEF:CandidateMatchType:Class]
# [DEF:CandidateStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status for semantic candidate proposals.
class CandidateStatus(str, enum.Enum):
PROPOSED = "proposed"
ACCEPTED = "accepted"
REJECTED = "rejected"
SUPERSEDED = "superseded"
# [/DEF:CandidateStatus:Class]
# [DEF:FilterSource:Class]
# @COMPLEXITY: 1
# @PURPOSE: Origin classification for imported filters.
class FilterSource(str, enum.Enum):
SUPERSET_NATIVE = "superset_native"
SUPERSET_URL = "superset_url"
SUPERSET_PERMALINK = "superset_permalink"
SUPERSET_NATIVE_FILTERS_KEY = "superset_native_filters_key"
MANUAL = "manual"
INFERRED = "inferred"
# [/DEF:FilterSource:Class]
# [DEF:FilterConfidenceState:Class]
# @COMPLEXITY: 1
# @PURPOSE: Confidence classification for imported filter values.
class FilterConfidenceState(str, enum.Enum):
CONFIRMED = "confirmed"
IMPORTED = "imported"
INFERRED = "inferred"
AI_DRAFT = "ai_draft"
UNRESOLVED = "unresolved"
# [/DEF:FilterConfidenceState:Class]
# [DEF:FilterRecoveryStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Recovery quality status for imported filters.
class FilterRecoveryStatus(str, enum.Enum):
RECOVERED = "recovered"
PARTIAL = "partial"
MISSING = "missing"
CONFLICTED = "conflicted"
# [/DEF:FilterRecoveryStatus:Class]
# [DEF:VariableKind:Class]
# @COMPLEXITY: 1
# @PURPOSE: Kind classification for template variables.
class VariableKind(str, enum.Enum):
NATIVE_FILTER = "native_filter"
PARAMETER = "parameter"
DERIVED = "derived"
UNKNOWN = "unknown"
# [/DEF:VariableKind:Class]
# [DEF:MappingStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status for template variable mapping.
class MappingStatus(str, enum.Enum):
UNMAPPED = "unmapped"
PROPOSED = "proposed"
APPROVED = "approved"
OVERRIDDEN = "overridden"
INVALID = "invalid"
# [/DEF:MappingStatus:Class]
# [DEF:MappingMethod:Class]
# @COMPLEXITY: 1
# @PURPOSE: Method classification for execution mapping creation.
class MappingMethod(str, enum.Enum):
DIRECT_MATCH = "direct_match"
HEURISTIC_MATCH = "heuristic_match"
SEMANTIC_MATCH = "semantic_match"
MANUAL_OVERRIDE = "manual_override"
# [/DEF:MappingMethod:Class]
# [DEF:MappingWarningLevel:Class]
# @COMPLEXITY: 1
# @PURPOSE: Warning severity for execution mapping quality indicators.
class MappingWarningLevel(str, enum.Enum):
LOW = "low"
MEDIUM = "medium"
HIGH = "high"
# [/DEF:MappingWarningLevel:Class]
# [DEF:ApprovalState:Class]
# @COMPLEXITY: 1
# @PURPOSE: Approval lifecycle for execution mapping gate checks.
class ApprovalState(str, enum.Enum):
PENDING = "pending"
APPROVED = "approved"
REJECTED = "rejected"
NOT_REQUIRED = "not_required"
# [/DEF:ApprovalState:Class]
# [DEF:ClarificationStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status for clarification sessions.
class ClarificationStatus(str, enum.Enum):
PENDING = "pending"
ACTIVE = "active"
PAUSED = "paused"
COMPLETED = "completed"
CANCELLED = "cancelled"
# [/DEF:ClarificationStatus:Class]
# [DEF:QuestionState:Class]
# @COMPLEXITY: 1
# @PURPOSE: State machine for individual clarification questions.
class QuestionState(str, enum.Enum):
OPEN = "open"
ANSWERED = "answered"
SKIPPED = "skipped"
EXPERT_REVIEW = "expert_review"
SUPERSEDED = "superseded"
# [/DEF:QuestionState:Class]
# [DEF:AnswerKind:Class]
# @COMPLEXITY: 1
# @PURPOSE: Classification of clarification answer types.
class AnswerKind(str, enum.Enum):
SELECTED = "selected"
CUSTOM = "custom"
SKIPPED = "skipped"
EXPERT_REVIEW = "expert_review"
# [/DEF:AnswerKind:Class]
# [DEF:PreviewStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Lifecycle status for compiled SQL previews.
class PreviewStatus(str, enum.Enum):
PENDING = "pending"
READY = "ready"
FAILED = "failed"
STALE = "stale"
# [/DEF:PreviewStatus:Class]
# [DEF:LaunchStatus:Class]
# @COMPLEXITY: 1
# @PURPOSE: Outcome status for dataset launch handoff.
class LaunchStatus(str, enum.Enum):
STARTED = "started"
SUCCESS = "success"
FAILED = "failed"
# [/DEF:LaunchStatus:Class]
# [DEF:ArtifactType:Class]
# @COMPLEXITY: 1
# @PURPOSE: Type classification for export artifacts.
class ArtifactType(str, enum.Enum):
DOCUMENTATION = "documentation"
VALIDATION_REPORT = "validation_report"
RUN_SUMMARY = "run_summary"
# [/DEF:ArtifactType:Class]
# [DEF:ArtifactFormat:Class]
# @COMPLEXITY: 1
# @PURPOSE: Format classification for export artifact output.
class ArtifactFormat(str, enum.Enum):
JSON = "json"
MARKDOWN = "markdown"
CSV = "csv"
PDF = "pdf"
# [/DEF:ArtifactFormat:Class]
# [/DEF:DatasetReviewEnums:Module]

View File

@@ -0,0 +1,140 @@
# [DEF:DatasetReviewExecutionModels:Module]
# @COMPLEXITY: 3
# @PURPOSE: Compiled preview, run context, session event, and export artifact models for execution and audit.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Text,
DateTime,
ForeignKey,
JSON,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
PreviewStatus,
LaunchStatus,
ArtifactType,
ArtifactFormat,
)
# [DEF:CompiledPreview:Class]
# @COMPLEXITY: 2
# @PURPOSE: One compiled SQL preview snapshot with fingerprint for staleness detection.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class CompiledPreview(Base):
__tablename__ = "compiled_previews"
preview_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
preview_status = Column(
SQLEnum(PreviewStatus), nullable=False, default=PreviewStatus.PENDING
)
compiled_sql = Column(Text, nullable=True)
preview_fingerprint = Column(String, nullable=False)
compiled_by = Column(String, nullable=False, default="superset")
error_code = Column(String, nullable=True)
error_details = Column(Text, nullable=True)
compiled_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="previews")
# [/DEF:CompiledPreview:Class]
# [DEF:DatasetRunContext:Class]
# @COMPLEXITY: 2
# @PURPOSE: Immutable launch audit snapshot capturing effective filters, template params, and approval state at launch time.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class DatasetRunContext(Base):
__tablename__ = "dataset_run_contexts"
run_context_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
dataset_ref = Column(String, nullable=False)
environment_id = Column(String, nullable=False)
preview_id = Column(String, nullable=False)
sql_lab_session_ref = Column(String, nullable=False)
effective_filters = Column(JSON, nullable=False)
template_params = Column(JSON, nullable=False)
approved_mapping_ids = Column(JSON, nullable=False)
semantic_decision_refs = Column(JSON, nullable=False)
open_warning_refs = Column(JSON, nullable=False)
launch_status = Column(SQLEnum(LaunchStatus), nullable=False)
launch_error = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="run_contexts")
# [/DEF:DatasetRunContext:Class]
# [DEF:SessionEvent:Class]
# @COMPLEXITY: 2
# @PURPOSE: One persisted audit event for dataset review session mutations.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class SessionEvent(Base):
__tablename__ = "session_events"
session_event_id = Column(
String, primary_key=True, default=lambda: str(uuid.uuid4())
)
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
actor_user_id = Column(String, ForeignKey("users.id"), nullable=False)
event_type = Column(String, nullable=False)
event_summary = Column(Text, nullable=False)
current_phase = Column(String, nullable=True)
readiness_state = Column(String, nullable=True)
event_details = Column(JSON, nullable=False, default=dict)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="events")
actor = relationship("User")
# [/DEF:SessionEvent:Class]
# [DEF:ExportArtifact:Class]
# @COMPLEXITY: 2
# @PURPOSE: One persisted export artifact reference for documentation and validation outputs.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class ExportArtifact(Base):
__tablename__ = "export_artifacts"
artifact_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
artifact_type = Column(SQLEnum(ArtifactType), nullable=False)
format = Column(SQLEnum(ArtifactFormat), nullable=False)
storage_ref = Column(String, nullable=False)
created_by_user_id = Column(String, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="export_artifacts")
# [/DEF:ExportArtifact:Class]
# [/DEF:DatasetReviewExecutionModels:Module]

View File

@@ -0,0 +1,95 @@
# [DEF:DatasetReviewFilterModels:Module]
# @COMPLEXITY: 3
# @PURPOSE: Imported filter and template variable models for Superset context recovery and execution mapping.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Text,
Boolean,
DateTime,
ForeignKey,
JSON,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
FilterSource,
FilterConfidenceState,
FilterRecoveryStatus,
VariableKind,
MappingStatus,
)
# [DEF:ImportedFilter:Class]
# @COMPLEXITY: 2
# @PURPOSE: Recovered Superset filter with confidence and recovery status tracking.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class ImportedFilter(Base):
__tablename__ = "imported_filters"
filter_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
filter_name = Column(String, nullable=False)
display_name = Column(String, nullable=True)
raw_value = Column(JSON, nullable=False)
raw_value_masked = Column(Boolean, nullable=False, default=False)
normalized_value = Column(JSON, nullable=True)
source = Column(SQLEnum(FilterSource), nullable=False)
confidence_state = Column(SQLEnum(FilterConfidenceState), nullable=False)
requires_confirmation = Column(Boolean, nullable=False, default=False)
recovery_status = Column(SQLEnum(FilterRecoveryStatus), nullable=False)
notes = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
session = relationship("DatasetReviewSession", back_populates="imported_filters")
# [/DEF:ImportedFilter:Class]
# [DEF:TemplateVariable:Class]
# @COMPLEXITY: 2
# @PURPOSE: Discovered template variable from dataset SQL with mapping status tracking.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class TemplateVariable(Base):
__tablename__ = "template_variables"
variable_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
variable_name = Column(String, nullable=False)
expression_source = Column(Text, nullable=False)
variable_kind = Column(SQLEnum(VariableKind), nullable=False)
is_required = Column(Boolean, nullable=False, default=True)
default_value = Column(JSON, nullable=True)
mapping_status = Column(
SQLEnum(MappingStatus), nullable=False, default=MappingStatus.UNMAPPED
)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
session = relationship("DatasetReviewSession", back_populates="template_variables")
# [/DEF:TemplateVariable:Class]
# [/DEF:DatasetReviewFilterModels:Module]

View File

@@ -0,0 +1,59 @@
# [DEF:DatasetReviewFindingModels:Module]
# @COMPLEXITY: 2
# @PURPOSE: Validation finding model for tracking blocking, warning, and informational issues during review.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Text,
DateTime,
ForeignKey,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
FindingArea,
FindingSeverity,
ResolutionState,
)
# [DEF:ValidationFinding:Class]
# @COMPLEXITY: 2
# @PURPOSE: Structured finding record for dataset review validation issues with resolution tracking.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class ValidationFinding(Base):
__tablename__ = "validation_findings"
finding_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
area = Column(SQLEnum(FindingArea), nullable=False)
severity = Column(SQLEnum(FindingSeverity), nullable=False)
code = Column(String, nullable=False)
title = Column(String, nullable=False)
message = Column(Text, nullable=False)
resolution_state = Column(
SQLEnum(ResolutionState), nullable=False, default=ResolutionState.OPEN
)
resolution_note = Column(Text, nullable=True)
caused_by_ref = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
resolved_at = Column(DateTime, nullable=True)
session = relationship("DatasetReviewSession", back_populates="findings")
# [/DEF:ValidationFinding:Class]
# [/DEF:DatasetReviewFindingModels:Module]

View File

@@ -0,0 +1,61 @@
# [DEF:DatasetReviewMappingModels:Module]
# @COMPLEXITY: 2
# @PURPOSE: Execution mapping model linking imported filters to template variables with approval gates.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Text,
Boolean,
DateTime,
ForeignKey,
JSON,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
MappingMethod,
MappingWarningLevel,
ApprovalState,
)
# [DEF:ExecutionMapping:Class]
# @COMPLEXITY: 2
# @PURPOSE: One filter-to-variable mapping with approval gate, effective value, and transformation metadata.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
# @INVARIANT: Explicit approval is required before launch when requires_explicit_approval is true.
class ExecutionMapping(Base):
__tablename__ = "execution_mappings"
mapping_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(String, ForeignKey("dataset_review_sessions.session_id"), nullable=False)
filter_id = Column(String, nullable=False)
variable_id = Column(String, nullable=False)
mapping_method = Column(SQLEnum(MappingMethod), nullable=False)
raw_input_value = Column(JSON, nullable=False)
effective_value = Column(JSON, nullable=True)
transformation_note = Column(Text, nullable=True)
warning_level = Column(SQLEnum(MappingWarningLevel), nullable=True)
requires_explicit_approval = Column(Boolean, nullable=False, default=False)
approval_state = Column(SQLEnum(ApprovalState), nullable=False, default=ApprovalState.NOT_REQUIRED)
approved_by_user_id = Column(String, nullable=True)
approved_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="execution_mappings")
# [/DEF:ExecutionMapping:Class]
# [/DEF:DatasetReviewMappingModels:Module]

View File

@@ -0,0 +1,68 @@
# [DEF:DatasetReviewProfileModels:Module]
# @COMPLEXITY: 2
# @PURPOSE: Dataset profile model capturing business summary, confidence, and completeness metadata.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Text,
Float,
Boolean,
DateTime,
ForeignKey,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
BusinessSummarySource,
ConfidenceState,
)
# [DEF:DatasetProfile:Class]
# @COMPLEXITY: 2
# @PURPOSE: One-to-one profile snapshot for a dataset review session, tracking business summary provenance and completeness.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class DatasetProfile(Base):
__tablename__ = "dataset_profiles"
profile_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String,
ForeignKey("dataset_review_sessions.session_id"),
nullable=False,
unique=True,
)
dataset_name = Column(String, nullable=False)
schema_name = Column(String, nullable=True)
database_name = Column(String, nullable=True)
business_summary = Column(Text, nullable=False)
business_summary_source = Column(SQLEnum(BusinessSummarySource), nullable=False)
description = Column(Text, nullable=True)
dataset_type = Column(String, nullable=True)
is_sqllab_view = Column(Boolean, nullable=False, default=False)
completeness_score = Column(Float, nullable=True)
confidence_state = Column(SQLEnum(ConfidenceState), nullable=False)
has_blocking_findings = Column(Boolean, nullable=False, default=False)
has_warning_findings = Column(Boolean, nullable=False, default=False)
manual_summary_locked = Column(Boolean, nullable=False, default=False)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
session = relationship("DatasetReviewSession", back_populates="profile")
# [/DEF:DatasetProfile:Class]
# [/DEF:DatasetReviewProfileModels:Module]

View File

@@ -0,0 +1,139 @@
# [DEF:DatasetReviewSemanticModels:Module]
# @COMPLEXITY: 3
# @PURPOSE: Semantic source, field entry, and candidate models for dictionary-driven semantic enrichment.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
# @INVARIANT: Manual overrides are never silently replaced by imported, inferred, or AI-generated values.
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Integer,
Text,
Float,
Boolean,
DateTime,
ForeignKey,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
SemanticSourceType,
TrustLevel,
SemanticSourceStatus,
FieldKind,
FieldProvenance,
CandidateMatchType,
CandidateStatus,
)
# [DEF:SemanticSource:Class]
# @COMPLEXITY: 2
# @PURPOSE: Registered semantic enrichment source with trust level and application status.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class SemanticSource(Base):
__tablename__ = "semantic_sources"
source_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
source_type = Column(SQLEnum(SemanticSourceType), nullable=False)
source_ref = Column(String, nullable=False)
source_version = Column(String, nullable=False)
display_name = Column(String, nullable=False)
trust_level = Column(SQLEnum(TrustLevel), nullable=False)
schema_overlap_score = Column(Float, nullable=True)
status = Column(
SQLEnum(SemanticSourceStatus),
nullable=False,
default=SemanticSourceStatus.AVAILABLE,
)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="semantic_sources")
# [/DEF:SemanticSource:Class]
# [DEF:SemanticFieldEntry:Class]
# @COMPLEXITY: 3
# @PURPOSE: Per-field semantic metadata entry with provenance tracking, lock state, and candidate set.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
# @RELATION: DEPENDS_ON -> [SemanticCandidate]
# @INVARIANT: Locked fields preserve their active value regardless of later candidate proposals.
class SemanticFieldEntry(Base):
__tablename__ = "semantic_field_entries"
field_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
field_name = Column(String, nullable=False)
field_kind = Column(SQLEnum(FieldKind), nullable=False)
verbose_name = Column(String, nullable=True)
description = Column(Text, nullable=True)
display_format = Column(String, nullable=True)
provenance = Column(
SQLEnum(FieldProvenance), nullable=False, default=FieldProvenance.UNRESOLVED
)
source_id = Column(String, nullable=True)
source_version = Column(String, nullable=True)
confidence_rank = Column(Integer, nullable=True)
is_locked = Column(Boolean, nullable=False, default=False)
has_conflict = Column(Boolean, nullable=False, default=False)
needs_review = Column(Boolean, nullable=False, default=True)
last_changed_by = Column(String, nullable=False)
user_feedback = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
session = relationship("DatasetReviewSession", back_populates="semantic_fields")
candidates = relationship(
"SemanticCandidate", back_populates="field", cascade="all, delete-orphan"
)
# [/DEF:SemanticFieldEntry:Class]
# [DEF:SemanticCandidate:Class]
# @COMPLEXITY: 2
# @PURPOSE: One proposed semantic value for a field entry, ranked by match type and confidence.
# @RELATION: DEPENDS_ON -> [SemanticFieldEntry]
class SemanticCandidate(Base):
__tablename__ = "semantic_candidates"
candidate_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
field_id = Column(
String, ForeignKey("semantic_field_entries.field_id"), nullable=False
)
source_id = Column(String, nullable=True)
candidate_rank = Column(Integer, nullable=False)
match_type = Column(SQLEnum(CandidateMatchType), nullable=False)
confidence_score = Column(Float, nullable=False)
proposed_verbose_name = Column(String, nullable=True)
proposed_description = Column(Text, nullable=True)
proposed_display_format = Column(String, nullable=True)
status = Column(
SQLEnum(CandidateStatus), nullable=False, default=CandidateStatus.PROPOSED
)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
field = relationship("SemanticFieldEntry", back_populates="candidates")
# [/DEF:SemanticCandidate:Class]
# [/DEF:DatasetReviewSemanticModels:Module]

View File

@@ -0,0 +1,156 @@
# [DEF:DatasetReviewSessionModels:Module]
# @COMPLEXITY: 3
# @PURPOSE: Session aggregate root and collaborator models for dataset review orchestration.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [DatasetReviewEnums:Module]
# @RELATION: DEPENDS_ON -> [MappingModels]
# @INVARIANT: Session and profile entities are strictly scoped to an authenticated user.
import uuid
from datetime import datetime
from sqlalchemy import (
Column,
String,
Integer,
DateTime,
ForeignKey,
Enum as SQLEnum,
)
from sqlalchemy.orm import relationship
from src.models.mapping import Base
from src.models.dataset_review_pkg._enums import (
SessionStatus,
SessionPhase,
ReadinessState,
RecommendedAction,
SessionCollaboratorRole,
)
# [DEF:SessionCollaborator:Class]
# @COMPLEXITY: 2
# @PURPOSE: RBAC collaborator record linking a user to a dataset review session with a specific role.
# @RELATION: DEPENDS_ON -> [DatasetReviewSession]
class SessionCollaborator(Base):
__tablename__ = "session_collaborators"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
session_id = Column(
String, ForeignKey("dataset_review_sessions.session_id"), nullable=False
)
user_id = Column(String, ForeignKey("users.id"), nullable=False)
role = Column(SQLEnum(SessionCollaboratorRole), nullable=False)
added_at = Column(DateTime, default=datetime.utcnow, nullable=False)
session = relationship("DatasetReviewSession", back_populates="collaborators")
user = relationship("User")
# [/DEF:SessionCollaborator:Class]
# [DEF:DatasetReviewSession:Class]
# @COMPLEXITY: 3
# @PURPOSE: Aggregate root for the dataset review lifecycle, owning all child entities and driving readiness transitions.
# @RELATION: DEPENDS_ON -> [SessionCollaborator]
# @RELATION: DEPENDS_ON -> [DatasetProfile]
# @RELATION: DEPENDS_ON -> [ValidationFinding]
# @RELATION: DEPENDS_ON -> [SemanticSource]
# @RELATION: DEPENDS_ON -> [SemanticFieldEntry]
# @RELATION: DEPENDS_ON -> [ImportedFilter]
# @RELATION: DEPENDS_ON -> [TemplateVariable]
# @RELATION: DEPENDS_ON -> [ExecutionMapping]
# @RELATION: DEPENDS_ON -> [ClarificationSession]
# @RELATION: DEPENDS_ON -> [CompiledPreview]
# @RELATION: DEPENDS_ON -> [DatasetRunContext]
# @RELATION: DEPENDS_ON -> [ExportArtifact]
# @RELATION: DEPENDS_ON -> [SessionEvent]
# @INVARIANT: Optimistic-lock version column prevents lost-update races on concurrent mutations.
class DatasetReviewSession(Base):
__tablename__ = "dataset_review_sessions"
session_id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
user_id = Column(String, ForeignKey("users.id"), nullable=False)
environment_id = Column(String, ForeignKey("environments.id"), nullable=False)
source_kind = Column(String, nullable=False)
source_input = Column(String, nullable=False)
dataset_ref = Column(String, nullable=False)
dataset_id = Column(Integer, nullable=True)
dashboard_id = Column(Integer, nullable=True)
readiness_state = Column(
SQLEnum(ReadinessState), nullable=False, default=ReadinessState.EMPTY
)
recommended_action = Column(
SQLEnum(RecommendedAction),
nullable=False,
default=RecommendedAction.IMPORT_FROM_SUPERSET,
)
version = Column(Integer, nullable=False, default=0)
__mapper_args__ = {"version_id_col": version, "version_id_generator": False}
status = Column(
SQLEnum(SessionStatus), nullable=False, default=SessionStatus.ACTIVE
)
current_phase = Column(
SQLEnum(SessionPhase), nullable=False, default=SessionPhase.INTAKE
)
active_task_id = Column(String, nullable=True)
last_preview_id = Column(String, nullable=True)
last_run_context_id = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
updated_at = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False
)
last_activity_at = Column(DateTime, default=datetime.utcnow, nullable=False)
closed_at = Column(DateTime, nullable=True)
owner = relationship("User")
collaborators = relationship(
"SessionCollaborator", back_populates="session", cascade="all, delete-orphan"
)
profile = relationship(
"DatasetProfile",
back_populates="session",
uselist=False,
cascade="all, delete-orphan",
)
findings = relationship(
"ValidationFinding", back_populates="session", cascade="all, delete-orphan"
)
semantic_sources = relationship(
"SemanticSource", back_populates="session", cascade="all, delete-orphan"
)
semantic_fields = relationship(
"SemanticFieldEntry", back_populates="session", cascade="all, delete-orphan"
)
imported_filters = relationship(
"ImportedFilter", back_populates="session", cascade="all, delete-orphan"
)
template_variables = relationship(
"TemplateVariable", back_populates="session", cascade="all, delete-orphan"
)
execution_mappings = relationship(
"ExecutionMapping", back_populates="session", cascade="all, delete-orphan"
)
clarification_sessions = relationship(
"ClarificationSession", back_populates="session", cascade="all, delete-orphan"
)
previews = relationship(
"CompiledPreview", back_populates="session", cascade="all, delete-orphan"
)
run_contexts = relationship(
"DatasetRunContext", back_populates="session", cascade="all, delete-orphan"
)
export_artifacts = relationship(
"ExportArtifact", back_populates="session", cascade="all, delete-orphan"
)
events = relationship(
"SessionEvent", back_populates="session", cascade="all, delete-orphan"
)
# [/DEF:DatasetReviewSession:Class]
# [/DEF:DatasetReviewSessionModels:Module]