busya
9de74baa08
- 039-dashboard-scenario-ui: agent workspace (WorkspaceModel, scenario views, parameters/baselines, artifact preview, HITL save/approval, evidence/VLM review, pipeline verification views) + contextVersion=2 scenario intent + prototype. - 040-dashboard-load-testing: capacity/matrix/profile, runner pool, timing, circuit breaker, PROD gate, cache/bounded/consistency, comparison/schedule, API + frontend + prototype. - 041-dataset-lineage-blast-radius: usage index, severity/schema-diff, propagation, deprecation, fanout, API + frontend + prototype (R9 labels/metrics/recreate). - 036/037 amendments: dataset_updated trigger + fanout_plan_id, lineage deprecation gate, 040 read-model traceability. - Includes pre-existing 042-rls-management-workspace and 043-idm-account-integration work.
199 lines
8.2 KiB
Python
199 lines
8.2 KiB
Python
# #region Alembic.AddDatasetLineageTables [C:3] [TYPE Module] [SEMANTICS alembic,lineage,blast-radius,dataset,deprecation,fanout]
|
|
# @ingroup Alembic
|
|
# @BRIEF Add dataset lineage and blast-radius tables (041) + additive verification_runs.fanout_plan_id.
|
|
# @LAYER Database
|
|
# @RELATION DEPENDS_ON -> [Models.Lineage]
|
|
# @RELATION DEPENDS_ON -> [Models.VerificationRun]
|
|
# @INVARIANT verification_runs.fanout_plan_id is nullable with ON DELETE SET NULL — a run
|
|
# survives plan deletion (R8); null fanout_plan_id = ordinary run.
|
|
# @INVARIANT FanoutPlan.dataset_impact_id FK is NOT NULL — a plan always names its impact cause.
|
|
# @INVARIANT Edge unique key (environment_id, dataset_uuid, chart_uuid, dashboard_uuid) matches
|
|
# the ORM DatasetUsageEdge constraint (data-model).
|
|
# @RATIONALE JSON columns hold consumed sets read whole (R1); the trigger value dataset_updated
|
|
# is data-level on a plain String column (R5) so no DB enum migration is required.
|
|
# @REJECTED A dedicated lineage scheduler table was rejected (LIN-FR-002) — refresh rides the
|
|
# existing ID-synchronization cycle. Storing lineage in ResourceMapping was rejected
|
|
# (R1) — stale-deletion semantics differ from identity rows.
|
|
|
|
"""add dataset lineage tables
|
|
|
|
Revision ID: m7n8o9p1q2r3
|
|
Revises: l6m7n8o9p1q2
|
|
Create Date: 2026-08-04 19:50:00.000000
|
|
"""
|
|
|
|
from collections.abc import Sequence
|
|
|
|
import sqlalchemy as sa
|
|
|
|
from alembic import op
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision: str = "m7n8o9p1q2r3"
|
|
down_revision: str | Sequence[str] | None = "l6m7n8o9p1q2"
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
"""Create lineage tables and add the additive verification_runs.fanout_plan_id column."""
|
|
op.create_table(
|
|
"dataset_usage_edges",
|
|
sa.Column("id", sa.String(), nullable=False),
|
|
sa.Column(
|
|
"environment_id",
|
|
sa.String(),
|
|
sa.ForeignKey("environments.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("dataset_uuid", sa.String(), nullable=False),
|
|
sa.Column("chart_uuid", sa.String(), nullable=False),
|
|
sa.Column("dashboard_uuid", sa.String(), nullable=False),
|
|
sa.Column("dashboard_title", sa.String(), nullable=True),
|
|
sa.Column("consumed_columns", sa.JSON(), nullable=False),
|
|
sa.Column("consumed_metrics", sa.JSON(), nullable=False),
|
|
sa.Column("projection_confidence", sa.String(length=16), nullable=False),
|
|
sa.Column("unresolved_refs", sa.JSON(), nullable=False),
|
|
sa.Column("edge_fingerprint", sa.String(length=64), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
sa.UniqueConstraint(
|
|
"environment_id",
|
|
"dataset_uuid",
|
|
"chart_uuid",
|
|
"dashboard_uuid",
|
|
name="uq_dataset_usage_edge_env_dataset_chart_dash",
|
|
),
|
|
)
|
|
op.create_index(
|
|
"ix_dataset_usage_edges_dataset_env",
|
|
"dataset_usage_edges",
|
|
["environment_id", "dataset_uuid"],
|
|
)
|
|
op.create_index(
|
|
"ix_dataset_usage_edges_dashboard",
|
|
"dataset_usage_edges",
|
|
["environment_id", "dashboard_uuid"],
|
|
)
|
|
|
|
op.create_table(
|
|
"lineage_index_snapshots",
|
|
sa.Column("environment_id", sa.String(), nullable=False),
|
|
sa.Column("index_fingerprint", sa.String(length=64), nullable=False),
|
|
sa.Column("built_at", sa.DateTime(), nullable=False),
|
|
sa.Column("stale_index", sa.Boolean(), nullable=False),
|
|
sa.Column("last_error", sa.JSON(), nullable=True),
|
|
sa.PrimaryKeyConstraint("environment_id"),
|
|
)
|
|
|
|
op.create_table(
|
|
"dataset_schema_observations",
|
|
sa.Column("id", sa.String(), nullable=False),
|
|
sa.Column("environment_id", sa.String(), nullable=False),
|
|
sa.Column("dataset_uuid", sa.String(), nullable=False),
|
|
sa.Column("dataset_physical_key", sa.JSON(), nullable=False),
|
|
sa.Column("schema_hash", sa.String(length=64), nullable=False),
|
|
sa.Column("schema_payload", sa.JSON(), nullable=False),
|
|
sa.Column("observed_at", sa.DateTime(), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
op.create_index(
|
|
"ix_dataset_schema_observations_env_dataset_time",
|
|
"dataset_schema_observations",
|
|
["environment_id", "dataset_uuid", "observed_at"],
|
|
)
|
|
|
|
op.create_table(
|
|
"dataset_impact_records",
|
|
sa.Column("id", sa.String(), nullable=False),
|
|
sa.Column("environment_id", sa.String(), nullable=False),
|
|
sa.Column("dataset_uuid", sa.String(), nullable=False),
|
|
sa.Column("diff", sa.JSON(), nullable=False),
|
|
sa.Column("rules_version", sa.String(length=16), nullable=False),
|
|
sa.Column("max_severity", sa.String(length=16), nullable=False),
|
|
sa.Column("affected_dashboards", sa.JSON(), nullable=False),
|
|
sa.Column("observation_pair", sa.JSON(), nullable=False),
|
|
sa.Column("created_at", sa.DateTime(), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
op.create_index(
|
|
"ix_dataset_impact_records_env_dataset_time",
|
|
"dataset_impact_records",
|
|
["environment_id", "dataset_uuid", "created_at"],
|
|
)
|
|
|
|
op.create_table(
|
|
"dataset_deprecations",
|
|
sa.Column("environment_id", sa.String(), nullable=False),
|
|
sa.Column("dataset_uuid", sa.String(), nullable=False),
|
|
sa.Column("successor_dataset_uuid", sa.String(), nullable=True),
|
|
sa.Column("deprecated_at", sa.DateTime(), nullable=False),
|
|
sa.Column("grace_window_days", sa.Integer(), nullable=False),
|
|
sa.Column("escalation_state", sa.String(length=16), nullable=False),
|
|
sa.Column("expired_at", sa.DateTime(), nullable=True),
|
|
sa.Column("dependent_status", sa.JSON(), nullable=False),
|
|
sa.PrimaryKeyConstraint("environment_id", "dataset_uuid"),
|
|
)
|
|
|
|
op.create_table(
|
|
"fanout_plans",
|
|
sa.Column("id", sa.String(), nullable=False),
|
|
sa.Column("environment_id", sa.String(), nullable=False),
|
|
sa.Column(
|
|
"dataset_impact_id",
|
|
sa.String(),
|
|
sa.ForeignKey("dataset_impact_records.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("trigger", sa.String(length=32), nullable=False),
|
|
sa.Column("status", sa.String(length=16), nullable=False),
|
|
sa.Column("entries", sa.JSON(), nullable=False),
|
|
sa.Column("created_at", sa.DateTime(), nullable=False),
|
|
sa.PrimaryKeyConstraint("id"),
|
|
)
|
|
op.create_index("ix_fanout_plans_env", "fanout_plans", ["environment_id"])
|
|
op.create_index("ix_fanout_plans_created", "fanout_plans", ["created_at"])
|
|
|
|
# Additive 041 amendment: nullable fanout_plan_id on verification_runs (null = ordinary run).
|
|
op.add_column(
|
|
"verification_runs",
|
|
sa.Column(
|
|
"fanout_plan_id",
|
|
sa.String(),
|
|
sa.ForeignKey("fanout_plans.id", ondelete="SET NULL"),
|
|
nullable=True,
|
|
),
|
|
)
|
|
op.create_index(
|
|
"ix_verification_runs_fanout_plan",
|
|
"verification_runs",
|
|
["fanout_plan_id"],
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
"""Drop the additive column and lineage tables in reverse dependency order."""
|
|
op.drop_index("ix_verification_runs_fanout_plan", table_name="verification_runs")
|
|
op.drop_column("verification_runs", "fanout_plan_id")
|
|
|
|
op.drop_index("ix_fanout_plans_created", table_name="fanout_plans")
|
|
op.drop_index("ix_fanout_plans_env", table_name="fanout_plans")
|
|
op.drop_table("fanout_plans")
|
|
|
|
op.drop_table("dataset_deprecations")
|
|
|
|
op.drop_index("ix_dataset_impact_records_env_dataset_time", table_name="dataset_impact_records")
|
|
op.drop_table("dataset_impact_records")
|
|
|
|
op.drop_index(
|
|
"ix_dataset_schema_observations_env_dataset_time",
|
|
table_name="dataset_schema_observations",
|
|
)
|
|
op.drop_table("dataset_schema_observations")
|
|
|
|
op.drop_table("lineage_index_snapshots")
|
|
|
|
op.drop_index("ix_dataset_usage_edges_dashboard", table_name="dataset_usage_edges")
|
|
op.drop_index("ix_dataset_usage_edges_dataset_env", table_name="dataset_usage_edges")
|
|
op.drop_table("dataset_usage_edges")
|
|
# #endregion Alembic.AddDatasetLineageTables
|