busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: finalize semantic repair and test updates

Browse Source
This commit is contained in:
busya
2026-03-21 15:07:06 +03:00
parent c827c2f098
commit 2da548fd71
99 changed files with 2484 additions and 985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/src/api/auth.py
View File

@@ -4,9 +4,10 @@
# @SEMANTICS: api, auth, routes, login, logout
# @PURPOSE: Authentication API endpoints.
# @LAYER: API
# @RELATION: USES ->[AuthService:Class]
# @RELATION: USES ->[get_auth_db:Function]
# @RELATION: DEPENDS_ON ->[AuthRepository:Class]
# @RELATION: DEPENDS_ON -> [AuthService]
# @RELATION: DEPENDS_ON -> [get_auth_db]
# @RELATION: DEPENDS_ON -> [get_current_user]
# @RELATION: DEPENDS_ON -> [is_adfs_configured]
# @INVARIANT: All auth endpoints must return consistent error codes.
# [SECTION: IMPORTS]
@@ -24,7 +25,7 @@ import starlette.requests
# [/SECTION]
# [DEF:router:Variable]
# @RELATION: DEPENDS_ON -> fastapi.APIRouter
# @RELATION: DEPENDS_ON -> [fastapi.APIRouter]
# @COMPLEXITY: 1
# @PURPOSE: APIRouter instance for authentication routes.
router = APIRouter(prefix="/api/auth", tags=["auth"])
@@ -40,8 +41,8 @@ router = APIRouter(prefix="/api/auth", tags=["auth"])
# @PARAM: form_data (OAuth2PasswordRequestForm) - Login credentials.
# @PARAM: db (Session) - Auth database session.
# @RETURN: Token - The generated JWT token.
# @RELATION: CALLS -> [authenticate_user]
# @RELATION: CALLS -> [create_session]
# @RELATION: CALLS -> [AuthService.authenticate_user]
# @RELATION: CALLS -> [AuthService.create_session]
@router.post("/login", response_model=Token)
async def login_for_access_token(
form_data: OAuth2PasswordRequestForm = Depends(), db: Session = Depends(get_auth_db)
@@ -125,8 +126,9 @@ async def login_adfs(request: starlette.requests.Request):
# @COMPLEXITY: 3
# @PURPOSE: Handles the callback from ADFS after successful authentication.
# @POST: Provisions user JIT and returns session token.
# @RELATION: CALLS -> [provision_adfs_user]
# @RELATION: CALLS -> [create_session]
# @RELATION: DEPENDS_ON -> [is_adfs_configured]
# @RELATION: CALLS -> [AuthService.provision_adfs_user]
# @RELATION: CALLS -> [AuthService.create_session]
@router.get("/callback/adfs", name="auth_callback_adfs")
async def auth_callback_adfs(
request: starlette.requests.Request, db: Session = Depends(get_auth_db)

31
backend/src/api/routes/__tests__/test_assistant_api.py
View File

@@ -1,4 +1,5 @@
import os
os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w="
# [DEF:AssistantApiTests:Module]
# @COMPLEXITY: 3
@@ -23,7 +24,7 @@ from src.models.assistant import AssistantMessageRecord
# [DEF:_run_async:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
def _run_async(coro):
return asyncio.run(coro)
@@ -33,6 +34,9 @@ def _run_async(coro):
# [DEF:_FakeTask:Class]
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 1
# @PURPOSE: Lightweight task model stub used as return value from _FakeTaskManager.create_task in assistant route tests.
# @INVARIANT: status is a bare string not a TaskStatus enum; callers must not depend on enum semantics.
class _FakeTask:
def __init__(
self,
@@ -94,6 +98,9 @@ class _FakeTaskManager:
# [DEF:_FakeConfigManager:Class]
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 2
# @PURPOSE: Deterministic config stub providing hardcoded dev/prod environments and minimal settings shape for assistant route tests.
# @INVARIANT: get_config() returns anonymous inner classes, not real GlobalSettings; only default_environment_id and llm fields are safe to access.
class _FakeConfigManager:
class _Env:
def __init__(self, id, name):
@@ -119,7 +126,9 @@ class _FakeConfigManager:
# [DEF:_admin_user:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 1
# @PURPOSE: Build admin principal with spec=User for assistant route authorization tests.
def _admin_user():
user = MagicMock(spec=User)
user.id = "u-admin"
@@ -134,7 +143,9 @@ def _admin_user():
# [DEF:_limited_user:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 1
# @PURPOSE: Build limited user principal with empty roles for assistant route denial tests.
def _limited_user():
user = MagicMock(spec=User)
user.id = "u-limited"
@@ -148,6 +159,9 @@ def _limited_user():
# [DEF:_FakeQuery:Class]
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 2
# @PURPOSE: Chainable SQLAlchemy-like query stub returning fixed item lists for assistant message persistence paths.
# @INVARIANT: filter() ignores all predicate arguments and returns self; no predicate-based filtering is emulated.
class _FakeQuery:
def __init__(self, items):
self.items = items
@@ -183,7 +197,7 @@ class _FakeQuery:
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @COMPLEXITY: 2
# @PURPOSE: Explicit in-memory DB session double limited to assistant message persistence paths.
# @INVARIANT: query/add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior.
# @INVARIANT: query() always returns _FakeQuery with intentionally non-evaluated predicates; add/merge stay deterministic and never emulate unrelated SQLAlchemy behavior.
class _FakeDb:
def __init__(self):
self.added = []
@@ -213,7 +227,7 @@ class _FakeDb:
# [DEF:_clear_assistant_state:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
def _clear_assistant_state():
assistant_routes.CONVERSATIONS.clear()
assistant_routes.USER_ACTIVE_CONVERSATION.clear()
@@ -225,7 +239,7 @@ def _clear_assistant_state():
# [DEF:test_unknown_command_returns_needs_clarification:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @PURPOSE: Unknown command should return clarification state and unknown intent.
def test_unknown_command_returns_needs_clarification(monkeypatch):
_clear_assistant_state()
@@ -252,7 +266,7 @@ def test_unknown_command_returns_needs_clarification(monkeypatch):
# [DEF:test_capabilities_question_returns_successful_help:Function]
# @RELATION: BINDS_TO -> AssistantApiTests
# @RELATION: BINDS_TO -> [AssistantApiTests]
# @PURPOSE: Capability query should return deterministic help response.
def test_capabilities_question_returns_successful_help(monkeypatch):
_clear_assistant_state()
@@ -274,7 +288,4 @@ def test_capabilities_question_returns_successful_help(monkeypatch):
# [/DEF:test_capabilities_question_returns_successful_help:Function]
# ... (rest of file trimmed for length, I've seen it and I'll keep the existing [DEF]s as is but add @RELATION)
# Note: I'll actually just provide the full file with all @RELATIONs added to reduce orphan count.
# [/DEF:AssistantApiTests:Module]

33
backend/src/api/routes/__tests__/test_assistant_authz.py
View File

@@ -1,4 +1,5 @@
import os
os.environ["ENCRYPTION_KEY"] = "OnrCzomBWbIjTf7Y-fnhL2adlU55bHZQjp8zX5zBC5w="
# [DEF:TestAssistantAuthz:Module]
# @COMPLEXITY: 3
@@ -34,7 +35,7 @@ from src.models.assistant import (
# [DEF:_run_async:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Execute async endpoint handler in synchronous test context.
# @PRE: coroutine is awaitable endpoint invocation.
@@ -47,9 +48,11 @@ def _run_async(coroutine):
# [DEF:_FakeTask:Class]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Lightweight task model used for assistant authz tests.
# @PRE: task_id is non-empty string.
# @POST: Returns task with provided id, status, and user_id accessible as attributes.
class _FakeTask:
def __init__(self, task_id: str, status: str = "RUNNING", user_id: str = "u-admin"):
self.id = task_id
@@ -59,7 +62,7 @@ class _FakeTask:
# [/DEF:_FakeTask:Class]
# [DEF:_FakeTaskManager:Class]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 2
# @PURPOSE: In-memory task manager double that records assistant-created tasks deterministically.
# @INVARIANT: Only create_task/get_task/get_tasks behavior used by assistant authz routes is emulated.
@@ -85,9 +88,12 @@ class _FakeTaskManager:
# [/DEF:_FakeTaskManager:Class]
# [DEF:_FakeConfigManager:Class]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Provide deterministic environment aliases required by intent parsing.
# @PRE: No external config or DB state is required.
# @POST: get_environments() returns two deterministic SimpleNamespace stubs with id/name.
# @INVARIANT: get_config() is absent; only get_environments() is emulated. Safe only for routes that do not invoke get_config() on the injected ConfigManager — verify against assistant.py route handler code before adding new test cases that use this fake.
class _FakeConfigManager:
def get_environments(self):
return [
@@ -98,7 +104,7 @@ class _FakeConfigManager:
# [/DEF:_FakeConfigManager:Class]
# [DEF:_admin_user:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Build admin principal fixture.
# @PRE: Test requires privileged principal for risky operations.
@@ -110,7 +116,7 @@ def _admin_user():
# [/DEF:_admin_user:Function]
# [DEF:_other_admin_user:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Build second admin principal fixture for ownership tests.
# @PRE: Ownership mismatch scenario needs distinct authenticated actor.
@@ -122,7 +128,7 @@ def _other_admin_user():
# [/DEF:_other_admin_user:Function]
# [DEF:_limited_user:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Build limited principal without required assistant execution privileges.
# @PRE: Permission denial scenario needs non-admin actor.
@@ -136,9 +142,10 @@ def _limited_user():
# [DEF:_FakeQuery:Class]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Minimal chainable query object for fake DB interactions.
# @INVARIANT: filter() deliberately discards predicate args and returns self; tests must not assume predicate evaluation.
class _FakeQuery:
def __init__(self, rows):
self._rows = list(rows)
@@ -169,7 +176,7 @@ class _FakeQuery:
# [/DEF:_FakeQuery:Class]
# [DEF:_FakeDb:Class]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 2
# @PURPOSE: In-memory DB session double constrained to assistant message/confirmation/audit persistence paths.
# @INVARIANT: query/add/merge are intentionally narrow and must not claim full SQLAlchemy Session semantics.
@@ -218,7 +225,7 @@ class _FakeDb:
# [/DEF:_FakeDb:Class]
# [DEF:_clear_assistant_state:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @COMPLEXITY: 1
# @PURPOSE: Reset assistant process-local state between test cases.
# @PRE: Assistant globals may contain state from prior tests.
@@ -234,7 +241,7 @@ def _clear_assistant_state():
# [DEF:test_confirmation_owner_mismatch_returns_403:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @PURPOSE: Confirm endpoint should reject requests from user that does not own the confirmation token.
# @PRE: Confirmation token is created by first admin actor.
# @POST: Second actor receives 403 on confirm operation.
@@ -273,7 +280,7 @@ def test_confirmation_owner_mismatch_returns_403():
# [DEF:test_expired_confirmation_cannot_be_confirmed:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @PURPOSE: Expired confirmation token should be rejected and not create task.
# @PRE: Confirmation token exists and is manually expired before confirm request.
# @POST: Confirm endpoint raises 400 and no task is created.
@@ -315,7 +322,7 @@ def test_expired_confirmation_cannot_be_confirmed():
# [DEF:test_limited_user_cannot_launch_restricted_operation:Function]
# @RELATION: BINDS_TO -> TestAssistantAuthz
# @RELATION: BINDS_TO -> [TestAssistantAuthz]
# @PURPOSE: Limited user should receive denied state for privileged operation.
# @PRE: Restricted user attempts dangerous deploy command.
# @POST: Assistant returns denied state and does not execute operation.

4
backend/src/api/routes/__tests__/test_clean_release_api.py
View File

@@ -79,6 +79,7 @@ def _repo_with_seed_data() -> CleanReleaseRepository:
# [DEF:test_start_check_and_get_status_contract:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseApi
# @PURPOSE: Validate checks start endpoint returns expected identifiers and status endpoint reflects the same run.
def test_start_check_and_get_status_contract():
repo = _repo_with_seed_data()
app.dependency_overrides[get_clean_release_repository] = lambda: repo
@@ -116,6 +117,7 @@ def test_start_check_and_get_status_contract():
# [DEF:test_get_report_not_found_returns_404:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseApi
# @PURPOSE: Validate reports endpoint returns 404 for an unknown report identifier.
def test_get_report_not_found_returns_404():
repo = _repo_with_seed_data()
app.dependency_overrides[get_clean_release_repository] = lambda: repo
@@ -132,6 +134,7 @@ def test_get_report_not_found_returns_404():
# [DEF:test_get_report_success:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseApi
# @PURPOSE: Validate reports endpoint returns persisted report payload for an existing report identifier.
def test_get_report_success():
repo = _repo_with_seed_data()
report = ComplianceReport(
@@ -161,6 +164,7 @@ def test_get_report_success():
# [DEF:test_prepare_candidate_api_success:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseApi
# @PURPOSE: Validate candidate preparation endpoint returns prepared status and manifest identifier on valid input.
def test_prepare_candidate_api_success():
repo = _repo_with_seed_data()
app.dependency_overrides[get_clean_release_repository] = lambda: repo

25
backend/src/api/routes/__tests__/test_clean_release_legacy_compat.py
View File

@@ -12,7 +12,9 @@ from datetime import datetime, timezone
from fastapi.testclient import TestClient
os.environ.setdefault("DATABASE_URL", "sqlite:///./test_clean_release_legacy_compat.db")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db")
os.environ.setdefault(
"AUTH_DATABASE_URL", "sqlite:///./test_clean_release_legacy_auth.db"
)
from src.app import app
from src.dependencies import get_clean_release_repository
@@ -103,17 +105,23 @@ def _seed_legacy_repo() -> CleanReleaseRepository:
created_at=now,
created_by="compat-tester",
source_snapshot_ref="git:legacy-001",
content_json={"items": [], "summary": {"included_count": 0, "prohibited_detected_count": 0}},
content_json={
"items": [],
"summary": {"included_count": 0, "prohibited_detected_count": 0},
},
immutable=True,
)
)
return repo
# [/DEF:_seed_legacy_repo:Function]
# [DEF:test_legacy_prepare_endpoint_still_available:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat
# @PURPOSE: Verify legacy prepare endpoint remains reachable and returns a status payload.
def test_legacy_prepare_endpoint_still_available() -> None:
repo = _seed_legacy_repo()
app.dependency_overrides[get_clean_release_repository] = lambda: repo
@@ -123,7 +131,9 @@ def test_legacy_prepare_endpoint_still_available() -> None:
"/api/clean-release/candidates/prepare",
json={
"candidate_id": "legacy-rc-001",
"artifacts": [{"path": "src/main.py", "category": "core", "reason": "required"}],
"artifacts": [
{"path": "src/main.py", "category": "core", "reason": "required"}
],
"sources": ["repo.intra.company.local"],
"operator_id": "compat-tester",
},
@@ -138,8 +148,10 @@ def test_legacy_prepare_endpoint_still_available() -> None:
# [/DEF:test_legacy_prepare_endpoint_still_available:Function]
# [DEF:test_legacy_checks_endpoints_still_available:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseLegacyCompat
# @PURPOSE: Verify legacy checks start/status endpoints remain available during v2 transition.
def test_legacy_checks_endpoints_still_available() -> None:
repo = _seed_legacy_repo()
app.dependency_overrides[get_clean_release_repository] = lambda: repo
@@ -159,7 +171,9 @@ def test_legacy_checks_endpoints_still_available() -> None:
assert "check_run_id" in start_payload
assert start_payload["candidate_id"] == "legacy-rc-001"
status_response = client.get(f"/api/clean-release/checks/{start_payload['check_run_id']}")
status_response = client.get(
f"/api/clean-release/checks/{start_payload['check_run_id']}"
)
assert status_response.status_code == 200
status_payload = status_response.json()
assert status_payload["check_run_id"] == start_payload["check_run_id"]
@@ -169,4 +183,5 @@ def test_legacy_checks_endpoints_still_available() -> None:
app.dependency_overrides.clear()
# [/DEF:TestCleanReleaseLegacyCompat:Module]# [/DEF:test_legacy_checks_endpoints_still_available:Function]
# [/DEF:test_legacy_checks_endpoints_still_available:Function]
# [/DEF:TestCleanReleaseLegacyCompat:Module]

12
backend/src/api/routes/__tests__/test_clean_release_source_policy.py
View File

@@ -24,6 +24,7 @@ from src.services.clean_release.repository import CleanReleaseRepository
# [DEF:_repo_with_seed_data:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy
# @PURPOSE: Seed repository with candidate, registry, and active policy for source isolation test flow.
def _repo_with_seed_data() -> CleanReleaseRepository:
repo = CleanReleaseRepository()
@@ -76,8 +77,10 @@ def _repo_with_seed_data() -> CleanReleaseRepository:
# [/DEF:_repo_with_seed_data:Function]
# [DEF:test_prepare_candidate_blocks_external_source:Function]
# @RELATION: BINDS_TO -> TestCleanReleaseSourcePolicy
# @PURPOSE: Verify candidate preparation is blocked when at least one source host is external to the trusted registry.
def test_prepare_candidate_blocks_external_source():
repo = _repo_with_seed_data()
app.dependency_overrides[get_clean_release_repository] = lambda: repo
@@ -89,7 +92,11 @@ def test_prepare_candidate_blocks_external_source():
json={
"candidate_id": "2026.03.03-rc1",
"artifacts": [
{"path": "cfg/system.yaml", "category": "system-init", "reason": "required"}
{
"path": "cfg/system.yaml",
"category": "system-init",
"reason": "required",
}
],
"sources": ["repo.intra.company.local", "pypi.org"],
"operator_id": "release-manager",
@@ -103,4 +110,5 @@ def test_prepare_candidate_blocks_external_source():
app.dependency_overrides.clear()
# [/DEF:TestCleanReleaseSourcePolicy:Module]# [/DEF:test_prepare_candidate_blocks_external_source:Function]
# [/DEF:test_prepare_candidate_blocks_external_source:Function]
# [/DEF:TestCleanReleaseSourcePolicy:Module]

3
backend/src/api/routes/__tests__/test_clean_release_v2_api.py
View File

@@ -27,6 +27,7 @@ client = TestClient(app)
# [REASON] Implementing API contract tests for candidate/artifact/manifest endpoints (T012).
# [DEF:test_candidate_registration_contract:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests
# @PURPOSE: Validate candidate registration endpoint creates a draft candidate with expected identifier contract.
def test_candidate_registration_contract():
"""
@TEST_SCENARIO: candidate_registration -> Should return 201 and candidate DTO.
@@ -50,6 +51,7 @@ def test_candidate_registration_contract():
# [DEF:test_artifact_import_contract:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests
# @PURPOSE: Validate artifact import endpoint accepts candidate artifacts and returns success status payload.
def test_artifact_import_contract():
"""
@TEST_SCENARIO: artifact_import -> Should return 200 and success status.
@@ -84,6 +86,7 @@ def test_artifact_import_contract():
# [DEF:test_manifest_build_contract:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ApiTests
# @PURPOSE: Validate manifest build endpoint produces manifest payload linked to the target candidate.
def test_manifest_build_contract():
"""
@TEST_SCENARIO: manifest_build -> Should return 201 and manifest DTO.

14
backend/src/api/routes/__tests__/test_clean_release_v2_release_api.py
View File

@@ -25,6 +25,7 @@ client = TestClient(test_app)
# [DEF:_seed_candidate_and_passed_report:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests
# @PURPOSE: Seed repository with approvable candidate and passed report for release endpoint contracts.
def _seed_candidate_and_passed_report() -> tuple[str, str]:
repository = get_clean_release_repository()
candidate_id = f"api-release-candidate-{uuid4()}"
@@ -46,7 +47,11 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]:
run_id=f"run-{uuid4()}",
candidate_id=candidate_id,
final_status=ComplianceDecision.PASSED.value,
summary_json={"operator_summary": "ok", "violations_count": 0, "blocking_violations_count": 0},
summary_json={
"operator_summary": "ok",
"violations_count": 0,
"blocking_violations_count": 0,
},
generated_at=datetime.now(timezone.utc),
immutable=True,
)
@@ -56,8 +61,10 @@ def _seed_candidate_and_passed_report() -> tuple[str, str]:
# [/DEF:_seed_candidate_and_passed_report:Function]
# [DEF:test_release_approve_and_publish_revoke_contract:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests
# @PURPOSE: Verify approve, publish, and revoke endpoints preserve expected release lifecycle contract.
def test_release_approve_and_publish_revoke_contract() -> None:
"""Contract for approve -> publish -> revoke lifecycle endpoints."""
candidate_id, report_id = _seed_candidate_and_passed_report()
@@ -98,8 +105,10 @@ def test_release_approve_and_publish_revoke_contract() -> None:
# [/DEF:test_release_approve_and_publish_revoke_contract:Function]
# [DEF:test_release_reject_contract:Function]
# @RELATION: BINDS_TO -> CleanReleaseV2ReleaseApiTests
# @PURPOSE: Verify reject endpoint returns successful rejection decision payload.
def test_release_reject_contract() -> None:
"""Contract for reject endpoint."""
candidate_id, report_id = _seed_candidate_and_passed_report()
@@ -114,4 +123,5 @@ def test_release_reject_contract() -> None:
assert payload["decision"] == "REJECTED"
# [/DEF:CleanReleaseV2ReleaseApiTests:Module]# [/DEF:test_release_reject_contract:Function]
# [/DEF:test_release_reject_contract:Function]
# [/DEF:CleanReleaseV2ReleaseApiTests:Module]

6
backend/src/api/routes/__tests__/test_connections_routes.py
View File

@@ -41,6 +41,7 @@ def db_session():
# [DEF:test_list_connections_bootstraps_missing_table:Function]
# @RELATION: BINDS_TO -> ConnectionsRoutesTests
# @PURPOSE: Ensure listing connections auto-creates missing table and returns empty payload.
def test_list_connections_bootstraps_missing_table(db_session):
from src.api.routes.connections import list_connections
@@ -53,8 +54,10 @@ def test_list_connections_bootstraps_missing_table(db_session):
# [/DEF:test_list_connections_bootstraps_missing_table:Function]
# [DEF:test_create_connection_bootstraps_missing_table:Function]
# @RELATION: BINDS_TO -> ConnectionsRoutesTests
# @PURPOSE: Ensure connection creation bootstraps table and persists returned connection fields.
def test_create_connection_bootstraps_missing_table(db_session):
from src.api.routes.connections import ConnectionCreate, create_connection
@@ -75,5 +78,6 @@ def test_create_connection_bootstraps_missing_table(db_session):
assert created.host == "warehouse.internal"
assert "connection_configs" in inspector.get_table_names()
# [/DEF:ConnectionsRoutesTests:Module]
# [/DEF:test_create_connection_bootstraps_missing_table:Function]
# [/DEF:ConnectionsRoutesTests:Module]

159
backend/src/api/routes/__tests__/test_datasets.py
View File

@@ -3,7 +3,7 @@
# @SEMANTICS: datasets, api, tests, pagination, mapping, docs
# @PURPOSE: Unit tests for datasets API endpoints.
# @LAYER: API
# @RELATION: DEPENDS_ON -> backend.src.api.routes.datasets
# @RELATION: DEPENDS_ON -> [src.api.routes.datasets:Module]
# @INVARIANT: Endpoint contracts remain stable for success and validation failure paths.
import pytest
@@ -11,7 +11,14 @@ from unittest.mock import MagicMock, patch, AsyncMock
from fastapi.testclient import TestClient
from src.app import app
from src.api.routes.datasets import DatasetsResponse, DatasetDetailResponse
from src.dependencies import get_current_user, has_permission, get_config_manager, get_task_manager, get_resource_service, get_mapping_service
from src.dependencies import (
get_current_user,
has_permission,
get_config_manager,
get_task_manager,
get_resource_service,
get_mapping_service,
)
# Global mock user for get_current_user dependency overrides
mock_user = MagicMock()
@@ -21,49 +28,58 @@ admin_role = MagicMock()
admin_role.name = "Admin"
mock_user.roles.append(admin_role)
@pytest.fixture(autouse=True)
def mock_deps():
# @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass
config_manager = MagicMock()
# @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass
task_manager = MagicMock()
# @INVARIANT: unconstrained mock — no spec= enforced; attribute typos will silently pass
resource_service = MagicMock()
mapping_service = MagicMock()
app.dependency_overrides[get_config_manager] = lambda: config_manager
app.dependency_overrides[get_task_manager] = lambda: task_manager
app.dependency_overrides[get_resource_service] = lambda: resource_service
app.dependency_overrides[get_mapping_service] = lambda: mapping_service
app.dependency_overrides[get_current_user] = lambda: mock_user
app.dependency_overrides[has_permission("plugin:migration", "READ")] = lambda: mock_user
app.dependency_overrides[has_permission("plugin:migration", "EXECUTE")] = lambda: mock_user
app.dependency_overrides[has_permission("plugin:backup", "EXECUTE")] = lambda: mock_user
app.dependency_overrides[has_permission("plugin:migration", "READ")] = (
lambda: mock_user
)
app.dependency_overrides[has_permission("plugin:migration", "EXECUTE")] = (
lambda: mock_user
)
app.dependency_overrides[has_permission("plugin:backup", "EXECUTE")] = (
lambda: mock_user
)
app.dependency_overrides[has_permission("tasks", "READ")] = lambda: mock_user
yield {
"config": config_manager,
"task": task_manager,
"resource": resource_service,
"mapping": mapping_service
"mapping": mapping_service,
}
app.dependency_overrides.clear()
client = TestClient(app)
# [DEF:test_get_datasets_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate successful datasets listing contract for an existing environment.
# @TEST: GET /api/datasets returns 200 and valid schema
# @PRE: env_id exists
# @POST: Response matches DatasetsResponse schema
# [DEF:test_get_datasets_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_get_datasets_success(mock_deps):
# Mock environment
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
# Mock resource service response
mock_deps["resource"].get_datasets_with_status = AsyncMock(
return_value=[
@@ -73,13 +89,13 @@ def test_get_datasets_success(mock_deps):
"schema": "public",
"database": "sales_db",
"mapped_fields": {"total": 10, "mapped": 5},
"last_task": {"task_id": "task-1", "status": "SUCCESS"}
"last_task": {"task_id": "task-1", "status": "SUCCESS"},
}
]
)
response = client.get("/api/datasets?env_id=prod")
assert response.status_code == 200
data = response.json()
assert "datasets" in data
@@ -92,20 +108,16 @@ def test_get_datasets_success(mock_deps):
# [DEF:test_get_datasets_env_not_found:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate datasets listing returns 404 when the requested environment does not exist.
# @TEST: GET /api/datasets returns 404 if env_id missing
# @PRE: env_id does not exist
# @POST: Returns 404 error
# [/DEF:test_get_datasets_success:Function]
# [DEF:test_get_datasets_env_not_found:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_get_datasets_env_not_found(mock_deps):
mock_deps["config"].get_environments.return_value = []
response = client.get("/api/datasets?env_id=nonexistent")
assert response.status_code == 404
assert "Environment not found" in response.json()["detail"]
@@ -114,15 +126,11 @@ def test_get_datasets_env_not_found(mock_deps):
# [DEF:test_get_datasets_invalid_pagination:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate datasets listing rejects invalid pagination parameters with 400 responses.
# @TEST: GET /api/datasets returns 400 for invalid page/page_size
# @PRE: page < 1 or page_size > 100
# @POST: Returns 400 error
# [/DEF:test_get_datasets_env_not_found:Function]
# [DEF:test_get_datasets_invalid_pagination:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_get_datasets_invalid_pagination(mock_deps):
mock_env = MagicMock()
mock_env.id = "prod"
@@ -132,7 +140,7 @@ def test_get_datasets_invalid_pagination(mock_deps):
response = client.get("/api/datasets?env_id=prod&page=0")
assert response.status_code == 400
assert "Page must be >= 1" in response.json()["detail"]
# Invalid page_size (too small)
response = client.get("/api/datasets?env_id=prod&page_size=0")
assert response.status_code == 400
@@ -148,21 +156,17 @@ def test_get_datasets_invalid_pagination(mock_deps):
# [DEF:test_map_columns_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate map-columns request creates an async mapping task and returns its identifier.
# @TEST: POST /api/datasets/map-columns creates mapping task
# @PRE: Valid env_id, dataset_ids, source_type
# @POST: Returns task_id
# [/DEF:test_get_datasets_invalid_pagination:Function]
# [DEF:test_map_columns_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_map_columns_success(mock_deps):
# Mock environment
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
# Mock task manager
mock_task = MagicMock()
mock_task.id = "task-123"
@@ -170,13 +174,9 @@ def test_map_columns_success(mock_deps):
response = client.post(
"/api/datasets/map-columns",
json={
"env_id": "prod",
"dataset_ids": [1, 2, 3],
"source_type": "postgresql"
}
json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "postgresql"},
)
assert response.status_code == 200
data = response.json()
assert "task_id" in data
@@ -188,25 +188,17 @@ def test_map_columns_success(mock_deps):
# [DEF:test_map_columns_invalid_source_type:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate map-columns rejects unsupported source types with a 400 contract response.
# @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type
# @PRE: source_type is not 'postgresql' or 'xlsx'
# @POST: Returns 400 error
# [/DEF:test_map_columns_success:Function]
# [DEF:test_map_columns_invalid_source_type:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_map_columns_invalid_source_type(mock_deps):
response = client.post(
"/api/datasets/map-columns",
json={
"env_id": "prod",
"dataset_ids": [1],
"source_type": "invalid"
}
json={"env_id": "prod", "dataset_ids": [1], "source_type": "invalid"},
)
assert response.status_code == 400
assert "Source type must be 'postgresql' or 'xlsx'" in response.json()["detail"]
@@ -215,21 +207,17 @@ def test_map_columns_invalid_source_type(mock_deps):
# [DEF:test_generate_docs_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @TEST: POST /api/datasets/generate-docs creates doc generation task
# @PRE: Valid env_id, dataset_ids, llm_provider
# @PURPOSE: Validate generate-docs request creates an async documentation task and returns its identifier.
# @POST: Returns task_id
# [/DEF:test_map_columns_invalid_source_type:Function]
# [DEF:test_generate_docs_success:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_generate_docs_success(mock_deps):
# Mock environment
mock_env = MagicMock()
mock_env.id = "prod"
mock_deps["config"].get_environments.return_value = [mock_env]
# Mock task manager
mock_task = MagicMock()
mock_task.id = "task-456"
@@ -237,13 +225,9 @@ def test_generate_docs_success(mock_deps):
response = client.post(
"/api/datasets/generate-docs",
json={
"env_id": "prod",
"dataset_ids": [1],
"llm_provider": "openai"
}
json={"env_id": "prod", "dataset_ids": [1], "llm_provider": "openai"},
)
assert response.status_code == 200
data = response.json()
assert "task_id" in data
@@ -255,87 +239,68 @@ def test_generate_docs_success(mock_deps):
# [DEF:test_map_columns_empty_ids:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate map-columns rejects empty dataset identifier lists.
# @TEST: POST /api/datasets/map-columns returns 400 for empty dataset_ids
# @PRE: dataset_ids is empty
# @POST: Returns 400 error
# [/DEF:test_generate_docs_success:Function]
# [DEF:test_map_columns_empty_ids:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_map_columns_empty_ids(mock_deps):
"""@PRE: dataset_ids must be non-empty."""
response = client.post(
"/api/datasets/map-columns",
json={
"env_id": "prod",
"dataset_ids": [],
"source_type": "postgresql"
}
json={"env_id": "prod", "dataset_ids": [], "source_type": "postgresql"},
)
assert response.status_code == 400
assert "At least one dataset ID must be provided" in response.json()["detail"]
# [/DEF:test_map_columns_empty_ids:Function]
# [DEF:test_generate_docs_empty_ids:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate generate-docs rejects empty dataset identifier lists.
# @TEST: POST /api/datasets/generate-docs returns 400 for empty dataset_ids
# @PRE: dataset_ids is empty
# @POST: Returns 400 error
# [/DEF:test_map_columns_empty_ids:Function]
# [DEF:test_generate_docs_empty_ids:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_generate_docs_empty_ids(mock_deps):
"""@PRE: dataset_ids must be non-empty."""
response = client.post(
"/api/datasets/generate-docs",
json={
"env_id": "prod",
"dataset_ids": [],
"llm_provider": "openai"
}
json={"env_id": "prod", "dataset_ids": [], "llm_provider": "openai"},
)
assert response.status_code == 400
assert "At least one dataset ID must be provided" in response.json()["detail"]
# [/DEF:test_generate_docs_empty_ids:Function]
# [DEF:test_generate_docs_env_not_found:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @TEST: POST /api/datasets/generate-docs returns 404 for missing env
# @PRE: env_id does not exist
# @PURPOSE: Validate generate-docs returns 404 when the requested environment cannot be resolved.
# @POST: Returns 404 error
# [/DEF:test_generate_docs_empty_ids:Function]
# [DEF:test_generate_docs_env_not_found:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
def test_generate_docs_env_not_found(mock_deps):
"""@PRE: env_id must be a valid environment."""
mock_deps["config"].get_environments.return_value = []
response = client.post(
"/api/datasets/generate-docs",
json={
"env_id": "ghost",
"dataset_ids": [1],
"llm_provider": "openai"
}
json={"env_id": "ghost", "dataset_ids": [1], "llm_provider": "openai"},
)
assert response.status_code == 404
assert "Environment not found" in response.json()["detail"]
# [/DEF:test_generate_docs_env_not_found:Function]
# [DEF:test_get_datasets_superset_failure:Function]
# @RELATION: BINDS_TO -> DatasetsApiTests
# @RELATION: BINDS_TO -> [DatasetsApiTests:Module]
# @PURPOSE: Validate datasets listing surfaces a 503 contract when Superset access fails.
# @TEST_EDGE: external_superset_failure -> {status: 503}
# [/DEF:test_generate_docs_env_not_found:Function]
# @POST: Returns 503 with stable error detail when upstream dataset fetch fails.
def test_get_datasets_superset_failure(mock_deps):
"""@TEST_EDGE: external_superset_failure -> {status: 503}"""
mock_env = MagicMock()
@@ -349,7 +314,9 @@ def test_get_datasets_superset_failure(mock_deps):
response = client.get("/api/datasets?env_id=bad_conn")
assert response.status_code == 503
assert "Failed to fetch datasets" in response.json()["detail"]
# [/DEF:test_get_datasets_superset_failure:Function]
# [/DEF:DatasetsApiTests:Module]
# [/DEF:DatasetsApiTests:Module]

44
backend/src/api/routes/__tests__/test_git_api.py
View File

@@ -1,6 +1,6 @@
# [DEF:TestGitApi:Module]
# @COMPLEXITY: 3
# @RELATION: VERIFIES ->[src.api.routes.git]
# @RELATION: VERIFIES -> [GitApi]
# @PURPOSE: API tests for Git configurations and repository operations.
import pytest
@@ -12,7 +12,7 @@ from src.models.git import GitServerConfig, GitProvider, GitStatus, GitRepositor
# [DEF:DbMock:Class]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @COMPLEXITY: 2
# @PURPOSE: In-memory session double for git route tests with minimal query/filter persistence semantics.
# @INVARIANT: Supports only the SQLAlchemy-like operations exercised by this test module.
@@ -88,7 +88,8 @@ class DbMock:
# [DEF:test_get_git_configs_masks_pat:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate listing git configs masks stored PAT values in API-facing responses.
def test_get_git_configs_masks_pat():
"""
@PRE: Database session `db` is available.
@@ -119,7 +120,8 @@ def test_get_git_configs_masks_pat():
# [DEF:test_create_git_config_persists_config:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate creating git config persists supplied server attributes in backing session.
def test_create_git_config_persists_config():
"""
@PRE: `config` contains valid GitServerConfigCreate data.
@@ -153,7 +155,8 @@ from src.api.routes.git_schemas import GitServerConfigUpdate
# [DEF:test_update_git_config_modifies_record:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate updating git config modifies mutable fields while preserving masked PAT semantics.
def test_update_git_config_modifies_record():
"""
@PRE: `config_id` corresponds to an existing configuration.
@@ -170,6 +173,7 @@ def test_update_git_config_modifies_record():
)
# The monkeypatched query will return existing_config as it's the only one in the list
# [DEF:SingleConfigDbMock:Class] @PURPOSE: Fake SQLAlchemy session returning single config row. @INVARIANT: Returns hardcoded single-item list; does not simulate empty or multi-row results.
class SingleConfigDbMock:
def query(self, *args):
return self
@@ -206,7 +210,8 @@ def test_update_git_config_modifies_record():
# [DEF:test_update_git_config_raises_404_if_not_found:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate updating non-existent git config raises HTTP 404 contract response.
def test_update_git_config_raises_404_if_not_found():
"""
@PRE: `config_id` corresponds to a missing configuration.
@@ -230,7 +235,8 @@ def test_update_git_config_raises_404_if_not_found():
# [DEF:test_delete_git_config_removes_record:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate deleting existing git config removes record and returns success payload.
def test_delete_git_config_removes_record():
"""
@PRE: `config_id` corresponds to an existing configuration.
@@ -238,6 +244,7 @@ def test_delete_git_config_removes_record():
"""
existing_config = GitServerConfig(id="config-1")
# [DEF:SingleConfigDbMock:Class] @PURPOSE: Fake SQLAlchemy session returning single config row. @INVARIANT: Returns hardcoded single-item list; does not simulate empty or multi-row results.
class SingleConfigDbMock:
def query(self, *args):
return self
@@ -266,13 +273,15 @@ def test_delete_git_config_removes_record():
# [DEF:test_test_git_config_validates_connection_successfully:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate test-connection endpoint returns success when provider connectivity check passes.
def test_test_git_config_validates_connection_successfully(monkeypatch):
"""
@PRE: `config` contains provider, url, and pat.
@POST: Returns success if the connection is validated via GitService.
"""
# [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths.
class MockGitService:
async def test_connection(self, provider, url, pat):
return True
@@ -297,13 +306,15 @@ def test_test_git_config_validates_connection_successfully(monkeypatch):
# [DEF:test_test_git_config_fails_validation:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate test-connection endpoint raises HTTP 400 when provider connectivity check fails.
def test_test_git_config_fails_validation(monkeypatch):
"""
@PRE: `config` contains provider, url, and pat BUT connection fails.
@THROW: HTTPException 400
"""
# [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths.
class MockGitService:
async def test_connection(self, provider, url, pat):
return False
@@ -330,13 +341,15 @@ def test_test_git_config_fails_validation(monkeypatch):
# [DEF:test_list_gitea_repositories_returns_payload:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate gitea repositories endpoint returns normalized list for GITEA provider configs.
def test_list_gitea_repositories_returns_payload(monkeypatch):
"""
@PRE: config_id exists and provider is GITEA.
@POST: Returns repositories visible to PAT user.
"""
# [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths.
class MockGitService:
async def list_gitea_repositories(self, url, pat):
return [
@@ -366,7 +379,8 @@ def test_list_gitea_repositories_returns_payload(monkeypatch):
# [DEF:test_list_gitea_repositories_rejects_non_gitea:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate gitea repositories endpoint rejects non-GITEA providers with HTTP 400.
def test_list_gitea_repositories_rejects_non_gitea(monkeypatch):
"""
@PRE: config_id exists and provider is NOT GITEA.
@@ -392,13 +406,15 @@ def test_list_gitea_repositories_rejects_non_gitea(monkeypatch):
# [DEF:test_create_remote_repository_creates_provider_repo:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate remote repository creation endpoint maps provider response into normalized payload.
def test_create_remote_repository_creates_provider_repo(monkeypatch):
"""
@PRE: config_id exists and PAT has creation permissions.
@POST: Returns normalized remote repository payload.
"""
# [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths.
class MockGitService:
async def create_gitlab_repository(
self, server_url, pat, name, private, description, auto_init, default_branch
@@ -438,7 +454,8 @@ def test_create_remote_repository_creates_provider_repo(monkeypatch):
# [DEF:test_init_repository_initializes_and_saves_binding:Function]
# @RELATION: BINDS_TO ->[TestGitApi]
# @RELATION: BINDS_TO -> [TestGitApi]
# @PURPOSE: Validate repository initialization endpoint creates local repo and persists dashboard binding.
def test_init_repository_initializes_and_saves_binding(monkeypatch):
"""
@PRE: `dashboard_ref` exists and `init_data` contains valid config_id and remote_url.
@@ -446,6 +463,7 @@ def test_init_repository_initializes_and_saves_binding(monkeypatch):
"""
from src.api.routes.git_schemas import RepoInitRequest
# [DEF:MockGitService:Class] @PURPOSE: Stub GitService returning controlled responses. @INVARIANT: Returns only the configured response; does not simulate partial failure or exception paths.
class MockGitService:
def init_repo(self, dashboard_id, remote_url, pat, repo_key, default_branch):
self.init_called = True

14
backend/src/api/routes/__tests__/test_reports_api.py
View File

@@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus
from src.dependencies import get_current_user, get_task_manager
# [DEF:_FakeTaskManager:Class]
# @RELATION: BINDS_TO -> [TestReportsApi]
# @COMPLEXITY: 1
# @PURPOSE: Minimal task-manager double exposing only get_all_tasks used by reports route tests.
# @INVARIANT: Returns pre-seeded tasks without mutation or side effects.
class _FakeTaskManager:
def __init__(self, tasks):
self._tasks = tasks
@@ -24,8 +29,12 @@ class _FakeTaskManager:
return self._tasks
# [/DEF:_FakeTaskManager:Class]
# [DEF:_admin_user:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Build deterministic admin principal accepted by reports authorization guard.
def _admin_user():
admin_role = SimpleNamespace(name="Admin", permissions=[])
return SimpleNamespace(username="test-admin", roles=[admin_role])
@@ -36,6 +45,7 @@ def _admin_user():
# [DEF:_make_task:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Build Task fixture with controlled timestamps/status for reports list/detail normalization.
def _make_task(
task_id: str,
plugin_id: str,
@@ -60,6 +70,7 @@ def _make_task(
# [DEF:test_get_reports_default_pagination_contract:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Validate reports list endpoint default pagination and contract keys for mixed task statuses.
def test_get_reports_default_pagination_contract():
now = datetime.utcnow()
tasks = [
@@ -113,6 +124,7 @@ def test_get_reports_default_pagination_contract():
# [DEF:test_get_reports_filter_and_pagination:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Validate reports list endpoint applies task-type/status filters and pagination boundaries.
def test_get_reports_filter_and_pagination():
now = datetime.utcnow()
tasks = [
@@ -166,6 +178,7 @@ def test_get_reports_filter_and_pagination():
# [DEF:test_get_reports_handles_mixed_naive_and_aware_datetimes:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Validate reports sorting remains stable when task timestamps mix naive and timezone-aware datetimes.
def test_get_reports_handles_mixed_naive_and_aware_datetimes():
naive_now = datetime.utcnow()
aware_now = datetime.now(timezone.utc)
@@ -205,6 +218,7 @@ def test_get_reports_handles_mixed_naive_and_aware_datetimes():
# [DEF:test_get_reports_invalid_filter_returns_400:Function]
# @RELATION: BINDS_TO -> TestReportsApi
# @PURPOSE: Validate reports list endpoint rejects unsupported task type filters with HTTP 400.
def test_get_reports_invalid_filter_returns_400():
now = datetime.utcnow()
tasks = [

12
backend/src/api/routes/__tests__/test_reports_detail_api.py
View File

@@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus
from src.dependencies import get_current_user, get_task_manager
# [DEF:_FakeTaskManager:Class]
# @RELATION: BINDS_TO -> [TestReportsDetailApi]
# @COMPLEXITY: 1
# @PURPOSE: Minimal task-manager double exposing pre-seeded tasks to detail endpoint under test.
# @INVARIANT: get_all_tasks returns exactly seeded tasks list.
class _FakeTaskManager:
def __init__(self, tasks):
self._tasks = tasks
@@ -24,8 +29,12 @@ class _FakeTaskManager:
return self._tasks
# [/DEF:_FakeTaskManager:Class]
# [DEF:_admin_user:Function]
# @RELATION: BINDS_TO -> TestReportsDetailApi
# @PURPOSE: Provide admin principal fixture accepted by reports detail authorization policy.
def _admin_user():
role = SimpleNamespace(name="Admin", permissions=[])
return SimpleNamespace(username="test-admin", roles=[role])
@@ -36,6 +45,7 @@ def _admin_user():
# [DEF:_make_task:Function]
# @RELATION: BINDS_TO -> TestReportsDetailApi
# @PURPOSE: Build deterministic Task payload for reports detail endpoint contract assertions.
def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None):
now = datetime.utcnow()
return Task(
@@ -56,6 +66,7 @@ def _make_task(task_id: str, plugin_id: str, status: TaskStatus, result=None):
# [DEF:test_get_report_detail_success:Function]
# @RELATION: BINDS_TO -> TestReportsDetailApi
# @PURPOSE: Validate report detail endpoint returns report body with diagnostics and next actions for existing task.
def test_get_report_detail_success():
task = _make_task(
"detail-1",
@@ -91,6 +102,7 @@ def test_get_report_detail_success():
# [DEF:test_get_report_detail_not_found:Function]
# @RELATION: BINDS_TO -> TestReportsDetailApi
# @PURPOSE: Validate report detail endpoint returns 404 when requested report identifier is absent.
def test_get_report_detail_not_found():
task = _make_task("detail-2", "superset-backup", TaskStatus.SUCCESS)

12
backend/src/api/routes/__tests__/test_reports_openapi_conformance.py
View File

@@ -16,6 +16,11 @@ from src.core.task_manager.models import Task, TaskStatus
from src.dependencies import get_current_user, get_task_manager
# [DEF:_FakeTaskManager:Class]
# @RELATION: BINDS_TO -> [TestReportsOpenapiConformance]
# @COMPLEXITY: 1
# @PURPOSE: Minimal task-manager fake exposing static task list for OpenAPI conformance checks.
# @INVARIANT: get_all_tasks returns seeded tasks unchanged.
class _FakeTaskManager:
def __init__(self, tasks):
self._tasks = tasks
@@ -24,8 +29,12 @@ class _FakeTaskManager:
return self._tasks
# [/DEF:_FakeTaskManager:Class]
# [DEF:_admin_user:Function]
# @RELATION: BINDS_TO -> TestReportsOpenapiConformance
# @PURPOSE: Provide admin principal fixture required by reports routes in conformance tests.
def _admin_user():
role = SimpleNamespace(name="Admin", permissions=[])
return SimpleNamespace(username="test-admin", roles=[role])
@@ -36,6 +45,7 @@ def _admin_user():
# [DEF:_task:Function]
# @RELATION: BINDS_TO -> TestReportsOpenapiConformance
# @PURPOSE: Construct deterministic task fixture consumed by reports list/detail payload assertions.
def _task(task_id: str, plugin_id: str, status: TaskStatus):
now = datetime.utcnow()
return Task(
@@ -54,6 +64,7 @@ def _task(task_id: str, plugin_id: str, status: TaskStatus):
# [DEF:test_reports_list_openapi_required_keys:Function]
# @RELATION: BINDS_TO -> TestReportsOpenapiConformance
# @PURPOSE: Verify reports list endpoint includes all required OpenAPI top-level keys.
def test_reports_list_openapi_required_keys():
tasks = [
_task("r-1", "superset-backup", TaskStatus.SUCCESS),
@@ -86,6 +97,7 @@ def test_reports_list_openapi_required_keys():
# [DEF:test_reports_detail_openapi_required_keys:Function]
# @RELATION: BINDS_TO -> TestReportsOpenapiConformance
# @PURPOSE: Verify reports detail endpoint returns payload containing the report object key.
def test_reports_detail_openapi_required_keys():
tasks = [_task("r-3", "llm_dashboard_validation", TaskStatus.SUCCESS)]
app.dependency_overrides[get_current_user] = lambda: _admin_user()

54
backend/src/api/routes/__tests__/test_tasks_logs.py
View File

@@ -1,7 +1,9 @@
# [DEF:__tests__/test_tasks_logs:Module]
# @RELATION: VERIFIES -> ../tasks.py
# [DEF:test_tasks_logs_module:Module]
# @RELATION: VERIFIES -> [src.api.routes.tasks:Module]
# @COMPLEXITY: 2
# @SEMANTICS: tests, tasks, logs, api, contract, validation
# @PURPOSE: Contract testing for task logs API endpoints.
# [/DEF:__tests__/test_tasks_logs:Module]
# @LAYER: Domain (Tests)
import pytest
from fastapi import FastAPI
@@ -10,35 +12,39 @@ from unittest.mock import MagicMock
from src.dependencies import get_task_manager, has_permission
from src.api.routes.tasks import router
# @TEST_FIXTURE: mock_app
@pytest.fixture
def client():
app = FastAPI()
app.include_router(router, prefix="/tasks")
# Mock TaskManager
# @INVARIANT: unconstrained mock — no spec= enforced
mock_tm = MagicMock()
app.dependency_overrides[get_task_manager] = lambda: mock_tm
# Mock permissions (bypass for unit test)
app.dependency_overrides[has_permission("tasks", "READ")] = lambda: True
return TestClient(app), mock_tm
# @TEST_CONTRACT: get_task_logs_api -> Invariants
# @TEST_FIXTURE: valid_task_logs_request
# [DEF:test_get_task_logs_success:Function]
# @RELATION: BINDS_TO -> __tests__/test_tasks_logs
# @RELATION: BINDS_TO -> test_tasks_logs_module
# @PURPOSE: Validate task logs endpoint returns filtered logs for an existing task.
def test_get_task_logs_success(client):
tc, tm = client
# Setup mock task
mock_task = MagicMock()
tm.get_task.return_value = mock_task
tm.get_task_logs.return_value = [{"level": "INFO", "message": "msg1"}]
response = tc.get("/tasks/task-1/logs?level=INFO")
assert response.status_code == 200
assert response.json() == [{"level": "INFO", "message": "msg1"}]
tm.get_task.assert_called_with("task-1")
@@ -47,42 +53,56 @@ def test_get_task_logs_success(client):
assert args[0][0] == "task-1"
assert args[0][1].level == "INFO"
# @TEST_EDGE: task_not_found
# [/DEF:test_get_task_logs_success:Function]
# [DEF:test_get_task_logs_not_found:Function]
# @RELATION: BINDS_TO -> __tests__/test_tasks_logs
# @RELATION: BINDS_TO -> test_tasks_logs_module
# @PURPOSE: Validate task logs endpoint returns 404 when the task identifier is missing.
def test_get_task_logs_not_found(client):
tc, tm = client
tm.get_task.return_value = None
response = tc.get("/tasks/missing/logs")
assert response.status_code == 404
assert response.json()["detail"] == "Task not found"
# @TEST_EDGE: invalid_limit
# [/DEF:test_get_task_logs_not_found:Function]
# [DEF:test_get_task_logs_invalid_limit:Function]
# @RELATION: BINDS_TO -> __tests__/test_tasks_logs
# @RELATION: BINDS_TO -> test_tasks_logs_module
# @PURPOSE: Validate task logs endpoint enforces query validation for limit lower bound.
def test_get_task_logs_invalid_limit(client):
tc, tm = client
# limit=0 is ge=1 in Query
response = tc.get("/tasks/task-1/logs?limit=0")
assert response.status_code == 422
# @TEST_INVARIANT: response_purity
# [/DEF:test_get_task_logs_invalid_limit:Function]
# [DEF:test_get_task_log_stats_success:Function]
# @RELATION: BINDS_TO -> __tests__/test_tasks_logs
# @RELATION: BINDS_TO -> test_tasks_logs_module
# @PURPOSE: Validate log stats endpoint returns success payload for an existing task.
def test_get_task_log_stats_success(client):
tc, tm = client
tm.get_task.return_value = MagicMock()
tm.get_task_log_stats.return_value = {"INFO": 5, "ERROR": 1}
response = tc.get("/tasks/task-1/logs/stats")
assert response.status_code == 200
# response_model=LogStats might wrap this, but let's check basic structure
# assuming tm.get_task_log_stats returns something compatible with LogStats
# response_model=LogStats might wrap this, but let's check basic structure
# assuming tm.get_task_log_stats returns something compatible with LogStats
# [/DEF:test_get_task_log_stats_success:Function]
# [/DEF:test_tasks_logs_module:Module]

153
backend/src/api/routes/admin.py
View File

@@ -4,8 +4,9 @@
# @SEMANTICS: api, admin, users, roles, permissions
# @PURPOSE: Admin API endpoints for user and role management.
# @LAYER: API
# @RELATION: [USES] ->[backend.src.core.auth.repository.AuthRepository]
# @RELATION: [USES] ->[backend.src.dependencies.has_permission]
# @RELATION: [DEPENDS_ON] ->[AuthRepository:Class]
# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]
# @RELATION: [DEPENDS_ON] ->[has_permission:Function]
#
# @INVARIANT: All endpoints in this module require 'Admin' role or 'admin' scope.
@@ -17,9 +18,15 @@ from ...core.database import get_auth_db
from ...core.auth.repository import AuthRepository
from ...core.auth.security import get_password_hash
from ...schemas.auth import (
User as UserSchema, UserCreate, UserUpdate,
RoleSchema, RoleCreate, RoleUpdate, PermissionSchema,
ADGroupMappingSchema, ADGroupMappingCreate
User as UserSchema,
UserCreate,
UserUpdate,
RoleSchema,
RoleCreate,
RoleUpdate,
PermissionSchema,
ADGroupMappingSchema,
ADGroupMappingCreate,
)
from ...models.auth import User, Role, ADGroupMapping
from ...dependencies import has_permission, get_plugin_loader
@@ -36,6 +43,7 @@ from ...services.rbac_permission_catalog import (
router = APIRouter(prefix="/api/admin", tags=["admin"])
# [/DEF:router:Variable]
# [DEF:list_users:Function]
# @COMPLEXITY: 3
# @PURPOSE: Lists all registered users.
@@ -46,14 +54,16 @@ router = APIRouter(prefix="/api/admin", tags=["admin"])
# @RELATION: CALLS -> User
@router.get("/users", response_model=List[UserSchema])
async def list_users(
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:users", "READ"))
db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:users", "READ"))
):
with belief_scope("api.admin.list_users"):
users = db.query(User).all()
return users
# [/DEF:list_users:Function]
# [DEF:create_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Creates a new local user.
@@ -62,37 +72,40 @@ async def list_users(
# @PARAM: user_in (UserCreate) - New user data.
# @PARAM: db (Session) - Auth database session.
# @RETURN: UserSchema - The created user.
# @RELATION: CALLS -> AuthRepository
# @RELATION: [CALLS] ->[AuthRepository:Class]
@router.post("/users", response_model=UserSchema, status_code=status.HTTP_201_CREATED)
async def create_user(
user_in: UserCreate,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:users", "WRITE"))
_=Depends(has_permission("admin:users", "WRITE")),
):
with belief_scope("api.admin.create_user"):
repo = AuthRepository(db)
if repo.get_user_by_username(user_in.username):
raise HTTPException(status_code=400, detail="Username already exists")
new_user = User(
username=user_in.username,
email=user_in.email,
password_hash=get_password_hash(user_in.password),
auth_source="LOCAL",
is_active=user_in.is_active
is_active=user_in.is_active,
)
for role_name in user_in.roles:
role = repo.get_role_by_name(role_name)
if role:
new_user.roles.append(role)
db.add(new_user)
db.commit()
db.refresh(new_user)
return new_user
# [/DEF:create_user:Function]
# [DEF:update_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Updates an existing user.
@@ -108,33 +121,36 @@ async def update_user(
user_id: str,
user_in: UserUpdate,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:users", "WRITE"))
_=Depends(has_permission("admin:users", "WRITE")),
):
with belief_scope("api.admin.update_user"):
repo = AuthRepository(db)
user = repo.get_user_by_id(user_id)
if not user:
raise HTTPException(status_code=404, detail="User not found")
if user_in.email is not None:
user.email = user_in.email
if user_in.is_active is not None:
user.is_active = user_in.is_active
if user_in.password is not None:
user.password_hash = get_password_hash(user_in.password)
if user_in.roles is not None:
user.roles = []
for role_name in user_in.roles:
role = repo.get_role_by_name(role_name)
if role:
user.roles.append(role)
db.commit()
db.refresh(user)
return user
# [/DEF:update_user:Function]
# [DEF:delete_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Deletes a user.
@@ -148,37 +164,50 @@ async def update_user(
async def delete_user(
user_id: str,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:users", "WRITE"))
_=Depends(has_permission("admin:users", "WRITE")),
):
with belief_scope("api.admin.delete_user"):
logger.info(f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}")
logger.info(
f"[DEBUG] Attempting to delete user context={{'user_id': '{user_id}'}}"
)
repo = AuthRepository(db)
user = repo.get_user_by_id(user_id)
if not user:
logger.warning(f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}")
logger.warning(
f"[DEBUG] User not found for deletion context={{'user_id': '{user_id}'}}"
)
raise HTTPException(status_code=404, detail="User not found")
logger.info(f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}")
logger.info(
f"[DEBUG] Found user to delete context={{'username': '{user.username}'}}"
)
db.delete(user)
db.commit()
logger.info(f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}")
logger.info(
f"[DEBUG] Successfully deleted user context={{'user_id': '{user_id}'}}"
)
return None
# [/DEF:delete_user:Function]
# [DEF:list_roles:Function]
# @COMPLEXITY: 3
# @PURPOSE: Lists all available roles.
# @RETURN: List[RoleSchema] - List of roles.
# @RELATION: CALLS -> backend.src.models.auth.Role
# @RELATION: [CALLS] ->[Role:Class]
@router.get("/roles", response_model=List[RoleSchema])
async def list_roles(
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:roles", "READ"))
db: Session = Depends(get_auth_db), _=Depends(has_permission("admin:roles", "READ"))
):
with belief_scope("api.admin.list_roles"):
return db.query(Role).all()
# [/DEF:list_roles:Function]
# [DEF:create_role:Function]
# @COMPLEXITY: 3
# @PURPOSE: Creates a new system role with associated permissions.
@@ -188,35 +217,38 @@ async def list_roles(
# @PARAM: db (Session) - Auth database session.
# @RETURN: RoleSchema - The created role.
# @SIDE_EFFECT: Commits new role and associations to auth.db.
# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_permission_by_id
# @RELATION: [CALLS] ->[get_permission_by_id:Function]
@router.post("/roles", response_model=RoleSchema, status_code=status.HTTP_201_CREATED)
async def create_role(
role_in: RoleCreate,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:roles", "WRITE"))
_=Depends(has_permission("admin:roles", "WRITE")),
):
with belief_scope("api.admin.create_role"):
if db.query(Role).filter(Role.name == role_in.name).first():
raise HTTPException(status_code=400, detail="Role already exists")
new_role = Role(name=role_in.name, description=role_in.description)
repo = AuthRepository(db)
for perm_id_or_str in role_in.permissions:
perm = repo.get_permission_by_id(perm_id_or_str)
if not perm and ":" in perm_id_or_str:
res, act = perm_id_or_str.split(":", 1)
perm = repo.get_permission_by_resource_action(res, act)
if perm:
new_role.permissions.append(perm)
db.add(new_role)
db.commit()
db.refresh(new_role)
return new_role
# [/DEF:create_role:Function]
# [DEF:update_role:Function]
# @COMPLEXITY: 3
# @PURPOSE: Updates an existing role's metadata and permissions.
@@ -227,25 +259,25 @@ async def create_role(
# @PARAM: db (Session) - Auth database session.
# @RETURN: RoleSchema - The updated role.
# @SIDE_EFFECT: Commits updates to auth.db.
# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_role_by_id
# @RELATION: [CALLS] ->[get_role_by_id:Function]
@router.put("/roles/{role_id}", response_model=RoleSchema)
async def update_role(
role_id: str,
role_in: RoleUpdate,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:roles", "WRITE"))
_=Depends(has_permission("admin:roles", "WRITE")),
):
with belief_scope("api.admin.update_role"):
repo = AuthRepository(db)
role = repo.get_role_by_id(role_id)
if not role:
raise HTTPException(status_code=404, detail="Role not found")
if role_in.name is not None:
role.name = role_in.name
if role_in.description is not None:
role.description = role_in.description
if role_in.permissions is not None:
role.permissions = []
for perm_id_or_str in role_in.permissions:
@@ -253,15 +285,18 @@ async def update_role(
if not perm and ":" in perm_id_or_str:
res, act = perm_id_or_str.split(":", 1)
perm = repo.get_permission_by_resource_action(res, act)
if perm:
role.permissions.append(perm)
db.commit()
db.refresh(role)
return role
# [/DEF:update_role:Function]
# [DEF:delete_role:Function]
# @COMPLEXITY: 3
# @PURPOSE: Removes a role from the system.
@@ -271,24 +306,27 @@ async def update_role(
# @PARAM: db (Session) - Auth database session.
# @RETURN: None
# @SIDE_EFFECT: Deletes record from auth.db and commits.
# @RELATION: CALLS -> backend.src.core.auth.repository.AuthRepository.get_role_by_id
# @RELATION: [CALLS] ->[get_role_by_id:Function]
@router.delete("/roles/{role_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_role(
role_id: str,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:roles", "WRITE"))
_=Depends(has_permission("admin:roles", "WRITE")),
):
with belief_scope("api.admin.delete_role"):
repo = AuthRepository(db)
role = repo.get_role_by_id(role_id)
if not role:
raise HTTPException(status_code=404, detail="Role not found")
db.delete(role)
db.commit()
return None
# [/DEF:delete_role:Function]
# [DEF:list_permissions:Function]
# @COMPLEXITY: 3
# @PURPOSE: Lists all available system permissions for assignment.
@@ -299,12 +337,16 @@ async def delete_role(
@router.get("/permissions", response_model=List[PermissionSchema])
async def list_permissions(
db: Session = Depends(get_auth_db),
plugin_loader = Depends(get_plugin_loader),
_ = Depends(has_permission("admin:roles", "READ"))
plugin_loader=Depends(get_plugin_loader),
_=Depends(has_permission("admin:roles", "READ")),
):
with belief_scope("api.admin.list_permissions"):
declared_permissions = discover_declared_permissions(plugin_loader=plugin_loader)
inserted_count = sync_permission_catalog(db=db, declared_permissions=declared_permissions)
declared_permissions = discover_declared_permissions(
plugin_loader=plugin_loader
)
inserted_count = sync_permission_catalog(
db=db, declared_permissions=declared_permissions
)
if inserted_count > 0:
logger.info(
"[api.admin.list_permissions][Action] Synchronized %s missing RBAC permissions into auth catalog",
@@ -313,8 +355,11 @@ async def list_permissions(
repo = AuthRepository(db)
return repo.list_permissions()
# [/DEF:list_permissions:Function]
# [DEF:list_ad_mappings:Function]
# @COMPLEXITY: 3
# @PURPOSE: Lists all AD Group to Role mappings.
@@ -322,31 +367,37 @@ async def list_permissions(
@router.get("/ad-mappings", response_model=List[ADGroupMappingSchema])
async def list_ad_mappings(
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:settings", "READ"))
_=Depends(has_permission("admin:settings", "READ")),
):
with belief_scope("api.admin.list_ad_mappings"):
return db.query(ADGroupMapping).all()
# [/DEF:list_ad_mappings:Function]
# [DEF:create_ad_mapping:Function]
# @RELATION: CALLS -> AuthRepository
# @RELATION: [DEPENDS_ON] ->[ADGroupMapping:Class]
# @RELATION: [DEPENDS_ON] ->[get_auth_db:Function]
# @RELATION: [DEPENDS_ON] ->[has_permission:Function]
# @COMPLEXITY: 2
# @PURPOSE: Creates a new AD Group mapping.
@router.post("/ad-mappings", response_model=ADGroupMappingSchema)
async def create_ad_mapping(
mapping_in: ADGroupMappingCreate,
db: Session = Depends(get_auth_db),
_ = Depends(has_permission("admin:settings", "WRITE"))
_=Depends(has_permission("admin:settings", "WRITE")),
):
with belief_scope("api.admin.create_ad_mapping"):
new_mapping = ADGroupMapping(
ad_group=mapping_in.ad_group,
role_id=mapping_in.role_id
ad_group=mapping_in.ad_group, role_id=mapping_in.role_id
)
db.add(new_mapping)
db.commit()
db.refresh(new_mapping)
return new_mapping
# [/DEF:create_ad_mapping:Function]
# [/DEF:AdminApi:Module]
# [/DEF:AdminApi:Module]

59
backend/src/api/routes/assistant.py
View File

@@ -3,8 +3,10 @@
# @SEMANTICS: api, assistant, chat, command, confirmation
# @PURPOSE: API routes for LLM assistant command parsing and safe execution orchestration.
# @LAYER: API
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.assistant]
# @RELATION: DEPENDS_ON -> [TaskManager]
# @RELATION: DEPENDS_ON -> [AssistantMessageRecord]
# @RELATION: DEPENDS_ON -> [AssistantConfirmationRecord]
# @RELATION: DEPENDS_ON -> [AssistantAuditRecord]
# @INVARIANT: Risky operations are never executed without valid confirmation token.
from __future__ import annotations
@@ -54,8 +56,12 @@ git_service = GitService()
# [DEF:AssistantMessageRequest:Class]
# @COMPLEXITY: 1
# @PURPOSE: Input payload for assistant message endpoint.
# @DATA_CONTRACT: Input[conversation_id?:str, message:str(1..4000)] -> Output[AssistantMessageRequest]
# @RELATION: USED_BY -> [send_message]
# @SIDE_EFFECT: None (schema declaration only).
# @PRE: message length is within accepted bounds.
# @POST: Request object provides message text and optional conversation binding.
# @INVARIANT: message is always non-empty and no longer than 4000 characters.
class AssistantMessageRequest(BaseModel):
conversation_id: Optional[str] = None
message: str = Field(..., min_length=1, max_length=4000)
@@ -67,8 +73,12 @@ class AssistantMessageRequest(BaseModel):
# [DEF:AssistantAction:Class]
# @COMPLEXITY: 1
# @PURPOSE: UI action descriptor returned with assistant responses.
# @DATA_CONTRACT: Input[type:str, label:str, target?:str] -> Output[AssistantAction]
# @RELATION: USED_BY -> [AssistantMessageResponse]
# @SIDE_EFFECT: None (schema declaration only).
# @PRE: type and label are provided by orchestration logic.
# @POST: Action can be rendered as button on frontend.
# @INVARIANT: type and label are required for every UI action.
class AssistantAction(BaseModel):
type: str
label: str
@@ -81,8 +91,14 @@ class AssistantAction(BaseModel):
# [DEF:AssistantMessageResponse:Class]
# @COMPLEXITY: 1
# @PURPOSE: Output payload contract for assistant interaction endpoints.
# @DATA_CONTRACT: Input[conversation_id,response_id,state,text,intent?,confirmation_id?,task_id?,actions[],created_at] -> Output[AssistantMessageResponse]
# @RELATION: RETURNED_BY -> [send_message]
# @RELATION: RETURNED_BY -> [confirm_operation]
# @RELATION: RETURNED_BY -> [cancel_operation]
# @SIDE_EFFECT: None (schema declaration only).
# @PRE: Response includes deterministic state and text.
# @POST: Payload may include task_id/confirmation_id/actions for UI follow-up.
# @INVARIANT: created_at and state are always present in endpoint responses.
class AssistantMessageResponse(BaseModel):
conversation_id: str
response_id: str
@@ -101,8 +117,14 @@ class AssistantMessageResponse(BaseModel):
# [DEF:ConfirmationRecord:Class]
# @COMPLEXITY: 1
# @PURPOSE: In-memory confirmation token model for risky operation dispatch.
# @DATA_CONTRACT: Input[id,user_id,conversation_id,intent,dispatch,expires_at,state?,created_at] -> Output[ConfirmationRecord]
# @RELATION: USED_BY -> [send_message]
# @RELATION: USED_BY -> [confirm_operation]
# @RELATION: USED_BY -> [cancel_operation]
# @SIDE_EFFECT: None (schema declaration only).
# @PRE: intent/dispatch/user_id are populated at confirmation request time.
# @POST: Record tracks lifecycle state and expiry timestamp.
# @INVARIANT: state defaults to "pending" and expires_at bounds confirmation validity.
class ConfirmationRecord(BaseModel):
id: str
user_id: str
@@ -143,8 +165,12 @@ INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = {
# [DEF:_append_history:Function]
# @COMPLEXITY: 2
# @PURPOSE: Append conversation message to in-memory history buffer.
# @DATA_CONTRACT: Input[user_id,conversation_id,role,text,state?,task_id?,confirmation_id?] -> Output[None]
# @RELATION: UPDATES -> [CONVERSATIONS]
# @SIDE_EFFECT: Mutates in-memory CONVERSATIONS store for user conversation history.
# @PRE: user_id and conversation_id identify target conversation bucket.
# @POST: Message entry is appended to CONVERSATIONS key list.
# @INVARIANT: every appended entry includes generated message_id and created_at timestamp.
def _append_history(
user_id: str,
conversation_id: str,
@@ -177,8 +203,12 @@ def _append_history(
# [DEF:_persist_message:Function]
# @COMPLEXITY: 2
# @PURPOSE: Persist assistant/user message record to database.
# @DATA_CONTRACT: Input[Session,user_id,conversation_id,role,text,state?,task_id?,confirmation_id?,metadata?] -> Output[None]
# @RELATION: DEPENDS_ON -> [AssistantMessageRecord]
# @SIDE_EFFECT: Writes AssistantMessageRecord rows and commits or rollbacks the DB session.
# @PRE: db session is writable and message payload is serializable.
# @POST: Message row is committed or persistence failure is logged.
# @INVARIANT: failed persistence attempts always rollback before returning.
def _persist_message(
db: Session,
user_id: str,
@@ -215,8 +245,12 @@ def _persist_message(
# [DEF:_audit:Function]
# @COMPLEXITY: 2
# @PURPOSE: Append in-memory audit record for assistant decision trace.
# @DATA_CONTRACT: Input[user_id,payload:Dict[str,Any]] -> Output[None]
# @RELATION: UPDATES -> [ASSISTANT_AUDIT]
# @SIDE_EFFECT: Mutates in-memory ASSISTANT_AUDIT store and emits structured log event.
# @PRE: payload describes decision/outcome fields.
# @POST: ASSISTANT_AUDIT list for user contains new timestamped entry.
# @INVARIANT: persisted in-memory audit entry always contains created_at in ISO format.
def _audit(user_id: str, payload: Dict[str, Any]):
if user_id not in ASSISTANT_AUDIT:
ASSISTANT_AUDIT[user_id] = []
@@ -852,8 +886,13 @@ def _build_task_observability_summary(task: Any, config_manager: ConfigManager)
# [DEF:_parse_command:Function]
# @COMPLEXITY: 4
# @PURPOSE: Deterministically parse RU/EN command text into intent payload.
# @DATA_CONTRACT: Input[message:str, config_manager:ConfigManager] -> Output[Dict[str,Any]{domain,operation,entities,confidence,risk_level,requires_confirmation}]
# @RELATION: DEPENDS_ON -> [_extract_id]
# @RELATION: DEPENDS_ON -> [_is_production_env]
# @SIDE_EFFECT: None (pure parsing logic).
# @PRE: message contains raw user text and config manager resolves environments.
# @POST: Returns intent dict with domain/operation/entities/confidence/risk fields.
# @INVARIANT: every return path includes domain, operation, entities, confidence, risk_level, requires_confirmation.
def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any]:
text = message.strip()
lower = text.lower()
@@ -1575,8 +1614,15 @@ def _authorize_intent(intent: Dict[str, Any], current_user: User):
# [DEF:_dispatch_intent:Function]
# @COMPLEXITY: 5
# @PURPOSE: Execute parsed assistant intent via existing task/plugin/git services.
# @DATA_CONTRACT: Input[intent,current_user,task_manager,config_manager,db] -> Output[Tuple[text:str,task_id:Optional[str],actions:List[AssistantAction]]]
# @RELATION: DEPENDS_ON -> [_check_any_permission]
# @RELATION: DEPENDS_ON -> [_resolve_dashboard_id_entity]
# @RELATION: DEPENDS_ON -> [TaskManager]
# @RELATION: DEPENDS_ON -> [GitService]
# @SIDE_EFFECT: May enqueue tasks, invoke git operations, and query/update external service state.
# @PRE: intent operation is known and actor permissions are validated per operation.
# @POST: Returns response text, optional task id, and UI actions for follow-up.
# @INVARIANT: unsupported operations are rejected via HTTPException(400).
async def _dispatch_intent(
intent: Dict[str, Any],
current_user: User,
@@ -1974,9 +2020,18 @@ async def _dispatch_intent(
# [DEF:send_message:Function]
# @COMPLEXITY: 5
# @PURPOSE: Parse assistant command, enforce safety gates, and dispatch executable intent.
# @DATA_CONTRACT: Input[AssistantMessageRequest,User,TaskManager,ConfigManager,Session] -> Output[AssistantMessageResponse]
# @RELATION: DEPENDS_ON -> [_plan_intent_with_llm]
# @RELATION: DEPENDS_ON -> [_parse_command]
# @RELATION: DEPENDS_ON -> [_dispatch_intent]
# @RELATION: DEPENDS_ON -> [_append_history]
# @RELATION: DEPENDS_ON -> [_persist_message]
# @RELATION: DEPENDS_ON -> [_audit]
# @SIDE_EFFECT: Persists chat/audit state, mutates in-memory conversation and confirmation stores, and may create confirmation records.
# @PRE: Authenticated user is available and message text is non-empty.
# @POST: Response state is one of clarification/confirmation/started/success/denied/failed.
# @RETURN: AssistantMessageResponse with operation feedback and optional actions.
# @INVARIANT: non-safe operations are gated with confirmation before execution from this endpoint.
async def send_message(
request: AssistantMessageRequest,
current_user: User = Depends(get_current_user),

57
backend/src/api/routes/connections.py
View File

@@ -1,8 +1,9 @@
# [DEF:ConnectionsRouter:Module]
# @SEMANTICS: api, router, connections, database
# @PURPOSE: Defines the FastAPI router for managing external database connections.
# @COMPLEXITY: 3
# @LAYER: UI (API)
# @RELATION: Depends on SQLAlchemy session.
# @RELATION: DEPENDS_ON -> Session
# @CONSTRAINT: Must use belief_scope for logging.
# [SECTION: IMPORTS]
@@ -21,15 +22,22 @@ router = APIRouter()
# [DEF:_ensure_connections_schema:Function]
# @PURPOSE: Ensures the connection_configs table exists before CRUD access.
# @COMPLEXITY: 3
# @PRE: db is an active SQLAlchemy session.
# @POST: The current bind can safely query ConnectionConfig.
# @RELATION: CALLS -> ensure_connection_configs_table
def _ensure_connections_schema(db: Session):
with belief_scope("ConnectionsRouter.ensure_schema"):
ensure_connection_configs_table(db.get_bind())
# [/DEF:_ensure_connections_schema:Function]
# [DEF:ConnectionSchema:Class]
# @PURPOSE: Pydantic model for connection response.
# @COMPLEXITY: 3
# @RELATION: BINDS_TO -> ConnectionConfig
class ConnectionSchema(BaseModel):
id: str
name: str
@@ -42,10 +50,15 @@ class ConnectionSchema(BaseModel):
class Config:
orm_mode = True
# [/DEF:ConnectionSchema:Class]
# [DEF:ConnectionCreate:Class]
# @PURPOSE: Pydantic model for creating a connection.
# @COMPLEXITY: 3
# @RELATION: BINDS_TO -> ConnectionConfig
class ConnectionCreate(BaseModel):
name: str
type: str
@@ -54,60 +67,92 @@ class ConnectionCreate(BaseModel):
database: Optional[str] = None
username: Optional[str] = None
password: Optional[str] = None
# [/DEF:ConnectionCreate:Class]
# [DEF:list_connections:Function]
# @PURPOSE: Lists all saved connections.
# @COMPLEXITY: 3
# @PRE: Database session is active.
# @POST: Returns list of connection configs.
# @PARAM: db (Session) - Database session.
# @RETURN: List[ConnectionSchema] - List of connections.
# @RELATION: CALLS -> _ensure_connections_schema
# @RELATION: DEPENDS_ON -> ConnectionConfig
@router.get("", response_model=List[ConnectionSchema])
async def list_connections(db: Session = Depends(get_db)):
with belief_scope("ConnectionsRouter.list_connections"):
_ensure_connections_schema(db)
connections = db.query(ConnectionConfig).all()
return connections
# [/DEF:list_connections:Function]
# [DEF:create_connection:Function]
# @PURPOSE: Creates a new connection configuration.
# @COMPLEXITY: 3
# @PRE: Connection name is unique.
# @POST: Connection is saved to DB.
# @PARAM: connection (ConnectionCreate) - Config data.
# @PARAM: db (Session) - Database session.
# @RETURN: ConnectionSchema - Created connection.
# @RELATION: CALLS -> _ensure_connections_schema
# @RELATION: DEPENDS_ON -> ConnectionConfig
@router.post("", response_model=ConnectionSchema, status_code=status.HTTP_201_CREATED)
async def create_connection(connection: ConnectionCreate, db: Session = Depends(get_db)):
async def create_connection(
connection: ConnectionCreate, db: Session = Depends(get_db)
):
with belief_scope("ConnectionsRouter.create_connection", f"name={connection.name}"):
_ensure_connections_schema(db)
db_connection = ConnectionConfig(**connection.dict())
db.add(db_connection)
db.commit()
db.refresh(db_connection)
logger.info(f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}")
logger.info(
f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}"
)
return db_connection
# [/DEF:create_connection:Function]
# [DEF:delete_connection:Function]
# @PURPOSE: Deletes a connection configuration.
# @COMPLEXITY: 3
# @PRE: Connection ID exists.
# @POST: Connection is removed from DB.
# @PARAM: connection_id (str) - ID to delete.
# @PARAM: db (Session) - Database session.
# @RETURN: None.
# @RELATION: CALLS -> _ensure_connections_schema
# @RELATION: DEPENDS_ON -> ConnectionConfig
@router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_connection(connection_id: str, db: Session = Depends(get_db)):
with belief_scope("ConnectionsRouter.delete_connection", f"id={connection_id}"):
_ensure_connections_schema(db)
db_connection = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first()
db_connection = (
db.query(ConnectionConfig)
.filter(ConnectionConfig.id == connection_id)
.first()
)
if not db_connection:
logger.error(f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found")
logger.error(
f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found"
)
raise HTTPException(status_code=404, detail="Connection not found")
db.delete(db_connection)
db.commit()
logger.info(f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}")
logger.info(
f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}"
)
return
# [/DEF:delete_connection:Function]
# [/DEF:ConnectionsRouter:Module]

292
backend/src/api/routes/dataset_review.py
View File

@@ -25,7 +25,12 @@ from sqlalchemy.orm import Session
from src.core.database import get_db
from src.core.logger import belief_scope, logger
from src.dependencies import get_config_manager, get_current_user, get_task_manager, has_permission
from src.dependencies import (
get_config_manager,
get_current_user,
get_task_manager,
has_permission,
)
from src.models.auth import User
from src.models.dataset_review import (
AnswerKind,
@@ -84,6 +89,8 @@ class StartSessionRequest(BaseModel):
source_kind: str = Field(..., pattern="^(superset_link|dataset_selection)$")
source_input: str = Field(..., min_length=1)
environment_id: str = Field(..., min_length=1)
# [/DEF:StartSessionRequest:Class]
@@ -93,6 +100,8 @@ class StartSessionRequest(BaseModel):
class UpdateSessionRequest(BaseModel):
status: SessionStatus
note: Optional[str] = None
# [/DEF:UpdateSessionRequest:Class]
@@ -105,6 +114,8 @@ class SessionCollectionResponse(BaseModel):
page: int
page_size: int
has_next: bool
# [/DEF:SessionCollectionResponse:Class]
@@ -120,6 +131,8 @@ class ExportArtifactResponse(BaseModel):
created_by_user_id: str
created_at: Optional[str] = None
content: Dict[str, Any]
# [/DEF:ExportArtifactResponse:Class]
@@ -133,6 +146,8 @@ class FieldSemanticUpdateRequest(BaseModel):
display_format: Optional[str] = None
lock_field: bool = False
resolution_note: Optional[str] = None
# [/DEF:FieldSemanticUpdateRequest:Class]
@@ -141,6 +156,8 @@ class FieldSemanticUpdateRequest(BaseModel):
# @PURPOSE: Request DTO for thumbs up/down feedback persistence on AI-assisted content.
class FeedbackRequest(BaseModel):
feedback: str = Field(..., pattern="^(up|down)$")
# [/DEF:FeedbackRequest:Class]
@@ -151,6 +168,8 @@ class ClarificationAnswerRequest(BaseModel):
question_id: str = Field(..., min_length=1)
answer_kind: AnswerKind
answer_value: Optional[str] = None
# [/DEF:ClarificationAnswerRequest:Class]
@@ -165,6 +184,8 @@ class ClarificationSessionSummaryResponse(BaseModel):
resolved_count: int
remaining_count: int
summary_delta: Optional[str] = None
# [/DEF:ClarificationSessionSummaryResponse:Class]
@@ -174,6 +195,8 @@ class ClarificationSessionSummaryResponse(BaseModel):
class ClarificationStateResponse(BaseModel):
clarification_session: ClarificationSessionSummaryResponse
current_question: Optional[ClarificationQuestionDto] = None
# [/DEF:ClarificationStateResponse:Class]
@@ -184,6 +207,8 @@ class ClarificationAnswerResultResponse(BaseModel):
clarification_state: ClarificationStateResponse
session: SessionSummary
changed_findings: List[ValidationFindingDto]
# [/DEF:ClarificationAnswerResultResponse:Class]
@@ -193,6 +218,8 @@ class ClarificationAnswerResultResponse(BaseModel):
class FeedbackResponse(BaseModel):
target_id: str
feedback: str
# [/DEF:FeedbackResponse:Class]
@@ -201,6 +228,8 @@ class FeedbackResponse(BaseModel):
# @PURPOSE: Optional request DTO for explicit mapping approval audit notes.
class ApproveMappingRequest(BaseModel):
approval_note: Optional[str] = None
# [/DEF:ApproveMappingRequest:Class]
@@ -211,6 +240,8 @@ class BatchApproveSemanticItemRequest(BaseModel):
field_id: str = Field(..., min_length=1)
candidate_id: str = Field(..., min_length=1)
lock_field: bool = False
# [/DEF:BatchApproveSemanticItemRequest:Class]
@@ -219,6 +250,8 @@ class BatchApproveSemanticItemRequest(BaseModel):
# @PURPOSE: Request DTO for explicit batch semantic approvals inside one owned session scope.
class BatchApproveSemanticRequest(BaseModel):
items: List[BatchApproveSemanticItemRequest] = Field(..., min_length=1)
# [/DEF:BatchApproveSemanticRequest:Class]
@@ -228,6 +261,8 @@ class BatchApproveSemanticRequest(BaseModel):
class BatchApproveMappingRequest(BaseModel):
mapping_ids: List[str] = Field(..., min_length=1)
approval_note: Optional[str] = None
# [/DEF:BatchApproveMappingRequest:Class]
@@ -238,6 +273,8 @@ class PreviewEnqueueResultResponse(BaseModel):
session_id: str
preview_status: str
task_id: Optional[str] = None
# [/DEF:PreviewEnqueueResultResponse:Class]
@@ -246,6 +283,8 @@ class PreviewEnqueueResultResponse(BaseModel):
# @PURPOSE: Contract-compliant wrapper for execution mapping list responses.
class MappingCollectionResponse(BaseModel):
items: List[ExecutionMappingDto]
# [/DEF:MappingCollectionResponse:Class]
@@ -254,8 +293,13 @@ class MappingCollectionResponse(BaseModel):
# @PURPOSE: Request DTO for one manual execution-mapping override update without introducing unrelated bulk mutation semantics.
class UpdateExecutionMappingRequest(BaseModel):
effective_value: Optional[Any] = None
mapping_method: Optional[str] = Field(default=None, pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$")
mapping_method: Optional[str] = Field(
default=None,
pattern="^(manual_override|direct_match|heuristic_match|semantic_match)$",
)
transformation_note: Optional[str] = None
# [/DEF:UpdateExecutionMappingRequest:Class]
@@ -265,6 +309,8 @@ class UpdateExecutionMappingRequest(BaseModel):
class LaunchDatasetResponse(BaseModel):
run_context: DatasetRunContextDto
redirect_url: str
# [/DEF:LaunchDatasetResponse:Class]
@@ -280,6 +326,8 @@ def _require_auto_review_flag(config_manager=Depends(get_config_manager)) -> boo
detail="Dataset auto review feature is disabled",
)
return True
# [/DEF:_require_auto_review_flag:Function]
@@ -295,6 +343,8 @@ def _require_clarification_flag(config_manager=Depends(get_config_manager)) -> b
detail="Dataset clarification feature is disabled",
)
return True
# [/DEF:_require_clarification_flag:Function]
@@ -310,6 +360,8 @@ def _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool:
detail="Dataset execution feature is disabled",
)
return True
# [/DEF:_require_execution_flag:Function]
@@ -318,6 +370,8 @@ def _require_execution_flag(config_manager=Depends(get_config_manager)) -> bool:
# @PURPOSE: Build repository dependency for dataset review session aggregate access.
def _get_repository(db: Session = Depends(get_db)) -> DatasetReviewSessionRepository:
return DatasetReviewSessionRepository(db)
# [/DEF:_get_repository:Function]
@@ -335,6 +389,8 @@ def _get_orchestrator(
config_manager=config_manager,
task_manager=task_manager,
)
# [/DEF:_get_orchestrator:Function]
@@ -346,15 +402,20 @@ def _get_clarification_engine(
repository: DatasetReviewSessionRepository = Depends(_get_repository),
) -> ClarificationEngine:
return ClarificationEngine(repository=repository)
# [/DEF:_get_clarification_engine:Function]
# [DEF:_serialize_session_summary:Function]
# @COMPLEXITY: 2
# @PURPOSE: Map SQLAlchemy session aggregate root into stable API summary DTO.
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]
# @RELATION: [DEPENDS_ON] ->[SessionSummary]
def _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary:
return SessionSummary.model_validate(session, from_attributes=True)
# [/DEF:_serialize_session_summary:Function]
@@ -364,6 +425,8 @@ def _serialize_session_summary(session: DatasetReviewSession) -> SessionSummary:
# @RELATION: [DEPENDS_ON] ->[SessionDetail]
def _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail:
return SessionDetail.model_validate(session, from_attributes=True)
# [/DEF:_serialize_session_detail:Function]
@@ -373,6 +436,8 @@ def _serialize_session_detail(session: DatasetReviewSession) -> SessionDetail:
# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntryDto]
def _serialize_semantic_field(field: SemanticFieldEntry) -> SemanticFieldEntryDto:
return SemanticFieldEntryDto.model_validate(field, from_attributes=True)
# [/DEF:_serialize_semantic_field:Function]
@@ -401,6 +466,8 @@ def _serialize_clarification_question_payload(
"updated_at": datetime.utcnow(),
}
)
# [/DEF:_serialize_clarification_question_payload:Function]
@@ -421,8 +488,12 @@ def _serialize_clarification_state(
remaining_count=state.clarification_session.remaining_count,
summary_delta=state.clarification_session.summary_delta,
),
current_question=_serialize_clarification_question_payload(state.current_question),
current_question=_serialize_clarification_question_payload(
state.current_question
),
)
# [/DEF:_serialize_clarification_state:Function]
@@ -446,8 +517,12 @@ def _get_owned_session_or_404(
"Dataset review session not found in current ownership scope",
extra={"session_id": session_id, "user_id": current_user.id},
)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Session not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Session not found"
)
return session
# [/DEF:_get_owned_session_or_404:Function]
@@ -469,6 +544,8 @@ def _require_owner_mutation_scope(
detail="Only the owner can mutate dataset review state",
)
return session
# [/DEF:_require_owner_mutation_scope:Function]
@@ -492,6 +569,8 @@ def _record_session_event(
event_summary=event_summary,
event_details=event_details or {},
)
# [/DEF:_record_session_event:Function]
@@ -510,7 +589,11 @@ def _get_owned_mapping_or_404(
for mapping in session.execution_mappings:
if mapping.mapping_id == mapping_id:
return mapping
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Execution mapping not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Execution mapping not found"
)
# [/DEF:_get_owned_mapping_or_404:Function]
@@ -529,7 +612,11 @@ def _get_owned_field_or_404(
for field in session.semantic_fields:
if field.field_id == field_id:
return field
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic field not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND, detail="Semantic field not found"
)
# [/DEF:_get_owned_field_or_404:Function]
@@ -541,12 +628,17 @@ def _get_latest_clarification_session_or_404(
session: DatasetReviewSession,
) -> ClarificationSession:
if not session.clarification_sessions:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification session not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Clarification session not found",
)
return sorted(
session.clarification_sessions,
key=lambda item: (item.started_at, item.clarification_session_id),
reverse=True,
)[0]
# [/DEF:_get_latest_clarification_session_or_404:Function]
@@ -561,6 +653,8 @@ def _map_candidate_provenance(candidate: SemanticCandidate) -> FieldProvenance:
if str(candidate.match_type.value) == "generated":
return FieldProvenance.AI_GENERATED
return FieldProvenance.FUZZY_INFERRED
# [/DEF:_map_candidate_provenance:Function]
@@ -569,7 +663,9 @@ def _map_candidate_provenance(candidate: SemanticCandidate) -> FieldProvenance:
# @PURPOSE: Resolve the semantic source version for one accepted candidate from the loaded session aggregate.
# @RELATION: [DEPENDS_ON] ->[SemanticFieldEntry]
# @RELATION: [DEPENDS_ON] ->[SemanticSource]
def _resolve_candidate_source_version(field: SemanticFieldEntry, source_id: Optional[str]) -> Optional[str]:
def _resolve_candidate_source_version(
field: SemanticFieldEntry, source_id: Optional[str]
) -> Optional[str]:
if not source_id:
return None
session = getattr(field, "session", None)
@@ -579,6 +675,8 @@ def _resolve_candidate_source_version(field: SemanticFieldEntry, source_id: Opti
if source.source_id == source_id:
return source.source_version
return None
# [/DEF:_resolve_candidate_source_version:Function]
@@ -603,11 +701,18 @@ def _update_semantic_field_state(
selected_candidate = None
if request.candidate_id:
selected_candidate = next(
(candidate for candidate in field.candidates if candidate.candidate_id == request.candidate_id),
(
candidate
for candidate in field.candidates
if candidate.candidate_id == request.candidate_id
),
None,
)
if selected_candidate is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Semantic candidate not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Semantic candidate not found",
)
if has_manual_override:
field.verbose_name = request.verbose_name
@@ -631,7 +736,9 @@ def _update_semantic_field_state(
field.display_format = selected_candidate.proposed_display_format
field.provenance = _map_candidate_provenance(selected_candidate)
field.source_id = selected_candidate.source_id
field.source_version = _resolve_candidate_source_version(field, selected_candidate.source_id)
field.source_version = _resolve_candidate_source_version(
field, selected_candidate.source_id
)
field.confidence_rank = selected_candidate.candidate_rank
field.is_locked = bool(request.lock_field or field.is_locked)
field.has_conflict = len(field.candidates) > 1
@@ -649,6 +756,8 @@ def _update_semantic_field_state(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Provide candidate_id or at least one manual override field",
)
# [/DEF:_update_semantic_field_state:Function]
@@ -658,6 +767,8 @@ def _update_semantic_field_state(
# @RELATION: [DEPENDS_ON] ->[ExecutionMappingDto]
def _serialize_execution_mapping(mapping: ExecutionMapping) -> ExecutionMappingDto:
return ExecutionMappingDto.model_validate(mapping, from_attributes=True)
# [/DEF:_serialize_execution_mapping:Function]
@@ -667,6 +778,8 @@ def _serialize_execution_mapping(mapping: ExecutionMapping) -> ExecutionMappingD
# @RELATION: [DEPENDS_ON] ->[DatasetRunContextDto]
def _serialize_run_context(run_context) -> DatasetRunContextDto:
return DatasetRunContextDto.model_validate(run_context, from_attributes=True)
# [/DEF:_serialize_run_context:Function]
@@ -688,6 +801,8 @@ def _build_sql_lab_redirect_url(environment_url: str, sql_lab_session_ref: str)
detail="SQL Lab session reference is missing",
)
return f"{base_url}/superset/sqllab?queryId={session_ref}"
# [/DEF:_build_sql_lab_redirect_url:Function]
@@ -695,9 +810,13 @@ def _build_sql_lab_redirect_url(environment_url: str, sql_lab_session_ref: str)
# @COMPLEXITY: 2
# @PURPOSE: Produce session documentation export content from current persisted review state.
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]
def _build_documentation_export(session: DatasetReviewSession, export_format: ArtifactFormat) -> Dict[str, Any]:
def _build_documentation_export(
session: DatasetReviewSession, export_format: ArtifactFormat
) -> Dict[str, Any]:
profile = session.profile
findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))
findings = sorted(
session.findings, key=lambda item: (item.severity.value, item.code)
)
if export_format == ArtifactFormat.MARKDOWN:
lines = [
f"# Dataset Review: {session.dataset_ref}",
@@ -724,7 +843,8 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar
else:
content = {
"session": _serialize_session_summary(session).model_dump(mode="json"),
"profile": profile and {
"profile": profile
and {
"dataset_name": profile.dataset_name,
"business_summary": profile.business_summary,
"confidence_state": profile.confidence_state.value,
@@ -743,6 +863,8 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar
}
storage_ref = f"inline://dataset-review/{session.session_id}/documentation.json"
return {"storage_ref": storage_ref, "content": content}
# [/DEF:_build_documentation_export:Function]
@@ -750,8 +872,12 @@ def _build_documentation_export(session: DatasetReviewSession, export_format: Ar
# @COMPLEXITY: 2
# @PURPOSE: Produce validation-focused export content from persisted findings and readiness state.
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSession]
def _build_validation_export(session: DatasetReviewSession, export_format: ArtifactFormat) -> Dict[str, Any]:
findings = sorted(session.findings, key=lambda item: (item.severity.value, item.code))
def _build_validation_export(
session: DatasetReviewSession, export_format: ArtifactFormat
) -> Dict[str, Any]:
findings = sorted(
session.findings, key=lambda item: (item.severity.value, item.code)
)
if export_format == ArtifactFormat.MARKDOWN:
lines = [
f"# Validation Report: {session.dataset_ref}",
@@ -790,6 +916,8 @@ def _build_validation_export(session: DatasetReviewSession, export_format: Artif
}
storage_ref = f"inline://dataset-review/{session.session_id}/validation.json"
return {"storage_ref": storage_ref, "content": content}
# [/DEF:_build_validation_export:Function]
@@ -823,13 +951,15 @@ async def list_sessions(
page_size=page_size,
has_next=end < len(sessions),
)
# [/DEF:list_sessions:Function]
# [DEF:start_session:Function]
# @COMPLEXITY: 4
# @PURPOSE: Start a new dataset review session from a Superset link or dataset selection.
# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.start_session]
# @RELATION: [CALLS] ->[start_session:Function]
# @PRE: feature flag enabled, user authenticated, and request body valid.
# @POST: returns persisted session summary scoped to the authenticated user.
# @SIDE_EFFECT: persists session/profile/findings and may enqueue recovery task.
@@ -864,10 +994,16 @@ async def start_session(
extra={"user_id": current_user.id, "error": str(exc)},
)
detail = str(exc)
status_code = status.HTTP_404_NOT_FOUND if detail == "Environment not found" else status.HTTP_400_BAD_REQUEST
status_code = (
status.HTTP_404_NOT_FOUND
if detail == "Environment not found"
else status.HTTP_400_BAD_REQUEST
)
raise HTTPException(status_code=status_code, detail=detail) from exc
return _serialize_session_summary(result.session)
# [/DEF:start_session:Function]
@@ -891,6 +1027,8 @@ async def get_session_detail(
with belief_scope("dataset_review.get_session_detail"):
session = _get_owned_session_or_404(repository, session_id, current_user)
return _serialize_session_detail(session)
# [/DEF:get_session_detail:Function]
@@ -923,7 +1061,11 @@ async def update_session(
session.status = request.status
if request.status == SessionStatus.PAUSED:
session.recommended_action = RecommendedAction.RESUME_SESSION
elif request.status in {SessionStatus.ARCHIVED, SessionStatus.CANCELLED, SessionStatus.COMPLETED}:
elif request.status in {
SessionStatus.ARCHIVED,
SessionStatus.CANCELLED,
SessionStatus.COMPLETED,
}:
session.active_task_id = None
repository.db.commit()
@@ -937,6 +1079,8 @@ async def update_session(
event_details={"status": session.status.value},
)
return _serialize_session_summary(session)
# [/DEF:update_session:Function]
@@ -992,6 +1136,8 @@ async def delete_session(
event_details={"hard_delete": False},
)
return Response(status_code=status.HTTP_204_NO_CONTENT)
# [/DEF:delete_session:Function]
@@ -1019,7 +1165,10 @@ async def export_documentation(
):
with belief_scope("dataset_review.export_documentation"):
if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Only json and markdown exports are supported")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Only json and markdown exports are supported",
)
session = _get_owned_session_or_404(repository, session_id, current_user)
export_payload = _build_documentation_export(session, format)
return ExportArtifactResponse(
@@ -1031,6 +1180,8 @@ async def export_documentation(
created_by_user_id=current_user.id,
content=export_payload["content"],
)
# [/DEF:export_documentation:Function]
@@ -1058,7 +1209,10 @@ async def export_validation(
):
with belief_scope("dataset_review.export_validation"):
if format not in {ArtifactFormat.JSON, ArtifactFormat.MARKDOWN}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Only json and markdown exports are supported")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Only json and markdown exports are supported",
)
session = _get_owned_session_or_404(repository, session_id, current_user)
export_payload = _build_validation_export(session, format)
return ExportArtifactResponse(
@@ -1070,13 +1224,15 @@ async def export_validation(
created_by_user_id=current_user.id,
content=export_payload["content"],
)
# [/DEF:export_validation:Function]
# [DEF:get_clarification_state:Function]
# @COMPLEXITY: 4
# @PURPOSE: Return the current clarification session summary and one active question payload.
# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload]
# @RELATION: [CALLS] ->[build_question_payload:Function]
# @PRE: Session is accessible to current user and clarification feature is enabled.
# @POST: Returns at most one active clarification question with why_it_matters, current_guess, and ordered options.
# @SIDE_EFFECT: May normalize clarification pointer and readiness state in persistence.
@@ -1108,13 +1264,15 @@ async def get_clarification_state(
changed_findings=[],
)
)
# [/DEF:get_clarification_state:Function]
# [DEF:resume_clarification:Function]
# @COMPLEXITY: 4
# @PURPOSE: Resume clarification mode on the highest-priority unresolved question for an owned session.
# @RELATION: [CALLS] ->[ClarificationEngine.build_question_payload]
# @RELATION: [CALLS] ->[build_question_payload:Function]
# @PRE: Session belongs to the current owner and clarification feature is enabled.
# @POST: Clarification session enters active state with one current question or completes deterministically when no unresolved items remain.
# @SIDE_EFFECT: Mutates clarification pointer, readiness, and recommended action.
@@ -1147,13 +1305,15 @@ async def resume_clarification(
changed_findings=[],
)
)
# [/DEF:resume_clarification:Function]
# [DEF:record_clarification_answer:Function]
# @COMPLEXITY: 4
# @PURPOSE: Persist one clarification answer before advancing the active pointer or readiness state.
# @RELATION: [CALLS] ->[ClarificationEngine.record_answer]
# @RELATION: [CALLS] ->[record_answer:Function]
# @PRE: Target question is the session's active clarification question and current user owns the session.
# @POST: Answer is persisted, changed findings are returned, and unresolved skipped/expert-review questions remain visible.
# @SIDE_EFFECT: Inserts answer row and mutates clarification/session state.
@@ -1188,7 +1348,9 @@ async def record_clarification_answer(
)
)
except ValueError as exc:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)
) from exc
return ClarificationAnswerResultResponse(
clarification_state=_serialize_clarification_state(result),
@@ -1198,6 +1360,8 @@ async def record_clarification_answer(
for item in result.changed_findings
],
)
# [/DEF:record_clarification_answer:Function]
@@ -1246,6 +1410,8 @@ async def update_field_semantic(
},
)
return _serialize_semantic_field(field)
# [/DEF:update_field_semantic:Function]
@@ -1288,6 +1454,8 @@ async def lock_field_semantic(
event_details={"field_id": field.field_id},
)
return _serialize_semantic_field(field)
# [/DEF:lock_field_semantic:Function]
@@ -1333,6 +1501,8 @@ async def unlock_field_semantic(
event_details={"field_id": field.field_id},
)
return _serialize_semantic_field(field)
# [/DEF:unlock_field_semantic:Function]
@@ -1367,7 +1537,9 @@ async def approve_batch_semantic_fields(
field = _get_owned_field_or_404(session, item.field_id)
updated_field = _update_semantic_field_state(
field,
FieldSemanticUpdateRequest(candidate_id=item.candidate_id, lock_field=item.lock_field),
FieldSemanticUpdateRequest(
candidate_id=item.candidate_id, lock_field=item.lock_field
),
changed_by="user",
)
updated_fields.append(updated_field)
@@ -1387,6 +1559,8 @@ async def approve_batch_semantic_fields(
},
)
return [_serialize_semantic_field(field) for field in updated_fields]
# [/DEF:approve_batch_semantic_fields:Function]
@@ -1415,8 +1589,13 @@ async def list_execution_mappings(
with belief_scope("dataset_review.list_execution_mappings"):
session = _get_owned_session_or_404(repository, session_id, current_user)
return MappingCollectionResponse(
items=[_serialize_execution_mapping(item) for item in session.execution_mappings]
items=[
_serialize_execution_mapping(item)
for item in session.execution_mappings
]
)
# [/DEF:list_execution_mappings:Function]
@@ -1456,7 +1635,9 @@ async def update_execution_mapping(
)
mapping.effective_value = request.effective_value
mapping.mapping_method = MappingMethod(request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value)
mapping.mapping_method = MappingMethod(
request.mapping_method or MappingMethod.MANUAL_OVERRIDE.value
)
mapping.transformation_note = request.transformation_note
mapping.approval_state = ApprovalState.APPROVED
mapping.approved_by_user_id = current_user.id
@@ -1491,6 +1672,8 @@ async def update_execution_mapping(
},
)
return _serialize_execution_mapping(mapping)
# [/DEF:update_execution_mapping:Function]
@@ -1544,6 +1727,8 @@ async def approve_execution_mapping(
},
)
return _serialize_execution_mapping(mapping)
# [/DEF:approve_execution_mapping:Function]
@@ -1603,13 +1788,15 @@ async def approve_batch_execution_mappings(
},
)
return [_serialize_execution_mapping(mapping) for mapping in updated_mappings]
# [/DEF:approve_batch_execution_mappings:Function]
# [DEF:trigger_preview_generation:Function]
# @COMPLEXITY: 4
# @PURPOSE: Trigger Superset-side preview compilation for the current owned execution context.
# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.prepare_launch_preview]
# @RELATION: [CALLS] ->[prepare_launch_preview:Function]
# @PRE: Session belongs to the current owner and required mapping inputs are available.
# @POST: Returns the compiled preview directly for synchronous success or enqueue-state shape when preview generation remains pending.
# @SIDE_EFFECT: Persists preview attempt and updates readiness state.
@@ -1640,8 +1827,10 @@ async def trigger_preview_generation(
except ValueError as exc:
detail = str(exc)
status_code = (
status.HTTP_404_NOT_FOUND if detail in {"Session not found", "Environment not found"}
else status.HTTP_409_CONFLICT if detail.startswith("Preview blocked:")
status.HTTP_404_NOT_FOUND
if detail in {"Session not found", "Environment not found"}
else status.HTTP_409_CONFLICT
if detail.startswith("Preview blocked:")
else status.HTTP_400_BAD_REQUEST
)
raise HTTPException(status_code=status_code, detail=detail) from exc
@@ -1656,13 +1845,15 @@ async def trigger_preview_generation(
response.status_code = status.HTTP_200_OK
return CompiledPreviewDto.model_validate(result.preview, from_attributes=True)
# [/DEF:trigger_preview_generation:Function]
# [DEF:launch_dataset:Function]
# @COMPLEXITY: 4
# @PURPOSE: Execute the current owned session launch handoff through the orchestrator and return audited SQL Lab run context.
# @RELATION: [CALLS] ->[DatasetReviewOrchestrator.launch_dataset]
# @RELATION: [CALLS] ->[launch_dataset:Function]
# @PRE: Session belongs to the current owner, execution feature is enabled, and launch gates are satisfied or a deterministic conflict is returned.
# @POST: Returns persisted run context plus redirect URL when launch handoff is accepted.
# @SIDE_EFFECT: Persists launch audit snapshot and may trigger SQL Lab session creation.
@@ -1703,7 +1894,9 @@ async def launch_dataset(
raise HTTPException(status_code=status_code, detail=detail) from exc
environment = config_manager.get_environment(result.session.environment_id)
environment_url = getattr(environment, "url", "") if environment is not None else ""
environment_url = (
getattr(environment, "url", "") if environment is not None else ""
)
return LaunchDatasetResponse(
run_context=_serialize_run_context(result.run_context),
redirect_url=_build_sql_lab_redirect_url(
@@ -1711,6 +1904,8 @@ async def launch_dataset(
sql_lab_session_ref=result.run_context.sql_lab_session_ref,
),
)
# [/DEF:launch_dataset:Function]
@@ -1752,6 +1947,8 @@ async def record_field_feedback(
event_details={"field_id": field.field_id, "feedback": request.feedback},
)
return FeedbackResponse(target_id=field.field_id, feedback=request.feedback)
# [/DEF:record_field_feedback:Function]
@@ -1784,13 +1981,23 @@ async def record_clarification_feedback(
_require_owner_mutation_scope(session, current_user)
clarification_session = _get_latest_clarification_session_or_404(session)
question = next(
(item for item in clarification_session.questions if item.question_id == question_id),
(
item
for item in clarification_session.questions
if item.question_id == question_id
),
None,
)
if question is None:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Clarification question not found")
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="Clarification question not found",
)
if question.answer is None:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Clarification answer not found")
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Clarification answer not found",
)
question.answer.user_feedback = request.feedback
repository.db.commit()
_record_session_event(
@@ -1799,9 +2006,16 @@ async def record_clarification_feedback(
current_user,
event_type="clarification_feedback_recorded",
event_summary="Clarification feedback persisted",
event_details={"question_id": question.question_id, "feedback": request.feedback},
event_details={
"question_id": question.question_id,
"feedback": request.feedback,
},
)
return FeedbackResponse(target_id=question.question_id, feedback=request.feedback)
return FeedbackResponse(
target_id=question.question_id, feedback=request.feedback
)
# [/DEF:record_clarification_feedback:Function]
# [/DEF:DatasetReviewApi:Module]
# [/DEF:DatasetReviewApi:Module]

97
backend/src/api/routes/llm.py
View File

@@ -31,21 +31,25 @@ def _is_valid_runtime_api_key(value: Optional[str]) -> bool:
if key in {"********", "EMPTY_OR_NONE"}:
return False
return len(key) >= 16
# [/DEF:_is_valid_runtime_api_key:Function]
# [DEF:get_providers:Function]
# @PURPOSE: Retrieve all LLM provider configurations.
# @PRE: User is authenticated.
# @POST: Returns list of LLMProviderConfig.
@router.get("/providers", response_model=List[LLMProviderConfig])
async def get_providers(
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)
):
"""
Get all LLM provider configurations.
"""
logger.info(f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}")
logger.info(
f"[llm_routes][get_providers][Action] Fetching providers for user: {current_user.username}"
)
service = LLMProviderService(db)
providers = service.get_all_providers()
return [
@@ -56,9 +60,12 @@ async def get_providers(
base_url=p.base_url,
api_key="********",
default_model=p.default_model,
is_active=p.is_active
) for p in providers
is_active=p.is_active,
)
for p in providers
]
# [/DEF:get_providers:Function]
@@ -68,8 +75,7 @@ async def get_providers(
# @POST: configured=true only when an active provider with valid decrypted key exists.
@router.get("/status")
async def get_llm_status(
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
current_user: User = Depends(get_current_active_user), db: Session = Depends(get_db)
):
service = LLMProviderService(db)
providers = service.get_all_providers()
@@ -90,17 +96,22 @@ async def get_llm_status(
"provider_type": active_provider.provider_type,
"default_model": active_provider.default_model,
}
# [/DEF:get_llm_status:Function]
# [DEF:create_provider:Function]
# @PURPOSE: Create a new LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
# @POST: Returns the created LLMProviderConfig.
@router.post("/providers", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED)
@router.post(
"/providers", response_model=LLMProviderConfig, status_code=status.HTTP_201_CREATED
)
async def create_provider(
config: LLMProviderConfig,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
"""
Create a new LLM provider configuration.
@@ -114,10 +125,13 @@ async def create_provider(
base_url=provider.base_url,
api_key="********",
default_model=provider.default_model,
is_active=provider.is_active
is_active=provider.is_active,
)
# [/DEF:create_provider:Function]
# [DEF:update_provider:Function]
# @PURPOSE: Update an existing LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
@@ -127,7 +141,7 @@ async def update_provider(
provider_id: str,
config: LLMProviderConfig,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
"""
Update an existing LLM provider configuration.
@@ -136,7 +150,7 @@ async def update_provider(
provider = service.update_provider(provider_id, config)
if not provider:
raise HTTPException(status_code=404, detail="Provider not found")
return LLMProviderConfig(
id=provider.id,
provider_type=LLMProviderType(provider.provider_type),
@@ -144,10 +158,13 @@ async def update_provider(
base_url=provider.base_url,
api_key="********",
default_model=provider.default_model,
is_active=provider.is_active
is_active=provider.is_active,
)
# [/DEF:update_provider:Function]
# [DEF:delete_provider:Function]
# @PURPOSE: Delete an LLM provider configuration.
# @PRE: User is authenticated and has admin permissions.
@@ -156,7 +173,7 @@ async def update_provider(
async def delete_provider(
provider_id: str,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
"""
Delete an LLM provider configuration.
@@ -165,8 +182,11 @@ async def delete_provider(
if not service.delete_provider(provider_id):
raise HTTPException(status_code=404, detail="Provider not found")
return
# [/DEF:delete_provider:Function]
# [DEF:test_connection:Function]
# @PURPOSE: Test connection to an LLM provider.
# @PRE: User is authenticated.
@@ -175,76 +195,87 @@ async def delete_provider(
async def test_connection(
provider_id: str,
current_user: User = Depends(get_current_active_user),
db: Session = Depends(get_db)
db: Session = Depends(get_db),
):
logger.info(f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}")
logger.info(
f"[llm_routes][test_connection][Action] Testing connection for provider_id: {provider_id}"
)
"""
Test connection to an LLM provider.
"""
from ...plugins.llm_analysis.service import LLMClient
service = LLMProviderService(db)
db_provider = service.get_provider(provider_id)
if not db_provider:
raise HTTPException(status_code=404, detail="Provider not found")
api_key = service.get_decrypted_api_key(provider_id)
# Check if API key was successfully decrypted
if not api_key:
logger.error(f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}")
logger.error(
f"[llm_routes][test_connection] Failed to decrypt API key for provider {provider_id}"
)
raise HTTPException(
status_code=500,
detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key."
detail="Failed to decrypt API key. The provider may have been encrypted with a different encryption key. Please update the provider with a new API key.",
)
client = LLMClient(
provider_type=LLMProviderType(db_provider.provider_type),
api_key=api_key,
base_url=db_provider.base_url,
default_model=db_provider.default_model
default_model=db_provider.default_model,
)
try:
await client.test_runtime_connection()
return {"success": True, "message": "Connection successful"}
except Exception as e:
return {"success": False, "error": str(e)}
# [/DEF:test_connection:Function]
# [DEF:test_provider_config:Function]
# @PURPOSE: Test connection with a provided configuration (not yet saved).
# @PRE: User is authenticated.
# @POST: Returns success status and message.
@router.post("/providers/test")
async def test_provider_config(
config: LLMProviderConfig,
current_user: User = Depends(get_current_active_user)
config: LLMProviderConfig, current_user: User = Depends(get_current_active_user)
):
"""
Test connection with a provided configuration.
"""
from ...plugins.llm_analysis.service import LLMClient
logger.info(f"[llm_routes][test_provider_config][Action] Testing config for {config.name}")
logger.info(
f"[llm_routes][test_provider_config][Action] Testing config for {config.name}"
)
# Check if API key is provided
if not config.api_key or config.api_key == "********":
raise HTTPException(
status_code=400,
detail="API key is required for testing connection"
status_code=400, detail="API key is required for testing connection"
)
client = LLMClient(
provider_type=config.provider_type,
api_key=config.api_key,
base_url=config.base_url,
default_model=config.default_model
default_model=config.default_model,
)
try:
await client.test_runtime_connection()
return {"success": True, "message": "Connection successful"}
except Exception as e:
return {"success": False, "error": str(e)}
# [/DEF:test_provider_config:Function]
# [/DEF:backend/src/api/routes/llm.py]
# [/DEF:backend/src/api/routes/llm.py:Module]

47
backend/src/api/routes/reports.py
View File

@@ -3,8 +3,10 @@
# @SEMANTICS: api, reports, list, detail, pagination, filters
# @PURPOSE: FastAPI router for unified task report list and detail retrieval endpoints.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [backend.src.services.reports.report_service.ReportsService]
# @RELATION: DEPENDS_ON -> [AppDependencies]
# @RELATION: [DEPENDS_ON] ->[ReportsService:Class]
# @RELATION: [DEPENDS_ON] ->[get_task_manager:Function]
# @RELATION: [DEPENDS_ON] ->[get_clean_release_repository:Function]
# @RELATION: [DEPENDS_ON] ->[has_permission:Function]
# @INVARIANT: Endpoints are read-only and do not trigger long-running tasks.
# @PRE: Reports service and dependencies are initialized.
# @POST: Router is configured and endpoints are ready for registration.
@@ -17,10 +19,20 @@ from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from ...dependencies import get_task_manager, has_permission, get_clean_release_repository
from ...dependencies import (
get_task_manager,
has_permission,
get_clean_release_repository,
)
from ...core.task_manager import TaskManager
from ...core.logger import belief_scope
from ...models.report import ReportCollection, ReportDetailView, ReportQuery, ReportStatus, TaskType
from ...models.report import (
ReportCollection,
ReportDetailView,
ReportQuery,
ReportStatus,
TaskType,
)
from ...services.clean_release.repository import CleanReleaseRepository
from ...services.reports.report_service import ReportsService
# [/SECTION]
@@ -60,6 +72,8 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List:
},
)
return parsed
# [/DEF:_parse_csv_enum_list:Function]
@@ -69,6 +83,9 @@ def _parse_csv_enum_list(raw: Optional[str], enum_cls, field_name: str) -> List:
# @PRE: authenticated/authorized request and validated query params.
# @POST: returns {items,total,page,page_size,has_next,applied_filters}.
# @POST: deterministic error payload for invalid filters.
# @RELATION: [CALLS] ->[_parse_csv_enum_list:Function]
# @RELATION: [DEPENDS_ON] ->[ReportQuery:Class]
# @RELATION: [DEPENDS_ON] ->[ReportsService:Class]
#
# @TEST_CONTRACT: ListReportsApi ->
# {
@@ -95,7 +112,9 @@ async def list_reports(
sort_by: str = Query("updated_at"),
sort_order: str = Query("desc"),
task_manager: TaskManager = Depends(get_task_manager),
clean_release_repository: CleanReleaseRepository = Depends(get_clean_release_repository),
clean_release_repository: CleanReleaseRepository = Depends(
get_clean_release_repository
),
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("list_reports"):
@@ -125,8 +144,12 @@ async def list_reports(
},
)
service = ReportsService(task_manager, clean_release_repository=clean_release_repository)
service = ReportsService(
task_manager, clean_release_repository=clean_release_repository
)
return service.list_reports(query)
# [/DEF:list_reports:Function]
@@ -139,11 +162,15 @@ async def list_reports(
async def get_report_detail(
report_id: str,
task_manager: TaskManager = Depends(get_task_manager),
clean_release_repository: CleanReleaseRepository = Depends(get_clean_release_repository),
clean_release_repository: CleanReleaseRepository = Depends(
get_clean_release_repository
),
_=Depends(has_permission("tasks", "READ")),
):
with belief_scope("get_report_detail", f"report_id={report_id}"):
service = ReportsService(task_manager, clean_release_repository=clean_release_repository)
service = ReportsService(
task_manager, clean_release_repository=clean_release_repository
)
detail = service.get_report_detail(report_id)
if not detail:
raise HTTPException(
@@ -151,6 +178,8 @@ async def get_report_detail(
detail={"message": "Report not found", "code": "REPORT_NOT_FOUND"},
)
return detail
# [/DEF:get_report_detail:Function]
# [/DEF:ReportsRouter:Module]
# [/DEF:ReportsRouter:Module]

11
backend/src/api/routes/settings.py
View File

@@ -4,8 +4,9 @@
# @SEMANTICS: settings, api, router, fastapi
# @PURPOSE: Provides API endpoints for managing application settings and Superset environments.
# @LAYER: UI (API)
# @RELATION: DEPENDS_ON -> [backend.src.core.config_manager.ConfigManager]
# @RELATION: DEPENDS_ON -> [backend.src.core.config_models]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[get_config_manager:Function]
# @RELATION: [DEPENDS_ON] ->[has_permission:Function]
#
# @INVARIANT: All settings changes must be persisted via ConfigManager.
# @PUBLIC_API: router
@@ -413,6 +414,12 @@ class ConsolidatedSettingsResponse(BaseModel):
# @PRE: Config manager is available.
# @POST: Returns all consolidated settings.
# @RETURN: ConsolidatedSettingsResponse - All settings categories.
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[LLMProviderService]
# @RELATION: [DEPENDS_ON] ->[AppConfigRecord]
# @RELATION: [DEPENDS_ON] ->[SessionLocal]
# @RELATION: [DEPENDS_ON] ->[has_permission:Function]
# @RELATION: [DEPENDS_ON] ->[normalize_llm_settings:Function]
@router.get("/consolidated", response_model=ConsolidatedSettingsResponse)
async def get_consolidated_settings(
config_manager: ConfigManager = Depends(get_config_manager),

27
backend/src/core/__tests__/test_config_manager_compat.py
View File

@@ -23,6 +23,8 @@ def test_get_payload_preserves_legacy_sections():
assert payload["settings"]["migration_sync_cron"] == "0 2 * * *"
assert payload["notifications"]["smtp"]["host"] == "mail.local"
# [/DEF:test_get_payload_preserves_legacy_sections:Function]
@@ -71,10 +73,22 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
credentials_id="legacy-user",
)
# [DEF:_FakeQuery:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal query stub returning hardcoded existing environment record list for sync tests.
# @INVARIANT: all() always returns [existing_record]; no parameterization or filtering.
class _FakeQuery:
def all(self):
return [existing_record]
# [/DEF:_FakeQuery:Class]
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions.
# @INVARIANT: query() always returns _FakeQuery; no real DB interaction.
class _FakeSession:
def query(self, model):
return _FakeQuery()
@@ -85,6 +99,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
def delete(self, value):
deleted_records.append(value)
# [/DEF:_FakeSession:Class]
session = _FakeSession()
config = AppConfig(
environments=[
@@ -107,6 +123,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
assert added_records[0].url == "http://superset.local"
assert added_records[0].credentials_id == "demo"
assert deleted_records == [existing_record]
# [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
@@ -123,6 +141,11 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
closed = {"value": False}
committed = {"value": False}
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload]
# @COMPLEXITY: 1
# @PURPOSE: Minimal session stub tracking commit/close signals for config load lifecycle assertions.
# @INVARIANT: No query or add semantics; only lifecycle signal tracking.
class _FakeSession:
def commit(self):
committed["value"] = True
@@ -130,6 +153,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
def close(self):
closed["value"] = True
# [/DEF:_FakeSession:Class]
fake_session = _FakeSession()
fake_record = SimpleNamespace(
id="global",
@@ -163,6 +188,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
assert sync_calls[0][1].environments[0].id == "dev"
assert committed["value"] is True
assert closed["value"] is True
# [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
# [/DEF:TestConfigManagerCompat:Module]

73
backend/src/core/__tests__/test_throttled_scheduler.py
View File

@@ -7,8 +7,10 @@ from src.core.scheduler import ThrottledSchedulerConfigurator
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic.
# [DEF:test_calculate_schedule_even_distribution:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate even spacing across a two-hour scheduling window for three tasks.
def test_calculate_schedule_even_distribution():
"""
@TEST_SCENARIO: 3 tasks in a 2-hour window should be spaced 1 hour apart.
@@ -17,18 +19,23 @@ def test_calculate_schedule_even_distribution():
end = time(3, 0)
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
assert schedule[1] == datetime(2024, 1, 1, 2, 0)
assert schedule[2] == datetime(2024, 1, 1, 3, 0)
# [/DEF:test_calculate_schedule_even_distribution:Function]
# [DEF:test_calculate_schedule_midnight_crossing:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate scheduler correctly rolls timestamps into the next day across midnight.
def test_calculate_schedule_midnight_crossing():
"""
@TEST_SCENARIO: Window from 23:00 to 01:00 (next day).
@@ -37,18 +44,23 @@ def test_calculate_schedule_midnight_crossing():
end = time(1, 0)
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 23, 0)
assert schedule[1] == datetime(2024, 1, 2, 0, 0)
assert schedule[2] == datetime(2024, 1, 2, 1, 0)
# [/DEF:test_calculate_schedule_midnight_crossing:Function]
# [DEF:test_calculate_schedule_single_task:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate single-task schedule returns only the window start timestamp.
def test_calculate_schedule_single_task():
"""
@TEST_SCENARIO: Single task should be scheduled at start time.
@@ -57,16 +69,21 @@ def test_calculate_schedule_single_task():
end = time(2, 0)
dashboards = ["d1"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 1
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
# [/DEF:test_calculate_schedule_single_task:Function]
# [DEF:test_calculate_schedule_empty_list:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate empty dashboard list produces an empty schedule.
def test_calculate_schedule_empty_list():
"""
@TEST_SCENARIO: Empty dashboard list returns empty schedule.
@@ -75,15 +92,20 @@ def test_calculate_schedule_empty_list():
end = time(2, 0)
dashboards = []
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert schedule == []
# [/DEF:test_calculate_schedule_empty_list:Function]
# [DEF:test_calculate_schedule_zero_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate zero-length window schedules all tasks at identical start timestamp.
def test_calculate_schedule_zero_window():
"""
@TEST_SCENARIO: Window start == end. All tasks at start time.
@@ -92,31 +114,40 @@ def test_calculate_schedule_zero_window():
end = time(1, 0)
dashboards = ["d1", "d2"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 2
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
assert schedule[1] == datetime(2024, 1, 1, 1, 0)
# [/DEF:test_calculate_schedule_zero_window:Function]
# [DEF:test_calculate_schedule_very_small_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate sub-second interpolation when task count exceeds near-zero window granularity.
def test_calculate_schedule_very_small_window():
"""
@TEST_SCENARIO: Window smaller than number of tasks (in seconds).
"""
start = time(1, 0, 0)
end = time(1, 0, 1) # 1 second window
end = time(1, 0, 1) # 1 second window
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 1, 0, 0)
assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s
assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s
assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1)
# [/DEF:test_throttled_scheduler:Module]# [/DEF:test_calculate_schedule_very_small_window:Function]
# [/DEF:test_calculate_schedule_very_small_window:Function]
# [/DEF:test_throttled_scheduler:Module]

1
backend/src/core/auth/config.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: auth, config, settings, jwt, adfs
# @PURPOSE: Centralized configuration for authentication and authorization.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> pydantic
#

30
backend/src/core/auth/jwt.py
View File

@@ -4,8 +4,8 @@
# @SEMANTICS: jwt, token, session, auth
# @PURPOSE: JWT token generation and validation logic.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> jose
# @RELATION: USES -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
# @RELATION: USES -> [auth_config]
#
# @INVARIANT: Tokens must include expiration time and user identifier.
@@ -17,11 +17,12 @@ from .config import auth_config
from ..logger import belief_scope
# [/SECTION]
# [DEF:create_access_token:Function]
# @PURPOSE: Generates a new JWT access token.
# @PRE: data dict contains 'sub' (user_id) and optional 'scopes' (roles).
# @POST: Returns a signed JWT string.
# @RELATION: DEPENDS_ON -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
#
# @PARAM: data (dict) - Payload data for the token.
# @PARAM: expires_delta (Optional[timedelta]) - Custom expiration time.
@@ -32,26 +33,37 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES)
expire = datetime.utcnow() + timedelta(
minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES
)
to_encode.update({"exp": expire})
encoded_jwt = jwt.encode(to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM)
encoded_jwt = jwt.encode(
to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM
)
return encoded_jwt
# [/DEF:create_access_token:Function]
# [DEF:decode_token:Function]
# @PURPOSE: Decodes and validates a JWT token.
# @PRE: token is a signed JWT string.
# @POST: Returns the decoded payload if valid.
# @RELATION: DEPENDS_ON -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
#
# @PARAM: token (str) - The JWT to decode.
# @RETURN: dict - The decoded payload.
# @THROW: jose.JWTError - If token is invalid or expired.
def decode_token(token: str) -> dict:
with belief_scope("decode_token"):
payload = jwt.decode(token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM])
payload = jwt.decode(
token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM]
)
return payload
# [/DEF:decode_token:Function]
# [/DEF:AuthJwtModule:Module]
# [/DEF:AuthJwtModule:Module]

1
backend/src/core/auth/oauth.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: auth, oauth, oidc, adfs
# @PURPOSE: ADFS OIDC configuration and client using Authlib.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> authlib
# @RELATION: USES -> auth_config

92
backend/src/core/auth/repository.py
View File

@@ -4,12 +4,12 @@
# @SEMANTICS: auth, repository, database, user, role, permission
# @PURPOSE: Data access layer for authentication and user preference entities.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[sqlalchemy.orm.Session]
# @RELATION: DEPENDS_ON ->[User:Class]
# @RELATION: DEPENDS_ON ->[Role:Class]
# @RELATION: DEPENDS_ON ->[Permission:Class]
# @RELATION: DEPENDS_ON ->[UserDashboardPreference:Class]
# @RELATION: DEPENDS_ON ->[belief_scope:Function]
# @RELATION: DEPENDS_ON -> [Session]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
# @RELATION: DEPENDS_ON -> [UserDashboardPreference]
# @RELATION: DEPENDS_ON -> [belief_scope]
# @INVARIANT: All database read/write operations must execute via the injected SQLAlchemy session boundary.
# @DATA_CONTRACT: Session -> [User | Role | Permission | UserDashboardPreference]
# @PRE: Database connection is active.
@@ -24,12 +24,13 @@ from ...models.profile import UserDashboardPreference
from ..logger import belief_scope, logger
# [/SECTION]
# [DEF:AuthRepository:Class]
# @PURPOSE: Provides low-level CRUD operations for identity and authorization records.
# @PRE: Database session is bound.
# @POST: Entity instances returned safely.
# @SIDE_EFFECT: Performs database reads.
# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session
# @RELATION: DEPENDS_ON -> [Session]
class AuthRepository:
# @PURPOSE: Initialize repository with database session.
def __init__(self, db: Session):
@@ -39,98 +40,125 @@ class AuthRepository:
# @PURPOSE: Retrieve user by UUID.
# @PRE: user_id is a valid UUID string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> User
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_id(self, user_id: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_id"):
logger.reason(f"Fetching user by id: {user_id}")
result = self.db.query(User).filter(User.id == user_id).first()
logger.reflect(f"User found: {result is not None}")
return result
# [/DEF:get_user_by_id:Function]
# [DEF:get_user_by_username:Function]
# @PURPOSE: Retrieve user by username.
# @PRE: username is a non-empty string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> User
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_username(self, username: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_username"):
logger.reason(f"Fetching user by username: {username}")
result = self.db.query(User).filter(User.username == username).first()
logger.reflect(f"User found: {result is not None}")
return result
# [/DEF:get_user_by_username:Function]
# [DEF:get_role_by_id:Function]
# @PURPOSE: Retrieve role by UUID with permissions preloaded.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
def get_role_by_id(self, role_id: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_id"):
return self.db.query(Role).options(selectinload(Role.permissions)).filter(Role.id == role_id).first()
return (
self.db.query(Role)
.options(selectinload(Role.permissions))
.filter(Role.id == role_id)
.first()
)
# [/DEF:get_role_by_id:Function]
# [DEF:get_role_by_name:Function]
# @PURPOSE: Retrieve role by unique name.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> [Role]
def get_role_by_name(self, name: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_name"):
return self.db.query(Role).filter(Role.name == name).first()
# [/DEF:get_role_by_name:Function]
# [DEF:get_permission_by_id:Function]
# @PURPOSE: Retrieve permission by UUID.
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_id(self, permission_id: str) -> Optional[Permission]:
with belief_scope("AuthRepository.get_permission_by_id"):
return self.db.query(Permission).filter(Permission.id == permission_id).first()
return (
self.db.query(Permission).filter(Permission.id == permission_id).first()
)
# [/DEF:get_permission_by_id:Function]
# [DEF:get_permission_by_resource_action:Function]
# @PURPOSE: Retrieve permission by resource and action tuple.
# @RELATION: DEPENDS_ON -> Permission
def get_permission_by_resource_action(self, resource: str, action: str) -> Optional[Permission]:
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_resource_action(
self, resource: str, action: str
) -> Optional[Permission]:
with belief_scope("AuthRepository.get_permission_by_resource_action"):
return self.db.query(Permission).filter(
Permission.resource == resource,
Permission.action == action
).first()
return (
self.db.query(Permission)
.filter(Permission.resource == resource, Permission.action == action)
.first()
)
# [/DEF:get_permission_by_resource_action:Function]
# [DEF:list_permissions:Function]
# @PURPOSE: List all system permissions.
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Permission]
def list_permissions(self) -> List[Permission]:
with belief_scope("AuthRepository.list_permissions"):
return self.db.query(Permission).all()
# [/DEF:list_permissions:Function]
# [DEF:get_user_dashboard_preference:Function]
# @PURPOSE: Retrieve dashboard filters/preferences for a user.
# @RELATION: DEPENDS_ON -> UserDashboardPreference
def get_user_dashboard_preference(self, user_id: str) -> Optional[UserDashboardPreference]:
# @RELATION: DEPENDS_ON -> [UserDashboardPreference]
def get_user_dashboard_preference(
self, user_id: str
) -> Optional[UserDashboardPreference]:
with belief_scope("AuthRepository.get_user_dashboard_preference"):
return self.db.query(UserDashboardPreference).filter(
UserDashboardPreference.user_id == user_id
).first()
return (
self.db.query(UserDashboardPreference)
.filter(UserDashboardPreference.user_id == user_id)
.first()
)
# [/DEF:get_user_dashboard_preference:Function]
# [DEF:get_roles_by_ad_groups:Function]
# @PURPOSE: Retrieve roles that match a list of AD group names.
# @PRE: groups is a list of strings representing AD group identifiers.
# @POST: Returns a list of Role objects mapped to the provided AD groups.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> ADGroupMapping
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [ADGroupMapping]
def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]:
with belief_scope("AuthRepository.get_roles_by_ad_groups"):
logger.reason(f"Fetching roles for AD groups: {groups}")
if not groups:
return []
return self.db.query(Role).join(ADGroupMapping).filter(
ADGroupMapping.ad_group.in_(groups)
).all()
return (
self.db.query(Role)
.join(ADGroupMapping)
.filter(ADGroupMapping.ad_group.in_(groups))
.all()
)
# [/DEF:get_roles_by_ad_groups:Function]
# [/DEF:AuthRepository:Class]
# [/DEF:AuthRepositoryModule:Module]

1
backend/src/core/auth/security.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: security, password, hashing, bcrypt
# @PURPOSE: Utility for password hashing and verification using Passlib.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> bcrypt
#

12
backend/src/core/encryption_key.py
View File

@@ -1,10 +1,14 @@
# [DEF:backend.src.core.encryption_key:Module]
# [DEF:EncryptionKeyModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: encryption, key, bootstrap, environment, startup
# @PURPOSE: Resolve and persist the Fernet encryption key required by runtime services.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Runtime key resolution never falls back to an ephemeral secret.
# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key]
from __future__ import annotations
@@ -51,6 +55,8 @@ def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str:
logger.reason(f"Generated ENCRYPTION_KEY and persisted it to {env_file_path}.")
logger.reflect("Encryption key is available for runtime services.")
return generated_key
# [/DEF:ensure_encryption_key:Function]
# [/DEF:backend.src.core.encryption_key:Module]
# [/DEF:EncryptionKeyModule:Module]

187
backend/src/core/mapping_service.py
View File

@@ -1,11 +1,15 @@
# [DEF:backend.src.core.mapping_service:Module]
# [DEF:IdMappingServiceModule:Module]
#
# @COMPLEXITY: 5
# @SEMANTICS: mapping, ids, synchronization, environments, cross-filters
# @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.models.mapping (ResourceMapping, ResourceType)
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [MappingModels]
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
#
# @INVARIANT: sync_environment must handle remote API failures gracefully.
@@ -20,9 +24,17 @@ from src.models.mapping import ResourceMapping, ResourceType
from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:IdMappingService:Class]
# @COMPLEXITY: 5
# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.
# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @RELATION: DEPENDS_ON -> [MappingModels]
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: self.db remains the authoritative session for all mapping operations.
# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]
#
# @TEST_CONTRACT: IdMappingServiceModel ->
# {
@@ -39,13 +51,13 @@ from src.core.logger import logger, belief_scope
# @TEST_EDGE: get_batch_empty_list -> returns empty dict
# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]
class IdMappingService:
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapping service.
def __init__(self, db_session: Session):
self.db = db_session
self.scheduler = BackgroundScheduler()
self._sync_job = None
# [/DEF:__init__:Function]
# [DEF:start_scheduler:Function]
@@ -53,30 +65,39 @@ class IdMappingService:
# @PARAM: cron_string (str) - Cron expression for the sync interval.
# @PARAM: environments (List[str]) - List of environment IDs to sync.
# @PARAM: superset_client_factory - Function to get a client for an environment.
def start_scheduler(self, cron_string: str, environments: List[str], superset_client_factory):
def start_scheduler(
self, cron_string: str, environments: List[str], superset_client_factory
):
with belief_scope("IdMappingService.start_scheduler"):
if self._sync_job:
self.scheduler.remove_job(self._sync_job.id)
logger.info("[IdMappingService.start_scheduler][Reflect] Removed existing sync job.")
logger.info(
"[IdMappingService.start_scheduler][Reflect] Removed existing sync job."
)
def sync_all():
for env_id in environments:
client = superset_client_factory(env_id)
if client:
self.sync_environment(env_id, client)
self._sync_job = self.scheduler.add_job(
sync_all,
CronTrigger.from_crontab(cron_string),
id='id_mapping_sync_job',
replace_existing=True
id="id_mapping_sync_job",
replace_existing=True,
)
if not self.scheduler.running:
self.scheduler.start()
logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}"
)
else:
logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}"
)
# [/DEF:start_scheduler:Function]
# [DEF:sync_environment:Function]
@@ -85,54 +106,74 @@ class IdMappingService:
# @PARAM: superset_client - Instance capable of hitting the Superset API.
# @PRE: environment_id exists in the database.
# @POST: ResourceMapping records for the environment are created or updated.
def sync_environment(self, environment_id: str, superset_client, incremental: bool = False) -> None:
def sync_environment(
self, environment_id: str, superset_client, incremental: bool = False
) -> None:
"""
Polls the Superset APIs for the target environment and updates the local mapping table.
If incremental=True, only fetches items changed since the max last_synced_at date.
"""
with belief_scope("IdMappingService.sync_environment"):
logger.info(f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})")
logger.info(
f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})"
)
# Implementation Note: In a real scenario, superset_client needs to be an instance
# capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/
# Here we structure the logic according to the spec.
types_to_poll = [
(ResourceType.CHART, "chart", "slice_name"),
(ResourceType.DATASET, "dataset", "table_name"),
(ResourceType.DASHBOARD, "dashboard", "slug") # Note: dashboard slug or dashboard_title
(
ResourceType.DASHBOARD,
"dashboard",
"slug",
), # Note: dashboard slug or dashboard_title
]
total_synced = 0
total_deleted = 0
try:
for res_enum, endpoint, name_field in types_to_poll:
logger.debug(f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint")
logger.debug(
f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint"
)
# Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... )
# This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] }
# We assume superset_client provides a generic method to fetch all pages.
try:
since_dttm = None
if incremental:
from sqlalchemy.sql import func
max_date = self.db.query(func.max(ResourceMapping.last_synced_at)).filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == res_enum
).scalar()
max_date = (
self.db.query(func.max(ResourceMapping.last_synced_at))
.filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == res_enum,
)
.scalar()
)
if max_date:
# We subtract a bit for safety overlap
from datetime import timedelta
since_dttm = max_date - timedelta(minutes=5)
logger.debug(f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}")
resources = superset_client.get_all_resources(endpoint, since_dttm=since_dttm)
since_dttm = max_date - timedelta(minutes=5)
logger.debug(
f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}"
)
resources = superset_client.get_all_resources(
endpoint, since_dttm=since_dttm
)
# Track which UUIDs we see in this sync cycle
synced_uuids = set()
for res in resources:
res_uuid = res.get("uuid")
raw_id = res.get("id")
@@ -140,16 +181,20 @@ class IdMappingService:
if not res_uuid or raw_id is None:
continue
synced_uuids.add(res_uuid)
res_id = str(raw_id) # Store as string
res_id = str(raw_id) # Store as string
# Upsert Logic
mapping = self.db.query(ResourceMapping).filter_by(
environment_id=environment_id,
resource_type=res_enum,
uuid=res_uuid
).first()
mapping = (
self.db.query(ResourceMapping)
.filter_by(
environment_id=environment_id,
resource_type=res_enum,
uuid=res_uuid,
)
.first()
)
if mapping:
mapping.remote_integer_id = res_id
@@ -162,10 +207,10 @@ class IdMappingService:
uuid=res_uuid,
remote_integer_id=res_id,
resource_name=res_name,
last_synced_at=datetime.now(timezone.utc)
last_synced_at=datetime.now(timezone.utc),
)
self.db.add(new_mapping)
total_synced += 1
# Delete stale mappings: rows for this env+type whose UUID
@@ -183,19 +228,28 @@ class IdMappingService:
deleted = stale_query.delete(synchronize_session="fetch")
if deleted:
total_deleted += deleted
logger.info(f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}")
logger.info(
f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}"
)
except Exception as loop_e:
logger.error(f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}")
logger.error(
f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}"
)
# Continue to next resource type instead of blowing up the whole sync
self.db.commit()
logger.info(f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.")
logger.info(
f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items."
)
except Exception as e:
self.db.rollback()
logger.error(f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}")
logger.error(
f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}"
)
raise
# [/DEF:sync_environment:Function]
# [DEF:get_remote_id:Function]
@@ -204,19 +258,24 @@ class IdMappingService:
# @PARAM: resource_type (ResourceType)
# @PARAM: uuid (str)
# @RETURN: Optional[int]
def get_remote_id(self, environment_id: str, resource_type: ResourceType, uuid: str) -> Optional[int]:
mapping = self.db.query(ResourceMapping).filter_by(
environment_id=environment_id,
resource_type=resource_type,
uuid=uuid
).first()
def get_remote_id(
self, environment_id: str, resource_type: ResourceType, uuid: str
) -> Optional[int]:
mapping = (
self.db.query(ResourceMapping)
.filter_by(
environment_id=environment_id, resource_type=resource_type, uuid=uuid
)
.first()
)
if mapping:
try:
return int(mapping.remote_integer_id)
except ValueError:
return None
return None
# [/DEF:get_remote_id:Function]
# [DEF:get_remote_ids_batch:Function]
@@ -225,15 +284,21 @@ class IdMappingService:
# @PARAM: resource_type (ResourceType)
# @PARAM: uuids (List[str])
# @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID
def get_remote_ids_batch(self, environment_id: str, resource_type: ResourceType, uuids: List[str]) -> Dict[str, int]:
def get_remote_ids_batch(
self, environment_id: str, resource_type: ResourceType, uuids: List[str]
) -> Dict[str, int]:
if not uuids:
return {}
mappings = self.db.query(ResourceMapping).filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == resource_type,
ResourceMapping.uuid.in_(uuids)
).all()
mappings = (
self.db.query(ResourceMapping)
.filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == resource_type,
ResourceMapping.uuid.in_(uuids),
)
.all()
)
result = {}
for m in mappings:
@@ -241,9 +306,11 @@ class IdMappingService:
result[m.uuid] = int(m.remote_integer_id)
except ValueError:
pass
return result
# [/DEF:get_remote_ids_batch:Function]
# [/DEF:IdMappingService:Class]
# [/DEF:backend.src.core.mapping_service:Module]
# [/DEF:IdMappingServiceModule:Module]

4
backend/src/core/migration/__init__.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.core.migration.__init__:Module]
# [DEF:MigrationPackage:Module]
# @COMPLEXITY: 1
# @SEMANTICS: migration, package, exports
# @PURPOSE: Namespace package for migration pre-flight orchestration components.
@@ -9,4 +9,4 @@ from .archive_parser import MigrationArchiveParser
__all__ = ["MigrationDryRunService", "MigrationArchiveParser"]
# [/DEF:backend.src.core.migration.__init__:Module]
# [/DEF:MigrationPackage:Module]

14
backend/src/core/migration/archive_parser.py
View File

@@ -1,9 +1,9 @@
# [DEF:backend.src.core.migration.archive_parser:Module]
# [DEF:MigrationArchiveParserModule:Module]
# @COMPLEXITY: 3
# @SEMANTICS: migration, zip, parser, yaml, metadata
# @PURPOSE: Parse Superset export ZIP archives into normalized object catalogs for diffing.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Parsing is read-only and never mutates archive files.
import json
@@ -19,11 +19,13 @@ from ..logger import logger, belief_scope
# [DEF:MigrationArchiveParser:Class]
# @PURPOSE: Extract normalized dashboards/charts/datasets metadata from ZIP archives.
# @RELATION: CONTAINS -> [extract_objects_from_zip, _collect_yaml_objects, _normalize_object_payload]
# @RELATION: CONTAINS -> [extract_objects_from_zip]
# @RELATION: CONTAINS -> [_collect_yaml_objects]
# @RELATION: CONTAINS -> [_normalize_object_payload]
class MigrationArchiveParser:
# [DEF:extract_objects_from_zip:Function]
# @PURPOSE: Extract object catalogs from Superset archive.
# @RELATION: DEPENDS_ON -> _collect_yaml_objects
# @RELATION: DEPENDS_ON -> [_collect_yaml_objects]
# @PRE: zip_path points to a valid readable ZIP.
# @POST: Returns object lists grouped by resource type.
# @RETURN: Dict[str, List[Dict[str, Any]]]
@@ -53,7 +55,7 @@ class MigrationArchiveParser:
# [DEF:_collect_yaml_objects:Function]
# @PURPOSE: Read and normalize YAML manifests for one object type.
# @RELATION: DEPENDS_ON -> _normalize_object_payload
# @RELATION: DEPENDS_ON -> [_normalize_object_payload]
# @PRE: object_type is one of dashboards/charts/datasets.
# @POST: Returns only valid normalized objects.
def _collect_yaml_objects(
@@ -153,4 +155,4 @@ class MigrationArchiveParser:
# [/DEF:MigrationArchiveParser:Class]
# [/DEF:backend.src.core.migration.archive_parser:Module]
# [/DEF:MigrationArchiveParserModule:Module]

29
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -1,12 +1,12 @@
# [DEF:backend.src.core.migration.dry_run_orchestrator:Module]
# [DEF:MigrationDryRunOrchestratorModule:Module]
# @COMPLEXITY: 3
# @SEMANTICS: migration, dry_run, diff, risk, superset
# @PURPOSE: Compute pre-flight migration diff and risk scoring without apply.
# @LAYER: Core
# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration_engine.MigrationEngine]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.archive_parser.MigrationArchiveParser]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.risk_assessor]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [MigrationEngine]
# @RELATION: DEPENDS_ON -> [MigrationArchiveParser]
# @RELATION: DEPENDS_ON -> [RiskAssessorModule]
# @INVARIANT: Dry run is informative only and must not mutate target environment.
from datetime import datetime, timezone
@@ -27,7 +27,14 @@ from ..utils.fileio import create_temp_file
# [DEF:MigrationDryRunService:Class]
# @PURPOSE: Build deterministic diff/risk payload for migration pre-flight.
# @RELATION: CONTAINS -> [__init__, run, _load_db_mapping, _accumulate_objects, _index_by_uuid, _build_object_diff, _build_target_signatures, _build_risks]
# @RELATION: CONTAINS -> [__init__]
# @RELATION: CONTAINS -> [run]
# @RELATION: CONTAINS -> [_load_db_mapping]
# @RELATION: CONTAINS -> [_accumulate_objects]
# @RELATION: CONTAINS -> [_index_by_uuid]
# @RELATION: CONTAINS -> [_build_object_diff]
# @RELATION: CONTAINS -> [_build_target_signatures]
# @RELATION: CONTAINS -> [_build_risks]
class MigrationDryRunService:
# [DEF:__init__:Function]
# @PURPOSE: Wire parser dependency for archive object extraction.
@@ -40,7 +47,11 @@ class MigrationDryRunService:
# [DEF:run:Function]
# @PURPOSE: Execute full dry-run computation for selected dashboards.
# @RELATION: DEPENDS_ON -> [_load_db_mapping, _accumulate_objects, _build_target_signatures, _build_object_diff, _build_risks]
# @RELATION: DEPENDS_ON -> [_load_db_mapping]
# @RELATION: DEPENDS_ON -> [_accumulate_objects]
# @RELATION: DEPENDS_ON -> [_build_target_signatures]
# @RELATION: DEPENDS_ON -> [_build_object_diff]
# @RELATION: DEPENDS_ON -> [_build_risks]
# @PRE: source/target clients are authenticated and selection validated by caller.
# @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk.
# @SIDE_EFFECT: Reads source export archives and target metadata via network.
@@ -195,7 +206,7 @@ class MigrationDryRunService:
# [DEF:_build_object_diff:Function]
# @PURPOSE: Compute create/update/delete buckets by UUID+signature.
# @RELATION: DEPENDS_ON -> _index_by_uuid
# @RELATION: DEPENDS_ON -> [_index_by_uuid]
def _build_object_diff(
self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]]
) -> Dict[str, List[Dict[str, Any]]]:
@@ -349,4 +360,4 @@ class MigrationDryRunService:
# [/DEF:MigrationDryRunService:Class]
# [/DEF:backend.src.core.migration.dry_run_orchestrator:Module]
# [/DEF:MigrationDryRunOrchestratorModule:Module]

19
backend/src/core/migration/risk_assessor.py
View File

@@ -1,12 +1,18 @@
# [DEF:backend.src.core.migration.risk_assessor:Module]
# [DEF:RiskAssessorModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: migration, dry_run, risk, scoring, preflight
# @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [backend.src.core.superset_client.SupersetClient]
# @RELATION: DISPATCHED_BY -> [backend.src.core.migration.dry_run_orchestrator.MigrationDryRunService.run]
# @RELATION: CONTAINS -> [index_by_uuid, extract_owner_identifiers, build_risks, score_risks]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: CONTAINS -> [index_by_uuid]
# @RELATION: CONTAINS -> [extract_owner_identifiers]
# @RELATION: CONTAINS -> [build_risks]
# @RELATION: CONTAINS -> [score_risks]
# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.
# @POST: Risk scoring output preserves item list and provides bounded score with derived level.
# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.
# @DATA_CONTRACT: Module[build_risks, score_risks]
# @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
# @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]
# @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
@@ -80,7 +86,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]:
# [DEF:build_risks:Function]
# @PURPOSE: Build risk list from computed diffs and target catalog state.
# @RELATION: DEPENDS_ON -> [index_by_uuid, extract_owner_identifiers]
# @RELATION: DEPENDS_ON -> [index_by_uuid]
# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]
# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
# @PRE: target_client is authenticated/usable for database list retrieval.
# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
@@ -192,4 +199,4 @@ def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:
# [/DEF:score_risks:Function]
# [/DEF:backend.src.core.migration.risk_assessor:Module]
# [/DEF:RiskAssessorModule:Module]

79
backend/src/core/scheduler.py
View File

@@ -3,20 +3,18 @@
# @SEMANTICS: scheduler, apscheduler, cron, backup
# @PURPOSE: Manages scheduled tasks using APScheduler.
# @LAYER: Core
# @RELATION: Uses TaskManager to run scheduled backups.
# @RELATION: DEPENDS_ON -> TaskManager
# [SECTION: IMPORTS]
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.date import DateTrigger
from .logger import logger, belief_scope
from .config_manager import ConfigManager
from .database import SessionLocal
from ..models.llm import ValidationPolicy
import asyncio
from datetime import datetime, time, timedelta, date
# [/SECTION]
# [DEF:SchedulerService:Class]
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, service, apscheduler
@@ -32,6 +30,7 @@ class SchedulerService:
self.config_manager = config_manager
self.scheduler = BackgroundScheduler()
self.loop = asyncio.get_event_loop()
# [/DEF:__init__:Function]
# [DEF:start:Function]
@@ -44,6 +43,7 @@ class SchedulerService:
self.scheduler.start()
logger.info("Scheduler started.")
self.load_schedules()
# [/DEF:start:Function]
# [DEF:stop:Function]
@@ -55,6 +55,7 @@ class SchedulerService:
if self.scheduler.running:
self.scheduler.shutdown()
logger.info("Scheduler stopped.")
# [/DEF:stop:Function]
# [DEF:load_schedules:Function]
@@ -65,11 +66,12 @@ class SchedulerService:
with belief_scope("SchedulerService.load_schedules"):
# Clear existing jobs
self.scheduler.remove_all_jobs()
config = self.config_manager.get_config()
for env in config.environments:
if env.backup_schedule and env.backup_schedule.enabled:
self.add_backup_job(env.id, env.backup_schedule.cron_expression)
# [/DEF:load_schedules:Function]
# [DEF:add_backup_job:Function]
@@ -79,7 +81,10 @@ class SchedulerService:
# @PARAM: env_id (str) - The ID of the environment.
# @PARAM: cron_expression (str) - The cron expression for the schedule.
def add_backup_job(self, env_id: str, cron_expression: str):
with belief_scope("SchedulerService.add_backup_job", f"env_id={env_id}, cron={cron_expression}"):
with belief_scope(
"SchedulerService.add_backup_job",
f"env_id={env_id}, cron={cron_expression}",
):
job_id = f"backup_{env_id}"
try:
self.scheduler.add_job(
@@ -87,11 +92,14 @@ class SchedulerService:
CronTrigger.from_crontab(cron_expression),
id=job_id,
args=[env_id],
replace_existing=True
replace_existing=True,
)
logger.info(
f"Scheduled backup job added for environment {env_id}: {cron_expression}"
)
logger.info(f"Scheduled backup job added for environment {env_id}: {cron_expression}")
except Exception as e:
logger.error(f"Failed to add backup job for environment {env_id}: {e}")
# [/DEF:add_backup_job:Function]
# [DEF:_trigger_backup:Function]
@@ -102,30 +110,45 @@ class SchedulerService:
def _trigger_backup(self, env_id: str):
with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"):
logger.info(f"Triggering scheduled backup for environment {env_id}")
# Check if a backup is already running for this environment
active_tasks = self.task_manager.get_tasks(limit=100)
for task in active_tasks:
if (task.plugin_id == "superset-backup" and
task.status in ["PENDING", "RUNNING"] and
task.params.get("environment_id") == env_id):
logger.warning(f"Backup already running for environment {env_id}. Skipping scheduled run.")
if (
task.plugin_id == "superset-backup"
and task.status in ["PENDING", "RUNNING"]
and task.params.get("environment_id") == env_id
):
logger.warning(
f"Backup already running for environment {env_id}. Skipping scheduled run."
)
return
# Run the backup task
# We need to run this in the event loop since create_task is async
asyncio.run_coroutine_threadsafe(
self.task_manager.create_task("superset-backup", {"environment_id": env_id}),
self.loop
self.task_manager.create_task(
"superset-backup", {"environment_id": env_id}
),
self.loop,
)
# [/DEF:_trigger_backup:Function]
# [/DEF:SchedulerService:Class]
# [DEF:ThrottledSchedulerConfigurator:Class]
# @COMPLEXITY: 5
# @SEMANTICS: scheduler, throttling, distribution
# @PURPOSE: Distributes validation tasks evenly within an execution window.
# @PRE: Validation policies provide a finite dashboard list and a valid execution window.
# @POST: Produces deterministic per-dashboard run timestamps within the configured window.
# @RELATION: DEPENDS_ON -> SchedulerModule
# @INVARIANT: Returned schedule size always matches number of dashboard IDs.
# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.
# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]
class ThrottledSchedulerConfigurator:
# [DEF:calculate_schedule:Function]
# @PURPOSE: Calculates execution times for N tasks within a window.
@@ -134,10 +157,7 @@ class ThrottledSchedulerConfigurator:
# @INVARIANT: Tasks are distributed with near-even spacing.
@staticmethod
def calculate_schedule(
window_start: time,
window_end: time,
dashboard_ids: list,
current_date: date
window_start: time, window_end: time, dashboard_ids: list, current_date: date
) -> list:
with belief_scope("ThrottledSchedulerConfigurator.calculate_schedule"):
n = len(dashboard_ids)
@@ -152,32 +172,39 @@ class ThrottledSchedulerConfigurator:
end_dt += timedelta(days=1)
total_seconds = (end_dt - start_dt).total_seconds()
# Minimum interval of 1 second to avoid division by zero or negative
if total_seconds <= 0:
logger.warning(f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.")
logger.warning(
f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks."
)
return [start_dt] * n
# If window is too small for even distribution (e.g. 10 tasks in 5 seconds),
# we still distribute them but they might be very close.
# The requirement says "near-even spacing".
if n == 1:
return [start_dt]
interval = total_seconds / (n - 1) if n > 1 else 0
# If interval is too small (e.g. < 1s), we might want a fallback,
# but the spec says "handle too-small windows with explicit fallback/warning".
if interval < 1:
logger.warning(f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.")
logger.warning(
f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated."
)
scheduled_times = []
for i in range(n):
scheduled_times.append(start_dt + timedelta(seconds=i * interval))
return scheduled_times
# [/DEF:calculate_schedule:Function]
# [/DEF:ThrottledSchedulerConfigurator:Class]
# [/DEF:SchedulerModule:Module]
# [/DEF:SchedulerModule:Module]

39
backend/src/core/task_manager/context.py
View File

@@ -2,9 +2,14 @@
# @SEMANTICS: task, context, plugin, execution, logger
# @PURPOSE: Provides execution context passed to plugins during task execution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins
# @RELATION: DEPENDS_ON -> [TaskLoggerModule]
# @RELATION: USED_BY -> [TaskManager]
# @COMPLEXITY: 5
# @INVARIANT: Each TaskContext is bound to a single task execution.
# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries.
# @POST: Plugins receive context instances with stable logger and parameter accessors.
# @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# [SECTION: IMPORTS]
# [SECTION: IMPORTS]
@@ -13,11 +18,17 @@ from .task_logger import TaskLogger
from ..logger import belief_scope
# [/SECTION]
# [DEF:TaskContext:Class]
# @SEMANTICS: context, task, execution, plugin
# @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities.
# @COMPLEXITY: 5
# @INVARIANT: logger is always a valid TaskLogger instance.
# @PRE: Constructor receives non-empty task_id, callable add_log_fn, and params mapping.
# @POST: Instance exposes immutable task identity with logger, params, and optional background task access.
# @RELATION: DEPENDS_ON -> [TaskLogger]
# @SIDE_EFFECT: Emits structured task logs through TaskLogger on plugin interactions.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @UX_STATE: Idle -> Active -> Complete
#
# @TEST_CONTRACT: TaskContextInit ->
@@ -36,7 +47,7 @@ from ..logger import belief_scope
class TaskContext:
"""
Execution context provided to plugins during task execution.
Usage:
def execute(params: dict, context: TaskContext = None):
if context:
@@ -44,7 +55,7 @@ class TaskContext:
context.logger.progress("Processing items", percent=50)
# ... plugin logic
"""
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskContext with task-specific resources.
# @PRE: task_id is a valid task identifier, add_log_fn is callable.
@@ -66,12 +77,11 @@ class TaskContext:
self._params = params
self._background_tasks = background_tasks
self._logger = TaskLogger(
task_id=task_id,
add_log_fn=add_log_fn,
source=default_source
task_id=task_id, add_log_fn=add_log_fn, source=default_source
)
# [/DEF:__init__:Function]
# [DEF:task_id:Function]
# @PURPOSE: Get the task ID.
# @PRE: TaskContext must be initialized.
@@ -81,8 +91,9 @@ class TaskContext:
def task_id(self) -> str:
with belief_scope("task_id"):
return self._task_id
# [/DEF:task_id:Function]
# [DEF:logger:Function]
# @PURPOSE: Get the TaskLogger instance for this context.
# @PRE: TaskContext must be initialized.
@@ -92,8 +103,9 @@ class TaskContext:
def logger(self) -> TaskLogger:
with belief_scope("logger"):
return self._logger
# [/DEF:logger:Function]
# [DEF:params:Function]
# @PURPOSE: Get the task parameters.
# @PRE: TaskContext must be initialized.
@@ -103,6 +115,7 @@ class TaskContext:
def params(self) -> Dict[str, Any]:
with belief_scope("params"):
return self._params
# [/DEF:params:Function]
# [DEF:background_tasks:Function]
@@ -113,8 +126,9 @@ class TaskContext:
def background_tasks(self) -> Optional[Any]:
with belief_scope("background_tasks"):
return self._background_tasks
# [/DEF:background_tasks:Function]
# [DEF:get_param:Function]
# @PURPOSE: Get a specific parameter value with optional default.
# @PRE: TaskContext must be initialized.
@@ -125,8 +139,9 @@ class TaskContext:
def get_param(self, key: str, default: Any = None) -> Any:
with belief_scope("get_param"):
return self._params.get(key, default)
# [/DEF:get_param:Function]
# [DEF:create_sub_context:Function]
# @PURPOSE: Create a sub-context with a different default source.
# @PRE: source is a non-empty string.
@@ -143,8 +158,10 @@ class TaskContext:
default_source=source,
background_tasks=self._background_tasks,
)
# [/DEF:create_sub_context:Function]
# [/DEF:TaskContext:Class]
# [/DEF:TaskContextModule:Module]

242
backend/src/core/task_manager/manager.py
View File

@@ -8,8 +8,9 @@
# @POST: Orchestrates task execution and persistence.
# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.
# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]
# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceModule:Module]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @INVARIANT: Task IDs are unique.
# @CONSTRAINT: Must use belief_scope for logging.
# @TEST_CONTRACT: TaskManagerModule -> {
@@ -38,15 +39,16 @@ from .context import TaskContext
from ..logger import logger, belief_scope, should_log_task_level
# [/SECTION]
# [DEF:TaskManager:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService:Class]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService:Class]
# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
@@ -56,42 +58,49 @@ class TaskManager:
"""
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
"""
# Log flush interval in seconds
LOG_FLUSH_INTERVAL = 2.0
# [DEF:__init__:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
# @PRE: plugin_loader is initialized.
# @POST: TaskManager is ready to accept tasks.
# @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @PARAM: plugin_loader - The plugin loader instance.
def __init__(self, plugin_loader):
with belief_scope("TaskManager.__init__"):
self.plugin_loader = plugin_loader
self.tasks: Dict[str, Task] = {}
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
self.executor = ThreadPoolExecutor(
max_workers=5
) # For CPU-bound plugin execution
self.persistence_service = TaskPersistenceService()
self.log_persistence_service = TaskLogPersistenceService()
# Log buffer: task_id -> List[LogEntry]
self._log_buffer: Dict[str, List[LogEntry]] = {}
self._log_buffer_lock = threading.Lock()
# Flusher thread for batch writing logs
self._flusher_stop_event = threading.Event()
self._flusher_thread = threading.Thread(target=self._flusher_loop, daemon=True)
self._flusher_thread = threading.Thread(
target=self._flusher_loop, daemon=True
)
self._flusher_thread.start()
try:
self.loop = asyncio.get_running_loop()
except RuntimeError:
self.loop = asyncio.get_event_loop()
self.task_futures: Dict[str, asyncio.Future] = {}
# Load persisted tasks on startup
self.load_persisted_tasks()
# [/DEF:__init__:Function]
# [DEF:_flusher_loop:Function]
@@ -99,11 +108,13 @@ class TaskManager:
# @PURPOSE: Background thread that periodically flushes log buffer to database.
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION: [CALLS] ->[TaskManager._flush_logs]
def _flusher_loop(self):
"""Background thread that flushes log buffer to database."""
while not self._flusher_stop_event.is_set():
self._flush_logs()
self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)
# [/DEF:_flusher_loop:Function]
# [DEF:_flush_logs:Function]
@@ -111,15 +122,16 @@ class TaskManager:
# @PURPOSE: Flush all buffered logs to the database.
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
def _flush_logs(self):
"""Flush all buffered logs to the database."""
with self._log_buffer_lock:
task_ids = list(self._log_buffer.keys())
for task_id in task_ids:
with self._log_buffer_lock:
logs = self._log_buffer.pop(task_id, [])
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
@@ -131,6 +143,7 @@ class TaskManager:
if task_id not in self._log_buffer:
self._log_buffer[task_id] = []
self._log_buffer[task_id].extend(logs)
# [/DEF:_flush_logs:Function]
# [DEF:_flush_task_logs:Function]
@@ -139,17 +152,19 @@ class TaskManager:
# @PRE: task_id exists.
# @POST: Task's buffered logs are written to database.
# @PARAM: task_id (str) - The task ID.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
def _flush_task_logs(self, task_id: str):
"""Flush logs for a specific task immediately."""
with belief_scope("_flush_task_logs"):
with self._log_buffer_lock:
logs = self._log_buffer.pop(task_id, [])
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
except Exception as e:
logger.error(f"Failed to flush logs for task {task_id}: {e}")
# [/DEF:_flush_task_logs:Function]
# [DEF:create_task:Function]
@@ -162,24 +177,30 @@ class TaskManager:
# @PARAM: user_id (Optional[str]) - ID of the user requesting the task.
# @RETURN: Task - The created task instance.
# @THROWS: ValueError if plugin not found or params invalid.
async def create_task(self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None) -> Task:
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def create_task(
self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None
) -> Task:
with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"):
if not self.plugin_loader.has_plugin(plugin_id):
logger.error(f"Plugin with ID '{plugin_id}' not found.")
raise ValueError(f"Plugin with ID '{plugin_id}' not found.")
self.plugin_loader.get_plugin(plugin_id)
if not isinstance(params, dict):
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
self.tasks[task.id] = task
self.persistence_service.persist_task(task)
logger.info(f"Task {task.id} created and scheduled for execution")
self.loop.create_task(self._run_task(task.id)) # Schedule task for execution
self.loop.create_task(
self._run_task(task.id)
) # Schedule task for execution
return task
# [/DEF:create_task:Function]
# [DEF:_run_task:Function]
@@ -188,25 +209,33 @@ class TaskManager:
# @PRE: Task exists in registry.
# @POST: Task is executed, status updated to SUCCESS or FAILED.
# @PARAM: task_id (str) - The ID of the task to run.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def _run_task(self, task_id: str):
with belief_scope("TaskManager._run_task", f"task_id={task_id}"):
task = self.tasks[task_id]
plugin = self.plugin_loader.get_plugin(task.plugin_id)
logger.info(f"Starting execution of task {task_id} for plugin '{plugin.name}'")
logger.info(
f"Starting execution of task {task_id} for plugin '{plugin.name}'"
)
task.status = TaskStatus.RUNNING
task.started_at = datetime.utcnow()
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'", source="system")
self._add_log(
task_id,
"INFO",
f"Task started for plugin '{plugin.name}'",
source="system",
)
try:
# Prepare params and check if plugin supports new TaskContext
params = {**task.params, "_task_id": task_id}
# Check if plugin's execute method accepts 'context' parameter
sig = inspect.signature(plugin.execute)
accepts_context = 'context' in sig.parameters
accepts_context = "context" in sig.parameters
if accepts_context:
# Create TaskContext for new-style plugins
context = TaskContext(
@@ -216,13 +245,13 @@ class TaskManager:
default_source="plugin",
background_tasks=None,
)
if asyncio.iscoroutinefunction(plugin.execute):
task.result = await plugin.execute(params, context=context)
else:
task.result = await self.loop.run_in_executor(
self.executor,
lambda: plugin.execute(params, context=context)
lambda: plugin.execute(params, context=context),
)
else:
# Backward compatibility: old-style plugins without context
@@ -230,24 +259,36 @@ class TaskManager:
task.result = await plugin.execute(params)
else:
task.result = await self.loop.run_in_executor(
self.executor,
plugin.execute,
params
self.executor, plugin.execute, params
)
logger.info(f"Task {task_id} completed successfully")
task.status = TaskStatus.SUCCESS
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'", source="system")
self._add_log(
task_id,
"INFO",
f"Task completed successfully for plugin '{plugin.name}'",
source="system",
)
except Exception as e:
logger.error(f"Task {task_id} failed: {e}")
task.status = TaskStatus.FAILED
self._add_log(task_id, "ERROR", f"Task failed: {e}", source="system", metadata={"error_type": type(e).__name__})
self._add_log(
task_id,
"ERROR",
f"Task failed: {e}",
source="system",
metadata={"error_type": type(e).__name__},
)
finally:
task.finished_at = datetime.utcnow()
# Flush any remaining buffered logs before persisting task
self._flush_task_logs(task_id)
self.persistence_service.persist_task(task)
logger.info(f"Task {task_id} execution finished with status: {task.status}")
logger.info(
f"Task {task_id} execution finished with status: {task.status}"
)
# [/DEF:_run_task:Function]
# [DEF:resolve_task:Function]
@@ -258,21 +299,23 @@ class TaskManager:
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.
# @THROWS: ValueError if task not found or not awaiting mapping.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):
with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task or task.status != TaskStatus.AWAITING_MAPPING:
raise ValueError("Task is not awaiting mapping.")
# Update task params with resolution
task.params.update(resolution_params)
task.status = TaskStatus.RUNNING
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task resumed after mapping resolution.")
# Signal the future to continue
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# [/DEF:resolve_task:Function]
# [DEF:wait_for_resolution:Function]
@@ -281,21 +324,23 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def wait_for_resolution(self, task_id: str):
with belief_scope("TaskManager.wait_for_resolution", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
return
task.status = TaskStatus.AWAITING_MAPPING
self.persistence_service.persist_task(task)
self.task_futures[task_id] = self.loop.create_future()
try:
await self.task_futures[task_id]
finally:
if task_id in self.task_futures:
del self.task_futures[task_id]
# [/DEF:wait_for_resolution:Function]
# [DEF:wait_for_input:Function]
@@ -304,20 +349,22 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[asyncio.AbstractEventLoop.create_future]
async def wait_for_input(self, task_id: str):
with belief_scope("TaskManager.wait_for_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
return
# Status is already set to AWAITING_INPUT by await_input()
self.task_futures[task_id] = self.loop.create_future()
try:
await self.task_futures[task_id]
finally:
if task_id in self.task_futures:
del self.task_futures[task_id]
# [/DEF:wait_for_input:Function]
# [DEF:get_task:Function]
@@ -327,9 +374,11 @@ class TaskManager:
# @POST: Returns Task object or None.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: Optional[Task] - The task or None.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_task(self, task_id: str) -> Optional[Task]:
with belief_scope("TaskManager.get_task", f"task_id={task_id}"):
return self.tasks.get(task_id)
# [/DEF:get_task:Function]
# [DEF:get_all_tasks:Function]
@@ -338,9 +387,11 @@ class TaskManager:
# @PRE: None.
# @POST: Returns list of all Task objects.
# @RETURN: List[Task] - All tasks.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_all_tasks(self) -> List[Task]:
with belief_scope("TaskManager.get_all_tasks"):
return list(self.tasks.values())
# [/DEF:get_all_tasks:Function]
# [DEF:get_tasks:Function]
@@ -352,13 +403,14 @@ class TaskManager:
# @PARAM: offset (int) - Number of tasks to skip.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: List[Task] - List of tasks matching criteria.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_tasks(
self,
limit: int = 10,
offset: int = 0,
status: Optional[TaskStatus] = None,
plugin_ids: Optional[List[str]] = None,
completed_only: bool = False
completed_only: bool = False,
) -> List[Task]:
with belief_scope("TaskManager.get_tasks"):
tasks = list(self.tasks.values())
@@ -368,7 +420,10 @@ class TaskManager:
plugin_id_set = set(plugin_ids)
tasks = [t for t in tasks if t.plugin_id in plugin_id_set]
if completed_only:
tasks = [t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]]
tasks = [
t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]
]
# Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.
def sort_key(task: Task) -> float:
started_at = task.started_at
@@ -381,7 +436,8 @@ class TaskManager:
return started_at.timestamp()
tasks.sort(key=sort_key, reverse=True)
return tasks[offset:offset + limit]
return tasks[offset : offset + limit]
# [/DEF:get_tasks:Function]
# [DEF:get_task_logs:Function]
@@ -392,10 +448,13 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
# @RETURN: List[LogEntry] - List of log entries.
def get_task_logs(self, task_id: str, log_filter: Optional[LogFilter] = None) -> List[LogEntry]:
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]
def get_task_logs(
self, task_id: str, log_filter: Optional[LogFilter] = None
) -> List[LogEntry]:
with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"):
task = self.tasks.get(task_id)
# For completed tasks, fetch from persistence
if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:
if log_filter is None:
@@ -408,15 +467,16 @@ class TaskManager:
level=log.level,
message=log.message,
source=log.source,
metadata=log.metadata
metadata=log.metadata,
)
for log in task_logs
]
# For running/pending tasks, return from memory
return task.logs if task else []
# [/DEF:get_task_logs:Function]
# [DEF:get_task_log_stats:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
@@ -424,11 +484,13 @@ class TaskManager:
# @POST: Returns LogStats with counts by level and source.
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]
def get_task_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
return self.log_persistence_service.get_log_stats(task_id)
# [/DEF:get_task_log_stats:Function]
# [DEF:get_task_log_sources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
@@ -436,9 +498,11 @@ class TaskManager:
# @POST: Returns list of unique source strings.
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]
def get_task_log_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
return self.log_persistence_service.get_sources(task_id)
# [/DEF:get_task_log_sources:Function]
# [DEF:_add_log:Function]
@@ -452,6 +516,7 @@ class TaskManager:
# @PARAM: source (str) - Source component (default: "system").
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
# @RELATION: [CALLS] ->[should_log_task_level]
def _add_log(
self,
task_id: str,
@@ -459,7 +524,7 @@ class TaskManager:
message: str,
source: str = "system",
metadata: Optional[Dict[str, Any]] = None,
context: Optional[Dict[str, Any]] = None
context: Optional[Dict[str, Any]] = None,
):
with belief_scope("TaskManager._add_log", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -476,12 +541,12 @@ class TaskManager:
message=message,
source=source,
metadata=metadata,
context=context # Keep for backward compatibility
context=context, # Keep for backward compatibility
)
# Add to in-memory logs (for backward compatibility with legacy JSON field)
task.logs.append(log_entry)
# Add to buffer for batch persistence
with self._log_buffer_lock:
if task_id not in self._log_buffer:
@@ -492,6 +557,7 @@ class TaskManager:
if task_id in self.subscribers:
for queue in self.subscribers[task_id]:
self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)
# [/DEF:_add_log:Function]
# [DEF:subscribe_logs:Function]
@@ -501,6 +567,7 @@ class TaskManager:
# @POST: Returns an asyncio.Queue for log entries.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: asyncio.Queue - Queue for log entries.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
with belief_scope("TaskManager.subscribe_logs", f"task_id={task_id}"):
queue = asyncio.Queue()
@@ -508,6 +575,7 @@ class TaskManager:
self.subscribers[task_id] = []
self.subscribers[task_id].append(queue)
return queue
# [/DEF:subscribe_logs:Function]
# [DEF:unsubscribe_logs:Function]
@@ -517,6 +585,7 @@ class TaskManager:
# @POST: Queue removed from subscribers.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: queue (asyncio.Queue) - Queue to remove.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
with belief_scope("TaskManager.unsubscribe_logs", f"task_id={task_id}"):
if task_id in self.subscribers:
@@ -524,6 +593,7 @@ class TaskManager:
self.subscribers[task_id].remove(queue)
if not self.subscribers[task_id]:
del self.subscribers[task_id]
# [/DEF:unsubscribe_logs:Function]
# [DEF:load_persisted_tasks:Function]
@@ -531,12 +601,14 @@ class TaskManager:
# @PURPOSE: Load persisted tasks using persistence service.
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
def load_persisted_tasks(self) -> None:
with belief_scope("TaskManager.load_persisted_tasks"):
loaded_tasks = self.persistence_service.load_tasks(limit=100)
for task in loaded_tasks:
if task.id not in self.tasks:
self.tasks[task.id] = task
# [/DEF:load_persisted_tasks:Function]
# [DEF:await_input:Function]
@@ -547,19 +619,28 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: input_request (Dict) - Details about required input.
# @THROWS: ValueError if task not found or not RUNNING.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:
with belief_scope("TaskManager.await_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
raise ValueError(f"Task {task_id} not found")
if task.status != TaskStatus.RUNNING:
raise ValueError(f"Task {task_id} is not RUNNING (current: {task.status})")
raise ValueError(
f"Task {task_id} is not RUNNING (current: {task.status})"
)
task.status = TaskStatus.AWAITING_INPUT
task.input_required = True
task.input_request = input_request
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task paused for user input", {"input_request": input_request})
self._add_log(
task_id,
"INFO",
"Task paused for user input",
{"input_request": input_request},
)
# [/DEF:await_input:Function]
# [DEF:resume_task_with_password:Function]
@@ -570,26 +651,39 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.
# @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.
def resume_task_with_password(self, task_id: str, passwords: Dict[str, str]) -> None:
with belief_scope("TaskManager.resume_task_with_password", f"task_id={task_id}"):
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
def resume_task_with_password(
self, task_id: str, passwords: Dict[str, str]
) -> None:
with belief_scope(
"TaskManager.resume_task_with_password", f"task_id={task_id}"
):
task = self.tasks.get(task_id)
if not task:
raise ValueError(f"Task {task_id} not found")
if task.status != TaskStatus.AWAITING_INPUT:
raise ValueError(f"Task {task_id} is not AWAITING_INPUT (current: {task.status})")
raise ValueError(
f"Task {task_id} is not AWAITING_INPUT (current: {task.status})"
)
if not isinstance(passwords, dict) or not passwords:
raise ValueError("Passwords must be a non-empty dictionary")
task.params["passwords"] = passwords
task.input_required = False
task.input_request = None
task.status = TaskStatus.RUNNING
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task resumed with passwords", {"databases": list(passwords.keys())})
self._add_log(
task_id,
"INFO",
"Task resumed with passwords",
{"databases": list(passwords.keys())},
)
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# [/DEF:resume_task_with_password:Function]
# [DEF:clear_tasks:Function]
@@ -599,6 +693,7 @@ class TaskManager:
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: int - Number of tasks cleared.
# @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]
def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:
with belief_scope("TaskManager.clear_tasks"):
tasks_to_remove = []
@@ -607,16 +702,20 @@ class TaskManager:
# If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)
# Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.
# RUNNING is active execution.
should_remove = False
if status:
if task.status == status:
should_remove = True
else:
# Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)
if task.status not in [TaskStatus.RUNNING, TaskStatus.AWAITING_INPUT, TaskStatus.AWAITING_MAPPING]:
if task.status not in [
TaskStatus.RUNNING,
TaskStatus.AWAITING_INPUT,
TaskStatus.AWAITING_MAPPING,
]:
should_remove = True
if should_remove:
tasks_to_remove.append(task_id)
@@ -625,18 +724,21 @@ class TaskManager:
if tid in self.task_futures:
self.task_futures[tid].cancel()
del self.task_futures[tid]
del self.tasks[tid]
# Remove from persistence (task_records and task_logs via CASCADE)
self.persistence_service.delete_tasks(tasks_to_remove)
# Also explicitly delete logs (in case CASCADE is not set up)
if tasks_to_remove:
self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)
logger.info(f"Cleared {len(tasks_to_remove)} tasks.")
return len(tasks_to_remove)
# [/DEF:clear_tasks:Function]
# [/DEF:TaskManager:Class]
# [/DEF:TaskManager:Module]

56
backend/src/core/task_manager/models.py
View File

@@ -3,7 +3,8 @@
# @SEMANTICS: task, models, pydantic, enum, state
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
# @LAYER: Core
# @RELATION: Used by TaskManager and API routes.
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [USED_BY] -> [ApiRoutes]
# @INVARIANT: Task IDs are immutable once created.
# @CONSTRAINT: Must use Pydantic for data validation.
@@ -16,6 +17,7 @@ from typing import Dict, Any, List, Optional
from pydantic import BaseModel, Field
# [/SECTION]
# [DEF:TaskStatus:Enum]
# @COMPLEXITY: 1
# @SEMANTICS: task, status, state, enum
@@ -27,23 +29,33 @@ class TaskStatus(str, Enum):
FAILED = "FAILED"
AWAITING_MAPPING = "AWAITING_MAPPING"
AWAITING_INPUT = "AWAITING_INPUT"
# [/DEF:TaskStatus:Enum]
# [DEF:LogLevel:Enum]
# @SEMANTICS: log, level, severity, enum
# @PURPOSE: Defines the possible log levels for task logging.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskLogger]
class LogLevel(str, Enum):
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
# [/DEF:LogLevel:Enum]
# [DEF:LogEntry:Class]
# @SEMANTICS: log, entry, record, pydantic
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
# @COMPLEXITY: 5
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [LogLevel]
# @RELATION: [PART_OF] -> [Task]
# @INVARIANT: Each log entry has a unique timestamp and source.
#
# @TEST_CONTRACT: LogEntryModel ->
@@ -57,16 +69,25 @@ class LogEntry(BaseModel):
timestamp: datetime = Field(default_factory=datetime.utcnow)
level: str = Field(default="INFO")
message: str
source: str = Field(default="system") # Component attribution: plugin, superset_api, git, etc.
context: Optional[Dict[str, Any]] = None # Legacy field, kept for backward compatibility
metadata: Optional[Dict[str, Any]] = None # Structured metadata (e.g., dashboard_id, progress)
source: str = Field(
default="system"
) # Component attribution: plugin, superset_api, git, etc.
context: Optional[Dict[str, Any]] = (
None # Legacy field, kept for backward compatibility
)
metadata: Optional[Dict[str, Any]] = (
None # Structured metadata (e.g., dashboard_id, progress)
)
# [/DEF:LogEntry:Class]
# [DEF:TaskLog:Class]
# @SEMANTICS: task, log, persistent, pydantic
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
# @COMPLEXITY: 3
# @RELATION: MAPS_TO -> TaskLogRecord
# @RELATION: [MAPS_TO] -> [TaskLogRecord]
class TaskLog(BaseModel):
id: int
task_id: str
@@ -78,34 +99,48 @@ class TaskLog(BaseModel):
class Config:
from_attributes = True
# [/DEF:TaskLog:Class]
# [DEF:LogFilter:Class]
# @SEMANTICS: log, filter, query, pydantic
# @PURPOSE: Filter parameters for querying task logs.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [TaskManager]
class LogFilter(BaseModel):
level: Optional[str] = None # Filter by log level
source: Optional[str] = None # Filter by source component
search: Optional[str] = None # Text search in message
offset: int = Field(default=0, ge=0)
limit: int = Field(default=100, ge=1, le=1000)
# [/DEF:LogFilter:Class]
# [DEF:LogStats:Class]
# @SEMANTICS: log, stats, aggregation, pydantic
# @PURPOSE: Statistics about log entries for a task.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [COMPUTED_FROM] -> [TaskLog]
class LogStats(BaseModel):
total_count: int
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7}
# [/DEF:LogStats:Class]
# [DEF:Task:Class]
# @COMPLEXITY: 3
# @SEMANTICS: task, job, execution, state, pydantic
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
# @RELATION: [DEPENDS_ON] -> [TaskStatus]
# @RELATION: [CONTAINS] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskManager]
class Task(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
plugin_id: str
@@ -129,7 +164,10 @@ class Task(BaseModel):
super().__init__(**data)
if self.status == TaskStatus.AWAITING_INPUT and not self.input_request:
raise ValueError("input_request is required when status is AWAITING_INPUT")
# [/DEF:__init__:Function]
# [/DEF:Task:Class]
# [/DEF:TaskManagerModels:Module]

17
backend/src/core/task_manager/persistence.py
View File

@@ -7,7 +7,7 @@
# @POST: Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
@@ -36,7 +36,7 @@ from ..logger import logger, belief_scope
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
# @RELATION: [DEPENDS_ON] ->[Environment]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @INVARIANT: Persistence must handle potentially missing task fields natively.
#
# @TEST_CONTRACT: TaskPersistenceService ->
@@ -100,6 +100,7 @@ class TaskPersistenceService:
# @PRE: Session is active
# @POST: Returns existing environments.id or None when unresolved.
# @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]
# @RELATION: [DEPENDS_ON] ->[Environment]
@staticmethod
def _resolve_environment_id(session: Session, env_id: Optional[str]) -> Optional[str]:
with belief_scope("_resolve_environment_id"):
@@ -287,6 +288,7 @@ class TaskPersistenceService:
# @POST: Specified task records deleted from database.
# @PARAM: task_ids (List[str]) - List of task IDs to delete.
# @SIDE_EFFECT: Deletes rows from task_records table.
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
def delete_tasks(self, task_ids: List[str]) -> None:
if not task_ids:
return
@@ -313,7 +315,7 @@ class TaskPersistenceService:
# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @INVARIANT: Log entries are batch-inserted for performance.
#
# @TEST_CONTRACT: TaskLogPersistenceService ->
@@ -352,6 +354,7 @@ class TaskLogPersistenceService:
# @PARAM: logs (List[LogEntry]) - Log entries to insert.
# @SIDE_EFFECT: Writes to task_logs table.
# @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:
if not logs:
return
@@ -385,6 +388,9 @@ class TaskLogPersistenceService:
# @PARAM: log_filter (LogFilter) - Filter parameters.
# @RETURN: List[TaskLog] - Filtered log entries.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[LogFilter]
# @RELATION: [DEPENDS_ON] ->[TaskLog]
def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -438,6 +444,8 @@ class TaskLogPersistenceService:
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[LogStats]
def get_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -485,6 +493,7 @@ class TaskLogPersistenceService:
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def get_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -505,6 +514,7 @@ class TaskLogPersistenceService:
# @POST: All logs for the task are deleted.
# @PARAM: task_id (str) - The task ID.
# @SIDE_EFFECT: Deletes from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def delete_logs_for_task(self, task_id: str) -> None:
with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -527,6 +537,7 @@ class TaskLogPersistenceService:
# @POST: All logs for the tasks are deleted.
# @PARAM: task_ids (List[str]) - List of task IDs.
# @SIDE_EFFECT: Deletes rows from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def delete_logs_for_tasks(self, task_ids: List[str]) -> None:
if not task_ids:
return

90
backend/src/core/task_manager/task_logger.py
View File

@@ -2,18 +2,23 @@
# @SEMANTICS: task, logger, context, plugin, attribution
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log
# @COMPLEXITY: 5
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @COMPLEXITY: 2
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
# [SECTION: IMPORTS]
from typing import Dict, Any, Optional, Callable
# [/SECTION]
# [DEF:TaskLogger:Class]
# @SEMANTICS: logger, task, source, attribution
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
# @COMPLEXITY: 5
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @RELATION: [USED_BY] -> [TaskManager]
# @INVARIANT: All log calls include the task_id and source.
# @UX_STATE: Idle -> Logging -> (system records log)
#
@@ -33,17 +38,17 @@ from typing import Dict, Any, Optional, Callable
class TaskLogger:
"""
A dedicated logger for tasks that automatically tags logs with source attribution.
Usage:
logger = TaskLogger(task_id="abc123", add_log_fn=task_manager._add_log, source="plugin")
logger.info("Starting backup process")
logger.error("Failed to connect", metadata={"error_code": 500})
# Create sub-logger with different source
api_logger = logger.with_source("superset_api")
api_logger.info("Fetching dashboards")
"""
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskLogger with task context.
# @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
@@ -51,17 +56,13 @@ class TaskLogger:
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
# @PARAM: source (str) - Default source for logs (default: "plugin").
def __init__(
self,
task_id: str,
add_log_fn: Callable,
source: str = "plugin"
):
def __init__(self, task_id: str, add_log_fn: Callable, source: str = "plugin"):
self._task_id = task_id
self._add_log = add_log_fn
self._default_source = source
# [/DEF:__init__:Function]
# [DEF:with_source:Function]
# @PURPOSE: Create a sub-logger with a different default source.
# @PRE: source is a non-empty string.
@@ -71,12 +72,11 @@ class TaskLogger:
def with_source(self, source: str) -> "TaskLogger":
"""Create a sub-logger with a different source context."""
return TaskLogger(
task_id=self._task_id,
add_log_fn=self._add_log,
source=source
task_id=self._task_id, add_log_fn=self._add_log, source=source
)
# [/DEF:with_source:Function]
# [DEF:_log:Function]
# @PURPOSE: Internal method to log a message at a given level.
# @PRE: level is a valid log level string.
@@ -87,11 +87,11 @@ class TaskLogger:
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @UX_STATE: Logging -> (writing internal log)
def _log(
self,
level: str,
message: str,
self,
level: str,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Internal logging method."""
self._add_log(
@@ -99,10 +99,11 @@ class TaskLogger:
level=level,
message=message,
source=source or self._default_source,
metadata=metadata
metadata=metadata,
)
# [/DEF:_log:Function]
# [DEF:debug:Function]
# @PURPOSE: Log a DEBUG level message.
# @PRE: message is a string.
@@ -111,14 +112,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def debug(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("DEBUG", message, source, metadata)
# [/DEF:debug:Function]
# [DEF:info:Function]
# @PURPOSE: Log an INFO level message.
# @PRE: message is a string.
@@ -127,14 +129,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def info(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("INFO", message, source, metadata)
# [/DEF:info:Function]
# [DEF:warning:Function]
# @PURPOSE: Log a WARNING level message.
# @PRE: message is a string.
@@ -143,14 +146,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def warning(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("WARNING", message, source, metadata)
# [/DEF:warning:Function]
# [DEF:error:Function]
# @PURPOSE: Log an ERROR level message.
# @PRE: message is a string.
@@ -159,14 +163,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def error(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("ERROR", message, source, metadata)
# [/DEF:error:Function]
# [DEF:progress:Function]
# @PURPOSE: Log a progress update with percentage.
# @PRE: percent is between 0 and 100.
@@ -175,16 +180,15 @@ class TaskLogger:
# @PARAM: percent (float) - Progress percentage (0-100).
# @PARAM: source (Optional[str]) - Override source.
def progress(
self,
message: str,
percent: float,
source: Optional[str] = None
self, message: str, percent: float, source: Optional[str] = None
) -> None:
"""Log a progress update with percentage."""
metadata = {"progress": min(100, max(0, percent))}
self._log("INFO", message, source, metadata)
# [/DEF:progress:Function]
# [/DEF:TaskLogger:Class]
# [/DEF:TaskLoggerModule:Module]

4
backend/src/models/__init__.py
View File

@@ -1,3 +1,3 @@
# [DEF:src.models:Package]
# [DEF:ModelsPackage:Package]
# @PURPOSE: Domain model package root.
# [/DEF:src.models:Package]
# [/DEF:ModelsPackage:Package]

73
backend/src/models/auth.py
View File

@@ -5,7 +5,7 @@
# @SEMANTICS: auth, models, user, role, permission, sqlalchemy
# @PURPOSE: SQLAlchemy models for multi-user authentication and authorization.
# @LAYER: Domain
# @RELATION: INHERITS_FROM -> [MappingModels:Base]
# @RELATION: INHERITS_FROM -> [Base]
#
# @INVARIANT: Usernames and emails must be unique.
@@ -17,19 +17,22 @@ from sqlalchemy.orm import relationship
from .mapping import Base
# [/SECTION]
# [DEF:generate_uuid:Function]
# @PURPOSE: Generates a unique UUID string.
# @POST: Returns a string representation of a new UUID.
# @RELATION: DEPENDS_ON -> uuid
# @RELATION: DEPENDS_ON -> [uuid]
def generate_uuid():
return str(uuid.uuid4())
# [/DEF:generate_uuid:Function]
# [DEF:user_roles:Table]
# @PURPOSE: Association table for many-to-many relationship between Users and Roles.
# @RELATION: DEPENDS_ON -> Base.metadata
# @RELATION: DEPENDS_ON -> User
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> [Base]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
user_roles = Table(
"user_roles",
Base.metadata,
@@ -40,9 +43,9 @@ user_roles = Table(
# [DEF:role_permissions:Table]
# @PURPOSE: Association table for many-to-many relationship between Roles and Permissions.
# @RELATION: DEPENDS_ON -> Base.metadata
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Base]
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
role_permissions = Table(
"role_permissions",
Base.metadata,
@@ -51,65 +54,81 @@ role_permissions = Table(
)
# [/DEF:role_permissions:Table]
# [DEF:User:Class]
# @PURPOSE: Represents an identity that can authenticate to the system.
# @RELATION: HAS_MANY -> Role (via user_roles)
# @RELATION: HAS_MANY -> [Role]
class User(Base):
__tablename__ = "users"
id = Column(String, primary_key=True, default=generate_uuid)
username = Column(String, unique=True, index=True, nullable=False)
email = Column(String, unique=True, index=True, nullable=True)
password_hash = Column(String, nullable=True)
full_name = Column(String, nullable=True)
auth_source = Column(String, default="LOCAL") # LOCAL or ADFS
auth_source = Column(String, default="LOCAL") # LOCAL or ADFS
is_active = Column(Boolean, default=True)
is_ad_user = Column(Boolean, default=False)
created_at = Column(DateTime, default=datetime.utcnow)
last_login = Column(DateTime, nullable=True)
roles = relationship("Role", secondary=user_roles, back_populates="users")
# [/DEF:User:Class]
# [DEF:Role:Class]
# @PURPOSE: Represents a collection of permissions.
# @RELATION: HAS_MANY -> User (via user_roles)
# @RELATION: HAS_MANY -> Permission (via role_permissions)
# @RELATION: HAS_MANY -> [User]
# @RELATION: HAS_MANY -> [Permission]
class Role(Base):
__tablename__ = "roles"
id = Column(String, primary_key=True, default=generate_uuid)
name = Column(String, unique=True, index=True, nullable=False)
description = Column(String, nullable=True)
users = relationship("User", secondary=user_roles, back_populates="roles")
permissions = relationship("Permission", secondary=role_permissions, back_populates="roles")
permissions = relationship(
"Permission", secondary=role_permissions, back_populates="roles"
)
# [/DEF:Role:Class]
# [DEF:Permission:Class]
# @PURPOSE: Represents a specific capability within the system.
# @RELATION: HAS_MANY -> Role (via role_permissions)
# @RELATION: HAS_MANY -> [Role]
class Permission(Base):
__tablename__ = "permissions"
id = Column(String, primary_key=True, default=generate_uuid)
resource = Column(String, nullable=False) # e.g. "plugin:backup"
action = Column(String, nullable=False) # e.g. "READ", "EXECUTE", "WRITE"
roles = relationship("Role", secondary=role_permissions, back_populates="permissions")
resource = Column(String, nullable=False) # e.g. "plugin:backup"
action = Column(String, nullable=False) # e.g. "READ", "EXECUTE", "WRITE"
roles = relationship(
"Role", secondary=role_permissions, back_populates="permissions"
)
# [/DEF:Permission:Class]
# [DEF:ADGroupMapping:Class]
# @PURPOSE: Maps an Active Directory group to a local System Role.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> [Role]
class ADGroupMapping(Base):
__tablename__ = "ad_group_mappings"
id = Column(String, primary_key=True, default=generate_uuid)
ad_group = Column(String, unique=True, index=True, nullable=False)
role_id = Column(String, ForeignKey("roles.id"), nullable=False)
role = relationship("Role")
# [/DEF:ADGroupMapping:Class]
# [/DEF:AuthModels:Module]
# [/DEF:AuthModels:Module]

20
backend/src/scripts/init_auth_db.py
View File

@@ -1,9 +1,12 @@
# [DEF:backend.src.scripts.init_auth_db:Module]
# [DEF:InitAuthDbScript:Module]
#
# @SEMANTICS: setup, database, auth, migration
# @PURPOSE: Initializes the auth database and creates the necessary tables.
# @COMPLEXITY: 2
# @LAYER: Scripts
# @RELATION: CALLS -> backend.src.core.database.init_db
# @RELATION: CALLS -> init_db
# @RELATION: CALLS -> ensure_encryption_key
# @RELATION: CALLS -> seed_permissions
#
# @INVARIANT: Safe to run multiple times (idempotent).
@@ -20,9 +23,14 @@ from src.core.logger import logger, belief_scope
from src.scripts.seed_permissions import seed_permissions
# [/SECTION]
# [DEF:run_init:Function]
# @PURPOSE: Main entry point for the initialization script.
# @COMPLEXITY: 3
# @POST: auth.db is initialized with the correct schema and seeded permissions.
# @RELATION: CALLS -> ensure_encryption_key
# @RELATION: CALLS -> init_db
# @RELATION: CALLS -> seed_permissions
def run_init():
with belief_scope("init_auth_db"):
logger.info("Initializing authentication database...")
@@ -30,16 +38,18 @@ def run_init():
ensure_encryption_key()
init_db()
logger.info("Authentication database initialized successfully.")
# Seed permissions
seed_permissions()
except Exception as e:
logger.error(f"Failed to initialize authentication database: {e}")
sys.exit(1)
# [/DEF:run_init:Function]
if __name__ == "__main__":
run_init()
# [/DEF:backend.src.scripts.init_auth_db:Module]
# [/DEF:InitAuthDbScript:Module]

52
backend/src/scripts/seed_permissions.py
View File

@@ -1,10 +1,13 @@
# [DEF:backend.src.scripts.seed_permissions:Module]
# [DEF:SeedPermissionsScript:Module]
#
# @SEMANTICS: setup, database, auth, permissions, seeding
# @PURPOSE: Populates the auth database with initial system permissions.
# @COMPLEXITY: 3
# @LAYER: Scripts
# @RELATION: USES -> backend.src.core.database.get_auth_db
# @RELATION: USES -> backend.src.models.auth.Permission
# @RELATION: DEPENDS_ON -> AuthSessionLocal
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> AuthRepository
#
# @INVARIANT: Safe to run multiple times (idempotent).
@@ -22,6 +25,9 @@ from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:INITIAL_PERMISSIONS:Constant]
# @PURPOSE: Canonical bootstrap permission tuples seeded into auth storage.
# @COMPLEXITY: 3
# @RELATION: DEPENDS_ON -> SeedPermissionsScript
INITIAL_PERMISSIONS = [
# Admin Permissions
{"resource": "admin:users", "action": "READ"},
@@ -34,7 +40,6 @@ INITIAL_PERMISSIONS = [
{"resource": "plugins", "action": "READ"},
{"resource": "tasks", "action": "READ"},
{"resource": "tasks", "action": "WRITE"},
# Plugin Permissions
{"resource": "plugin:backup", "action": "EXECUTE"},
{"resource": "plugin:migration", "action": "EXECUTE"},
@@ -46,7 +51,6 @@ INITIAL_PERMISSIONS = [
{"resource": "plugin:storage", "action": "WRITE"},
{"resource": "plugin:debug", "action": "EXECUTE"},
{"resource": "git_config", "action": "READ"},
# Dataset Review Permissions
{"resource": "dataset:session", "action": "READ"},
{"resource": "dataset:session", "action": "MANAGE"},
@@ -57,9 +61,16 @@ INITIAL_PERMISSIONS = [
]
# [/DEF:INITIAL_PERMISSIONS:Constant]
# [DEF:seed_permissions:Function]
# @PURPOSE: Inserts missing permissions into the database.
# @COMPLEXITY: 3
# @POST: All INITIAL_PERMISSIONS exist in the DB.
# @RELATION: DEPENDS_ON -> AuthSessionLocal
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> AuthRepository
# @RELATION: DEPENDS_ON -> INITIAL_PERMISSIONS
def seed_permissions():
with belief_scope("seed_permissions"):
db = AuthSessionLocal()
@@ -67,19 +78,22 @@ def seed_permissions():
logger.info("Seeding permissions...")
count = 0
for perm_data in INITIAL_PERMISSIONS:
exists = db.query(Permission).filter(
Permission.resource == perm_data["resource"],
Permission.action == perm_data["action"]
).first()
exists = (
db.query(Permission)
.filter(
Permission.resource == perm_data["resource"],
Permission.action == perm_data["action"],
)
.first()
)
if not exists:
new_perm = Permission(
resource=perm_data["resource"],
action=perm_data["action"]
resource=perm_data["resource"], action=perm_data["action"]
)
db.add(new_perm)
count += 1
db.commit()
logger.info(f"Seeding completed. Added {count} new permissions.")
@@ -87,10 +101,12 @@ def seed_permissions():
repo = AuthRepository(db)
user_role = repo.get_role_by_name("User")
if not user_role:
user_role = Role(name="User", description="Standard user with plugin access")
user_role = Role(
name="User", description="Standard user with plugin access"
)
db.add(user_role)
db.flush()
user_permissions = [
("plugin:mapper", "EXECUTE"),
("plugin:migration", "EXECUTE"),
@@ -113,7 +129,7 @@ def seed_permissions():
perm = repo.get_permission_by_resource_action(res, act)
if perm and perm not in user_role.permissions:
user_role.permissions.append(perm)
db.commit()
logger.info("User role permissions updated.")
@@ -122,9 +138,11 @@ def seed_permissions():
db.rollback()
finally:
db.close()
# [/DEF:seed_permissions:Function]
if __name__ == "__main__":
seed_permissions()
# [/DEF:backend.src.scripts.seed_permissions:Module]
# [/DEF:SeedPermissionsScript:Module]

6
backend/src/scripts/test_dataset_dashboard_relations.py
View File

@@ -1,4 +1,8 @@
#!/usr/bin/env python3
# [DEF:test_dataset_dashboard_relations_script:Module]
# @SEMANTICS: scripts, test, dataset, dashboard, superset, relations
# @PURPOSE: Tests and inspects dataset-to-dashboard relationship responses from Superset API.
# @COMPLEXITY: 2
"""
Script to test dataset-to-dashboard relationships from Superset API.
@@ -164,3 +168,5 @@ def test_dashboard_dataset_relations():
if __name__ == "__main__":
test_dashboard_dataset_relations()
# [/DEF:test_dataset_dashboard_relations_script:Module]

48
backend/src/services/__tests__/test_llm_plugin_persistence.py
View File

@@ -1,5 +1,5 @@
# [DEF:test_llm_plugin_persistence:Module]
# @RELATION: VERIFIES ->[src.plugins.llm_analysis.plugin.DashboardValidationPlugin]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context.
@@ -10,7 +10,7 @@ from src.plugins.llm_analysis import plugin as plugin_module
# [DEF:_DummyLogger:Class]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 1
# @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests.
# @INVARIANT: Logging methods are no-ops and must not mutate test state.
@@ -35,7 +35,7 @@ class _DummyLogger:
# [DEF:_FakeDBSession:Class]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 2
# @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin.
# @INVARIANT: add/commit/close provide only persistence signals asserted by this test.
@@ -59,8 +59,11 @@ class _FakeDBSession:
# [DEF:test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 2
# @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id.
# @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals.
@pytest.mark.asyncio
async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
tmp_path, monkeypatch
@@ -77,6 +80,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
is_active=True,
)
# [DEF:_FakeProviderService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests.
# @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic.
class _FakeProviderService:
def __init__(self, _db):
return None
@@ -87,6 +95,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
def get_decrypted_api_key(self, _provider_id):
return "a" * 32
# [/DEF:_FakeProviderService:Class]
# [DEF:_FakeScreenshotService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects.
# @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values.
class _FakeScreenshotService:
def __init__(self, _env):
return None
@@ -94,8 +109,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
async def capture_dashboard(self, _dashboard_id, _screenshot_path):
return None
# [/DEF:_FakeScreenshotService:Class]
# [DEF:_FakeLLMClient:Class]
# @RELATION: BINDS_TO ->[test_dashboard_validation_plugin_persists_task_and_environment_ids]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 2
# @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions.
# @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result.
@@ -112,6 +129,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# [/DEF:_FakeLLMClient:Class]
# [DEF:_FakeNotificationService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects.
# @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema.
class _FakeNotificationService:
def __init__(self, *_args, **_kwargs):
return None
@@ -119,6 +141,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
async def dispatch_report(self, **_kwargs):
return None
# [/DEF:_FakeNotificationService:Class]
# [DEF:_FakeConfigManager:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path.
# @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent.
class _FakeConfigManager:
def get_environment(self, _env_id):
return env
@@ -131,12 +160,21 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
)
)
# [/DEF:_FakeConfigManager:Class]
# [DEF:_FakeSupersetClient:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list.
# @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency.
class _FakeSupersetClient:
def __init__(self, _env):
self.network = types.SimpleNamespace(
request=lambda **_kwargs: {"result": []}
)
# [/DEF:_FakeSupersetClient:Class]
monkeypatch.setattr(plugin_module, "SessionLocal", lambda: fake_db)
monkeypatch.setattr(plugin_module, "LLMProviderService", _FakeProviderService)
monkeypatch.setattr(plugin_module, "ScreenshotService", _FakeScreenshotService)

30
backend/src/services/__tests__/test_llm_prompt_templates.py
View File

@@ -3,7 +3,7 @@
# @SEMANTICS: tests, llm, prompts, templates, settings
# @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates.
# @LAYER: Domain Tests
# @RELATION: DEPENDS_ON ->[backend.src.services.llm_prompt_templates:Function]
# @RELATION: DEPENDS_ON -> [backend.src.services.llm_prompt_templates:Function]
# @INVARIANT: All required prompt keys remain available after normalization.
from src.services.llm_prompt_templates import (
@@ -23,8 +23,6 @@ from src.services.llm_prompt_templates import (
# @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults.
# @PRE: Input llm settings do not contain complete prompts object.
# @POST: Returned structure includes required prompt templates with fallback defaults.
# [DEF:test_normalize_llm_settings_adds_default_prompts:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_normalize_llm_settings_adds_default_prompts():
normalized = normalize_llm_settings({"default_provider": "x"})
@@ -38,6 +36,8 @@ def test_normalize_llm_settings_adds_default_prompts():
assert key in normalized["provider_bindings"]
for key in DEFAULT_LLM_ASSISTANT_SETTINGS:
assert key in normalized
# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function]
@@ -47,17 +47,13 @@ def test_normalize_llm_settings_adds_default_prompts():
# @PURPOSE: Ensure user-customized prompt values are preserved during normalization.
# @PRE: Input llm settings contain custom prompt override.
# @POST: Custom prompt value remains unchanged in normalized output.
# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function]
# [DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_normalize_llm_settings_keeps_custom_prompt_values():
custom = "Doc for {dataset_name} using {columns_json}"
normalized = normalize_llm_settings(
{"prompts": {"documentation_prompt": custom}}
)
normalized = normalize_llm_settings({"prompts": {"documentation_prompt": custom}})
assert normalized["prompts"]["documentation_prompt"] == custom
# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
@@ -67,10 +63,6 @@ def test_normalize_llm_settings_keeps_custom_prompt_values():
# @PURPOSE: Ensure template placeholders are deterministically replaced.
# @PRE: Template contains placeholders matching provided variables.
# @POST: Rendered prompt string contains substituted values.
# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
# [DEF:test_render_prompt_replaces_known_placeholders:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_render_prompt_replaces_known_placeholders():
rendered = render_prompt(
"Hello {name}, diff={diff}",
@@ -78,6 +70,8 @@ def test_render_prompt_replaces_known_placeholders():
)
assert rendered == "Hello bot, diff=A->B"
# [/DEF:test_render_prompt_replaces_known_placeholders:Function]
@@ -85,13 +79,13 @@ def test_render_prompt_replaces_known_placeholders():
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 2
# @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names.
# [/DEF:test_render_prompt_replaces_known_placeholders:Function]
def test_is_multimodal_model_detects_known_vision_models():
assert is_multimodal_model("gpt-4o") is True
assert is_multimodal_model("claude-3-5-sonnet") is True
assert is_multimodal_model("stepfun/step-3.5-flash:free", "openrouter") is False
assert is_multimodal_model("text-only-model") is False
# [/DEF:test_is_multimodal_model_detects_known_vision_models:Function]
@@ -106,6 +100,8 @@ def test_resolve_bound_provider_id_prefers_binding_then_default():
}
assert resolve_bound_provider_id(settings, "dashboard_validation") == "vision-1"
assert resolve_bound_provider_id(settings, "documentation") == "default-1"
# [/DEF:test_resolve_bound_provider_id_prefers_binding_then_default:Function]
@@ -122,6 +118,8 @@ def test_normalize_llm_settings_keeps_assistant_planner_settings():
)
assert normalized["assistant_planner_provider"] == "provider-a"
assert normalized["assistant_planner_model"] == "gpt-4.1-mini"
# [/DEF:test_normalize_llm_settings_keeps_assistant_planner_settings:Function]

76
backend/src/services/__tests__/test_llm_provider.py
View File

@@ -1,7 +1,9 @@
# [DEF:__tests__/test_llm_provider:Module]
# @RELATION: VERIFIES -> ../llm_provider.py
# [DEF:test_llm_provider:Module]
# @RELATION: VERIFIES -> [src.services.llm_provider:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-provider, encryption, contract
# @PURPOSE: Contract testing for LLMProviderService and EncryptionManager
# [/DEF:__tests__/test_llm_provider:Module]
# [/DEF:test_llm_provider:Module]
import pytest
import os
@@ -14,14 +16,16 @@ from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType
# [DEF:_test_encryption_key_fixture:Global]
# @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key.
# @RELATION: DEPENDS_ON ->[pytest:Module]
# @RELATION: DEPENDS_ON -> [pytest:Module]
os.environ.setdefault("ENCRYPTION_KEY", Fernet.generate_key().decode())
# [/DEF:_test_encryption_key_fixture:Global]
# @TEST_CONTRACT: EncryptionManagerModel -> Invariants
# @TEST_INVARIANT: symmetric_encryption
# [DEF:test_encryption_cycle:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets.
def test_encryption_cycle():
"""Verify encrypted data can be decrypted back to original string."""
manager = EncryptionManager()
@@ -30,49 +34,78 @@ def test_encryption_cycle():
assert encrypted != original
assert manager.decrypt(encrypted) == original
# @TEST_EDGE: empty_string_encryption
# [/DEF:test_encryption_cycle:Function]
# [DEF:test_empty_string_encryption:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle.
def test_empty_string_encryption():
manager = EncryptionManager()
original = ""
encrypted = manager.encrypt(original)
assert manager.decrypt(encrypted) == ""
# @TEST_EDGE: decrypt_invalid_data
# [/DEF:test_empty_string_encryption:Function]
# [DEF:test_decrypt_invalid_data:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure decrypt rejects invalid ciphertext input by raising an exception.
def test_decrypt_invalid_data():
manager = EncryptionManager()
with pytest.raises(Exception):
manager.decrypt("not-encrypted-string")
# @TEST_FIXTURE: mock_db_session
# [/DEF:test_decrypt_invalid_data:Function]
# [DEF:mock_db:Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests.
# @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced.
@pytest.fixture
def mock_db():
return MagicMock(spec=Session)
# [/DEF:mock_db:Fixture]
# [DEF:service:Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests.
@pytest.fixture
def service(mock_db):
return LLMProviderService(db=mock_db)
# [/DEF:service:Fixture]
# [DEF:test_get_all_providers:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify provider list retrieval issues query/all calls on the backing DB session.
def test_get_all_providers(service, mock_db):
service.get_all_providers()
mock_db.query.assert_called()
mock_db.query().all.assert_called()
# [/DEF:test_get_all_providers:Function]
# [DEF:test_create_provider:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure provider creation persists entity and stores API key in encrypted form.
def test_create_provider(service, mock_db):
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
@@ -80,11 +113,11 @@ def test_create_provider(service, mock_db):
base_url="https://api.openai.com",
api_key="sk-test",
default_model="gpt-4",
is_active=True
is_active=True,
)
provider = service.create_provider(config)
mock_db.add.assert_called()
mock_db.commit.assert_called()
# Verify API key was encrypted
@@ -92,31 +125,40 @@ def test_create_provider(service, mock_db):
# Decrypt to verify it matches
assert EncryptionManager().decrypt(provider.api_key) == "sk-test"
# [/DEF:test_create_provider:Function]
# [DEF:test_get_decrypted_api_key:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify service decrypts stored provider API key for an existing provider record.
def test_get_decrypted_api_key(service, mock_db):
# Setup mock provider
encrypted_key = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(id="p1", api_key=encrypted_key)
mock_db.query().filter().first.return_value = mock_provider
key = service.get_decrypted_api_key("p1")
assert key == "secret-value"
# [/DEF:test_get_decrypted_api_key:Function]
# [DEF:test_get_decrypted_api_key_not_found:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify missing provider lookup returns None instead of attempting decryption.
def test_get_decrypted_api_key_not_found(service, mock_db):
mock_db.query().filter().first.return_value = None
assert service.get_decrypted_api_key("missing") is None
# [/DEF:test_get_decrypted_api_key_not_found:Function]
# [DEF:test_update_provider_ignores_masked_placeholder_api_key:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets.
def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db):
existing_encrypted = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(
@@ -145,4 +187,6 @@ def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db):
assert updated.api_key == existing_encrypted
assert EncryptionManager().decrypt(updated.api_key) == "secret-value"
assert updated.is_active is False
# [/DEF:test_update_provider_ignores_masked_placeholder_api_key:Function]

48
backend/src/services/auth_service.py
View File

@@ -3,11 +3,11 @@
# @SEMANTICS: auth, service, business-logic, login, jwt, adfs, jit-provisioning
# @PURPOSE: Orchestrates credential authentication and ADFS JIT user provisioning.
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.security.verify_password]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.jwt.create_access_token]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.User]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.Role]
# @RELATION: DEPENDS_ON -> [AuthRepository]
# @RELATION: DEPENDS_ON -> [verify_password]
# @RELATION: DEPENDS_ON -> [create_access_token]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
# @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles.
# @PRE: Core auth models and security utilities available.
# @POST: User identity verified and session tokens issued according to role scopes.
@@ -41,9 +41,10 @@ class AuthService:
def __init__(self, db: Session):
self.db = db
self.repo = AuthRepository(db)
# [/DEF:AuthService_init:Function]
# [DEF:authenticate_user:Function]
# [DEF:AuthService.authenticate_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Validates credentials and account state for local username/password authentication.
# @PRE: username and password are non-empty credential inputs.
@@ -58,19 +59,20 @@ class AuthService:
user = self.repo.get_user_by_username(username)
if not user or not user.is_active:
return None
if not verify_password(password, user.password_hash):
return None
# Update last login
user.last_login = datetime.utcnow()
self.db.commit()
self.db.refresh(user)
return user
# [/DEF:authenticate_user:Function]
# [DEF:create_session:Function]
return user
# [/DEF:AuthService.authenticate_user:Function]
# [DEF:AuthService.create_session:Function]
# @COMPLEXITY: 3
# @PURPOSE: Issues an access token payload for an already authenticated user.
# @PRE: user is a valid User entity containing username and iterable roles with role.name values.
@@ -86,9 +88,10 @@ class AuthService:
data={"sub": user.username, "scopes": roles}
)
return {"access_token": access_token, "token_type": "bearer"}
# [/DEF:create_session:Function]
# [DEF:provision_adfs_user:Function]
# [/DEF:AuthService.create_session:Function]
# [DEF:AuthService.provision_adfs_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings.
# @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent.
@@ -102,7 +105,7 @@ class AuthService:
username = user_info.get("upn") or user_info.get("email")
email = user_info.get("email")
groups = user_info.get("groups", [])
user = self.repo.get_user_by_username(username)
if not user:
user = User(
@@ -111,21 +114,24 @@ class AuthService:
full_name=user_info.get("name"),
auth_source="ADFS",
is_active=True,
is_ad_user=True
is_ad_user=True,
)
self.db.add(user)
log_security_event("USER_PROVISIONED", username, {"source": "ADFS"})
# Sync roles from AD groups
mapped_roles = self.repo.get_roles_by_ad_groups(groups)
user.roles = mapped_roles
user.last_login = datetime.utcnow()
self.db.commit()
self.db.refresh(user)
return user
# [/DEF:provision_adfs_user:Function]
# [/DEF:AuthService.provision_adfs_user:Function]
# [/DEF:AuthService:Class]
# [/DEF:auth_service:Module]
# [/DEF:auth_service:Module]

17
backend/src/services/dataset_review/orchestrator.py
View File

@@ -5,10 +5,10 @@
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @RELATION: [DEPENDS_ON] ->[ClarificationEngine]
# @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor]
# @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [CONTAINS] ->[DatasetReviewOrchestrator]
# @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user.
# @POST: state transitions are persisted atomically and emit observable progress for long-running steps.
# @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts.
@@ -158,8 +158,8 @@ class LaunchDatasetResult:
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @PRE: constructor dependencies are valid and tied to the current request/task scope.
# @POST: orchestrator instance can execute session-scoped mutations for one authenticated user.
# @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks.
@@ -169,8 +169,13 @@ class DatasetReviewOrchestrator:
# [DEF:DatasetReviewOrchestrator_init:Function]
# @COMPLEXITY: 3
# @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary.
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @PRE: repository/config_manager are valid collaborators for the current request scope.
# @POST: Instance holds collaborator references used by start/preview/launch orchestration methods.
# @SIDE_EFFECT: Stores dependency references for later session lifecycle operations.
def __init__(
self,
repository: DatasetReviewSessionRepository,
@@ -188,9 +193,9 @@ class DatasetReviewOrchestrator:
# [DEF:start_session:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery.
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [CALLS] ->[SupersetContextExtractor.parse_superset_link]
# @RELATION: [CALLS] ->[create_task]
# @RELATION: [CALLS] ->[TaskManager.create_task]
# @PRE: source input is non-empty and environment is accessible.
# @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required.
# @SIDE_EFFECT: persists session and may enqueue recovery task.
@@ -1112,7 +1117,7 @@ class DatasetReviewOrchestrator:
# [DEF:_enqueue_recovery_task:Function]
# @COMPLEXITY: 4
# @PURPOSE: Link session start to observable async recovery when task infrastructure is available.
# @RELATION: [CALLS] ->[create_task]
# @RELATION: [CALLS] ->[TaskManager.create_task]
# @PRE: session is already persisted.
# @POST: returns task identifier when a task could be enqueued, otherwise None.
# @SIDE_EFFECT: may create one background task for progressive recovery.

169
backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py
View File

@@ -16,34 +16,46 @@ from src.models.dataset_review import (
ReadinessState,
RecommendedAction,
SessionCollaborator,
SessionCollaboratorRole
SessionCollaboratorRole,
)
from src.services.dataset_review.repositories.session_repository import (
DatasetReviewSessionRepository,
)
from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository
# [DEF:SessionRepositoryTests:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 2
# @PURPOSE: Unit tests for DatasetReviewSessionRepository.
@pytest.fixture
def db_session():
# [DEF:db_session:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows.
# @RELATION: BINDS_TO -> [SessionRepositoryTests]
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
# Create test data
user = User(id="user1", username="testuser", email="test@example.com", password_hash="pw")
env = Environment(id="env1", name="Prod", url="http://superset", credentials_id="cred1")
user = User(
id="user1", username="testuser", email="test@example.com", password_hash="pw"
)
env = Environment(
id="env1", name="Prod", url="http://superset", credentials_id="cred1"
)
session.add_all([user, env])
session.commit()
yield session
session.close()
# [/DEF:db_session:Function]
# [DEF:test_create_session:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_create_session(db_session):
@@ -54,126 +66,156 @@ def test_create_session(db_session):
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1"
dataset_ref="dataset1",
)
repo.create_session(session)
assert session.session_id is not None
loaded = db_session.query(DatasetReviewSession).filter_by(session_id=session.session_id).first()
loaded = (
db_session.query(DatasetReviewSession)
.filter_by(session_id=session.session_id)
.first()
)
assert loaded.user_id == "user1"
# [/DEF:test_create_session:Function]
# [DEF:test_load_session_detail_ownership:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_load_session_detail_ownership(db_session):
# @PURPOSE: Verify ownership enforcement in detail loading.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
# Correct user
loaded = repo.load_session_detail(session.session_id, "user1")
assert loaded is not None
# Wrong user
loaded_wrong = repo.load_session_detail(session.session_id, "wrong_user")
assert loaded_wrong is None
# [/DEF:test_load_session_detail_ownership:Function]
# [DEF:test_load_session_detail_collaborator:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_load_session_detail_collaborator(db_session):
# @PURPOSE: Verify collaborator access in detail loading.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
# Add collaborator
collab_user = User(id="collab1", username="collab", email="c@e.com", password_hash="p")
collab_user = User(
id="collab1", username="collab", email="c@e.com", password_hash="p"
)
db_session.add(collab_user)
collaborator = SessionCollaborator(
session_id=session.session_id,
user_id="collab1",
role=SessionCollaboratorRole.REVIEWER
role=SessionCollaboratorRole.REVIEWER,
)
db_session.add(collaborator)
db_session.commit()
# Collaborator access
loaded = repo.load_session_detail(session.session_id, "collab1")
assert loaded is not None
assert loaded.session_id == session.session_id
# [/DEF:test_load_session_detail_collaborator:Function]
# [DEF:test_save_preview_marks_stale:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_preview_marks_stale(db_session):
# @PURPOSE: Verify that saving a new preview marks old ones as stale.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
p1 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f1")
p1 = CompiledPreview(
session_id=session.session_id, preview_status="ready", preview_fingerprint="f1"
)
repo.save_preview(session.session_id, "user1", p1)
p2 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f2")
p2 = CompiledPreview(
session_id=session.session_id, preview_status="ready", preview_fingerprint="f2"
)
repo.save_preview(session.session_id, "user1", p2)
db_session.refresh(p1)
assert p1.preview_status == "stale"
assert p2.preview_status == "ready"
assert session.last_preview_id == p2.preview_id
# [/DEF:test_save_preview_marks_stale:Function]
# [DEF:test_save_profile_and_findings:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_profile_and_findings(db_session):
# @PURPOSE: Verify persistence of profile and findings.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
profile = DatasetProfile(
session_id=session.session_id,
dataset_name="Test DS",
business_summary="Summary",
business_summary_source=BusinessSummarySource.INFERRED,
confidence_state=ConfidenceState.UNRESOLVED
confidence_state=ConfidenceState.UNRESOLVED,
)
finding = ValidationFinding(
session_id=session.session_id,
area=FindingArea.SOURCE_INTAKE,
severity=FindingSeverity.BLOCKING,
code="ERR1",
title="Error",
message="Failure"
message="Failure",
)
repo.save_profile_and_findings(session.session_id, "user1", profile, [finding])
updated_session = repo.load_session_detail(session.session_id, "user1")
assert updated_session.profile.dataset_name == "Test DS"
assert len(updated_session.findings) == 1
assert updated_session.findings[0].code == "ERR1"
# Verify removal of old findings
new_finding = ValidationFinding(
session_id=session.session_id,
@@ -181,29 +223,34 @@ def test_save_profile_and_findings(db_session):
severity=FindingSeverity.WARNING,
code="WARN1",
title="Warning",
message="Something"
message="Something",
)
repo.save_profile_and_findings(session.session_id, "user1", profile, [new_finding])
db_session.expire_all()
final_session = repo.load_session_detail(session.session_id, "user1")
assert len(final_session.findings) == 1
assert final_session.findings[0].code == "WARN1"
# [/DEF:test_save_profile_and_findings:Function]
# [DEF:test_save_run_context:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_run_context(db_session):
# @PURPOSE: Verify saving of run context.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
rc = DatasetRunContext(
session_id=session.session_id,
dataset_ref="ds1",
@@ -215,28 +262,50 @@ def test_save_run_context(db_session):
approved_mapping_ids=[],
semantic_decision_refs=[],
open_warning_refs=[],
launch_status="success"
launch_status="success",
)
repo.save_run_context(session.session_id, "user1", rc)
assert session.last_run_context_id == rc.run_context_id
# [/DEF:test_save_run_context:Function]
# [DEF:test_list_sessions_for_user:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_list_sessions_for_user(db_session):
# @PURPOSE: Verify listing of sessions by user.
repo = DatasetReviewSessionRepository(db_session)
s1 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r1")
s2 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r2")
s3 = DatasetReviewSession(user_id="other", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r3")
s1 = DatasetReviewSession(
user_id="user1",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r1",
)
s2 = DatasetReviewSession(
user_id="user1",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r2",
)
s3 = DatasetReviewSession(
user_id="other",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r3",
)
db_session.add_all([s1, s2, s3])
db_session.commit()
sessions = repo.list_sessions_for_user("user1")
assert len(sessions) == 2
assert all(s.user_id == "user1" for s in sessions)
# [/DEF:SessionRepositoryTests:Module]# [/DEF:test_list_sessions_for_user:Function]
# [/DEF:test_list_sessions_for_user:Function]
# [/DEF:SessionRepositoryTests:Module]

5
backend/src/services/dataset_review/repositories/session_repository.py
View File

@@ -34,13 +34,14 @@ from src.core.logger import belief_scope, logger
from src.services.dataset_review.event_logger import SessionEventLogger
# [DEF:SessionRepo:Class]
# [DEF:DatasetReviewSessionRepository:Class]
# @COMPLEXITY: 4
# @PURPOSE: Enforce ownership-scoped persistence and retrieval for dataset review session aggregates.
# @RELATION: [DEPENDS_ON] -> [DatasetReviewSession]
# @RELATION: [DEPENDS_ON] -> [DatasetProfile]
# @RELATION: [DEPENDS_ON] -> [ValidationFinding]
# @RELATION: [DEPENDS_ON] -> [CompiledPreview]
# @RELATION: [DEPENDS_ON] -> [SessionEventLogger]
# @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope for guarded reads and writes.
# @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning.
# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session.
@@ -409,6 +410,6 @@ class DatasetReviewSessionRepository:
# [/DEF:list_user_sess:Function]
# [/DEF:SessionRepo:Class]
# [/DEF:DatasetReviewSessionRepository:Class]
# [/DEF:DatasetReviewSessionRepository:Module]

119
backend/src/services/health_service.py
View File

@@ -19,30 +19,44 @@ from ..core.superset_client import SupersetClient
from ..core.task_manager.cleanup import TaskCleanupService
from ..core.task_manager import TaskManager
# [DEF:HealthService:Class]
# @COMPLEXITY: 4
# @PURPOSE: Aggregate latest dashboard validation state and manage persisted health report lifecycle.
# @PRE: Service is constructed with a live SQLAlchemy session and optional config manager.
# @POST: Exposes health summary aggregation and validation report deletion operations.
# @SIDE_EFFECT: Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators.
# @DATA_CONTRACT: Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.DashboardHealthItem]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.HealthSummaryResponse]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService]
class HealthService:
_dashboard_summary_cache: Dict[str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]] = {}
_dashboard_summary_cache: Dict[
str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]
] = {}
_dashboard_summary_cache_ttl_seconds = 60.0
"""
@PURPOSE: Service for managing and querying dashboard health data.
"""
# [DEF:HealthService_init:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution.
# @PRE: db is a valid SQLAlchemy session.
# @POST: Service is ready to aggregate summaries and delete health reports.
def __init__(self, db: Session, config_manager = None):
# @SIDE_EFFECT: Initializes per-instance dashboard metadata cache.
# @DATA_CONTRACT: Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService]
# @RELATION: [BINDS_TO] ->[backend.src.services.health_service.HealthService]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
def __init__(self, db: Session, config_manager=None):
self.db = db
self.config_manager = config_manager
self._dashboard_meta_cache: Dict[Tuple[str, str], Dict[str, Optional[str]]] = {}
# [/DEF:HealthService_init:Function]
# [DEF:_prime_dashboard_meta_cache:Function]
@@ -51,6 +65,10 @@ class HealthService:
# @PRE: records may contain mixed numeric and slug dashboard identifiers.
# @POST: Numeric dashboard ids for known environments are cached when discoverable.
# @SIDE_EFFECT: May call Superset dashboard list API once per referenced environment.
# @DATA_CONTRACT: Input[records: List[ValidationRecord]] -> Output[None]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[config_manager.get_environments]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dashboards_summary]
def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None:
if not self.config_manager or not records:
@@ -87,10 +105,13 @@ class HealthService:
continue
try:
cached_meta = self.__class__._dashboard_summary_cache.get(environment_id)
cached_meta = self.__class__._dashboard_summary_cache.get(
environment_id
)
cache_is_fresh = (
cached_meta is not None
and (time.monotonic() - cached_meta[0]) < self.__class__._dashboard_summary_cache_ttl_seconds
and (time.monotonic() - cached_meta[0])
< self.__class__._dashboard_summary_cache_ttl_seconds
)
if cache_is_fresh:
dashboard_meta_map = cached_meta[1]
@@ -109,9 +130,11 @@ class HealthService:
dashboard_meta_map,
)
for dashboard_id in dashboard_ids:
self._dashboard_meta_cache[(environment_id, dashboard_id)] = dashboard_meta_map.get(
dashboard_id,
{"slug": None, "title": None},
self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
dashboard_meta_map.get(
dashboard_id,
{"slug": None, "title": None},
)
)
except Exception as exc:
logger.warning(
@@ -124,6 +147,7 @@ class HealthService:
"slug": None,
"title": None,
}
# [/DEF:_prime_dashboard_meta_cache:Function]
# [DEF:_resolve_dashboard_meta:Function]
@@ -131,7 +155,10 @@ class HealthService:
# @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record.
# @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty.
# @POST: Returns dict with `slug` and `title` keys, using cache when possible.
def _resolve_dashboard_meta(self, dashboard_id: str, environment_id: Optional[str]) -> Dict[str, Optional[str]]:
# @SIDE_EFFECT: Writes default cache entries for unresolved numeric dashboard ids.
def _resolve_dashboard_meta(
self, dashboard_id: str, environment_id: Optional[str]
) -> Dict[str, Optional[str]]:
normalized_dashboard_id = str(dashboard_id or "").strip()
normalized_environment_id = str(environment_id or "").strip()
if not normalized_dashboard_id:
@@ -151,6 +178,7 @@ class HealthService:
meta = {"slug": None, "title": None}
self._dashboard_meta_cache[cache_key] = meta
return meta
# [/DEF:_resolve_dashboard_meta:Function]
# [DEF:get_health_summary:Function]
@@ -162,7 +190,9 @@ class HealthService:
# @DATA_CONTRACT: Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse]
# @RELATION: [CALLS] ->[self._prime_dashboard_meta_cache]
# @RELATION: [CALLS] ->[self._resolve_dashboard_meta]
async def get_health_summary(self, environment_id: str = None) -> HealthSummaryResponse:
async def get_health_summary(
self, environment_id: str = None
) -> HealthSummaryResponse:
"""
@PURPOSE: Aggregates the latest validation status for all dashboards.
@PRE: environment_id (optional) to filter by environment.
@@ -170,23 +200,25 @@ class HealthService:
"""
# [REASON] We need the latest ValidationRecord for each unique dashboard_id.
# We use a subquery to find the max timestamp per dashboard_id.
subquery = self.db.query(
ValidationRecord.dashboard_id,
func.max(ValidationRecord.timestamp).label("max_ts")
func.max(ValidationRecord.timestamp).label("max_ts"),
)
if environment_id:
subquery = subquery.filter(ValidationRecord.environment_id == environment_id)
subquery = subquery.filter(
ValidationRecord.environment_id == environment_id
)
subquery = subquery.group_by(ValidationRecord.dashboard_id).subquery()
query = self.db.query(ValidationRecord).join(
subquery,
(ValidationRecord.dashboard_id == subquery.c.dashboard_id) &
(ValidationRecord.timestamp == subquery.c.max_ts)
(ValidationRecord.dashboard_id == subquery.c.dashboard_id)
& (ValidationRecord.timestamp == subquery.c.max_ts),
)
records = query.all()
self._prime_dashboard_meta_cache(records)
items = []
@@ -208,27 +240,32 @@ class HealthService:
status = "UNKNOWN"
meta = self._resolve_dashboard_meta(rec.dashboard_id, rec.environment_id)
items.append(DashboardHealthItem(
record_id=rec.id,
dashboard_id=rec.dashboard_id,
dashboard_slug=meta.get("slug"),
dashboard_title=meta.get("title"),
environment_id=rec.environment_id or "unknown",
status=status,
last_check=rec.timestamp,
task_id=rec.task_id,
summary=rec.summary
))
items.append(
DashboardHealthItem(
record_id=rec.id,
dashboard_id=rec.dashboard_id,
dashboard_slug=meta.get("slug"),
dashboard_title=meta.get("title"),
environment_id=rec.environment_id or "unknown",
status=status,
last_check=rec.timestamp,
task_id=rec.task_id,
summary=rec.summary,
)
)
logger.info(
f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records."
)
logger.info(f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records.")
return HealthSummaryResponse(
items=items,
pass_count=pass_count,
warn_count=warn_count,
fail_count=fail_count,
unknown_count=unknown_count
unknown_count=unknown_count,
)
# [/DEF:get_health_summary:Function]
# [DEF:delete_validation_report:Function]
@@ -238,9 +275,19 @@ class HealthService:
# @POST: Returns True only when a matching record was deleted.
# @SIDE_EFFECT: Deletes DB rows, optional screenshot file, and optional task/log persistence.
# @DATA_CONTRACT: Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.TaskManager]
# @RELATION: [CALLS] ->[os.path.exists]
# @RELATION: [CALLS] ->[os.remove]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService.delete_task_with_logs]
def delete_validation_report(self, record_id: str, task_manager: Optional[TaskManager] = None) -> bool:
record = self.db.query(ValidationRecord).filter(ValidationRecord.id == record_id).first()
def delete_validation_report(
self, record_id: str, task_manager: Optional[TaskManager] = None
) -> bool:
record = (
self.db.query(ValidationRecord)
.filter(ValidationRecord.id == record_id)
.first()
)
if not record:
return False
@@ -250,7 +297,9 @@ class HealthService:
if record.environment_id is None:
peer_query = peer_query.filter(ValidationRecord.environment_id.is_(None))
else:
peer_query = peer_query.filter(ValidationRecord.environment_id == record.environment_id)
peer_query = peer_query.filter(
ValidationRecord.environment_id == record.environment_id
)
records_to_delete = peer_query.all()
screenshot_paths = [
@@ -306,8 +355,10 @@ class HealthService:
)
return True
# [/DEF:delete_validation_report:Function]
# [/DEF:HealthService:Class]
# [/DEF:health_service:Module]

44
backend/src/services/mapping_service.py
View File

@@ -2,9 +2,14 @@
#
# @SEMANTICS: service, mapping, fuzzy-matching, superset
# @PURPOSE: Orchestrates database fetching and fuzzy matching suggestions.
# @COMPLEXITY: 3
# @LAYER: Service
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
# @RELATION: DEPENDS_ON -> backend.src.core.utils.matching
# @PRE: source/target environment identifiers are provided by caller.
# @POST: Exposes stateless mapping suggestion orchestration over configured environments.
# @SIDE_EFFECT: Performs remote metadata reads through Superset API clients.
# @DATA_CONTRACT: Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]]
# @RELATION: DEPENDS_ON -> SupersetClient
# @RELATION: DEPENDS_ON -> suggest_mappings
#
# @INVARIANT: Suggestions are based on database names.
@@ -15,57 +20,80 @@ from ..core.superset_client import SupersetClient
from ..core.utils.matching import suggest_mappings
# [/SECTION]
# [DEF:MappingService:Class]
# @PURPOSE: Service for handling database mapping logic.
# @COMPLEXITY: 3
# @PRE: config_manager exposes get_environments() with environment objects containing id.
# @POST: Provides client resolution and mapping suggestion methods.
# @SIDE_EFFECT: Instantiates Superset clients and performs upstream metadata reads.
# @DATA_CONTRACT: Input[config_manager] -> Output[List[Dict]]
# @RELATION: DEPENDS_ON -> SupersetClient
# @RELATION: DEPENDS_ON -> suggest_mappings
class MappingService:
# [DEF:init:Function]
# @PURPOSE: Initializes the mapping service with a config manager.
# @COMPLEXITY: 3
# @PRE: config_manager is provided.
# @PARAM: config_manager (ConfigManager) - The configuration manager.
# @POST: Service is initialized.
# @RELATION: DEPENDS_ON -> MappingService
def __init__(self, config_manager):
with belief_scope("MappingService.__init__"):
self.config_manager = config_manager
# [/DEF:init:Function]
# [DEF:_get_client:Function]
# @PURPOSE: Helper to get an initialized SupersetClient for an environment.
# @COMPLEXITY: 3
# @PARAM: env_id (str) - The ID of the environment.
# @PRE: environment must exist in config.
# @POST: Returns an initialized SupersetClient.
# @RETURN: SupersetClient - Initialized client.
# @RELATION: CALLS -> SupersetClient
def _get_client(self, env_id: str) -> SupersetClient:
with belief_scope("MappingService._get_client", f"env_id={env_id}"):
envs = self.config_manager.get_environments()
env = next((e for e in envs if e.id == env_id), None)
if not env:
raise ValueError(f"Environment {env_id} not found")
return SupersetClient(env)
# [/DEF:_get_client:Function]
# [DEF:get_suggestions:Function]
# @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions.
# @COMPLEXITY: 3
# @PARAM: source_env_id (str) - Source environment ID.
# @PARAM: target_env_id (str) - Target environment ID.
# @PRE: Both environments must be accessible.
# @POST: Returns fuzzy-matched database suggestions.
# @RETURN: List[Dict] - Suggested mappings.
async def get_suggestions(self, source_env_id: str, target_env_id: str) -> List[Dict]:
with belief_scope("MappingService.get_suggestions", f"source={source_env_id}, target={target_env_id}"):
# @RELATION: CALLS -> _get_client
# @RELATION: CALLS -> suggest_mappings
async def get_suggestions(
self, source_env_id: str, target_env_id: str
) -> List[Dict]:
with belief_scope(
"MappingService.get_suggestions",
f"source={source_env_id}, target={target_env_id}",
):
"""
Get suggested mappings between two environments.
"""
source_client = self._get_client(source_env_id)
target_client = self._get_client(target_env_id)
source_dbs = source_client.get_databases_summary()
target_dbs = target_client.get_databases_summary()
return suggest_mappings(source_dbs, target_dbs)
# [/DEF:get_suggestions:Function]
# [/DEF:MappingService:Class]
# [/DEF:mapping_service:Module]

21
backend/src/services/notifications/providers.py
View File

@@ -3,8 +3,17 @@
# @COMPLEXITY: 5
# @SEMANTICS: notifications, providers, smtp, slack, telegram, abstraction
# @PURPOSE: Defines abstract base and concrete implementations for external notification delivery.
# @RELATION: IMPLEMENTS ->[NotificationService:Class]
# @RELATION: [DEPENDED_ON_BY] ->[NotificationService]
# @RELATION: [DEPENDS_ON] ->[NotificationProvider]
# @RELATION: [DEPENDS_ON] ->[SMTPProvider]
# @RELATION: [DEPENDS_ON] ->[TelegramProvider]
# @RELATION: [DEPENDS_ON] ->[SlackProvider]
# @LAYER: Infra
# @PRE: Provider configuration dictionaries are supplied by trusted configuration sources.
# @POST: Each provider exposes async send contract returning boolean delivery outcome.
# @SIDE_EFFECT: Performs outbound network I/O to SMTP or HTTP endpoints.
# @DATA_CONTRACT: Input[target, subject, body, context?] -> Output[bool]
# @INVARIANT: Concrete providers preserve boolean send contract and swallow transport exceptions into False.
#
# @INVARIANT: Providers must be stateless and resilient to network failures.
# @INVARIANT: Sensitive credentials must be handled via encrypted config.
@@ -20,7 +29,11 @@ from ...core.logger import logger
# [DEF:NotificationProvider:Class]
# @COMPLEXITY: 2
# @PURPOSE: Abstract base class for all notification providers.
# @RELATION: [DEPENDED_ON_BY] ->[SMTPProvider]
# @RELATION: [DEPENDED_ON_BY] ->[TelegramProvider]
# @RELATION: [DEPENDED_ON_BY] ->[SlackProvider]
class NotificationProvider(ABC):
@abstractmethod
async def send(
@@ -45,7 +58,9 @@ class NotificationProvider(ABC):
# [DEF:SMTPProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via SMTP.
# @RELATION: [INHERITS] ->[NotificationProvider]
class SMTPProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.host = config.get("host")
@@ -88,7 +103,9 @@ class SMTPProvider(NotificationProvider):
# [DEF:TelegramProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via Telegram Bot API.
# @RELATION: [INHERITS] ->[NotificationProvider]
class TelegramProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.bot_token = config.get("bot_token")
@@ -126,7 +143,9 @@ class TelegramProvider(NotificationProvider):
# [DEF:SlackProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via Slack Webhooks or API.
# @RELATION: [INHERITS] ->[NotificationProvider]
class SlackProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.webhook_url = config.get("webhook_url")

11
backend/src/services/profile_service.py
View File

@@ -86,9 +86,18 @@ class ProfileAuthorizationError(Exception):
# [DEF:ProfileService:Class]
# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_profile_lookup.SupersetAccountLookupAdapter]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.profile.UserDashboardPreference]
# @RELATION: [CALLS] ->[backend.src.services.rbac_permission_catalog.discover_declared_permissions]
# @COMPLEXITY: 5
# @PURPOSE: Implements profile preference read/update flow and Superset account lookup degradation strategy.
# @PRE: Caller provides authenticated User context for external service methods.
# @POST: Preference operations remain user-scoped and return normalized profile/lookup responses.
# @SIDE_EFFECT: Writes preference records and encrypted tokens; performs external account lookups when requested.
# @DATA_CONTRACT: Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool]
class ProfileService:
# [DEF:init:Function]
# @RELATION: BINDS_TO -> ProfileService

8
backend/src/services/reports/__tests__/test_report_normalizer.py
View File

@@ -14,6 +14,7 @@ from src.services.reports.normalizer import normalize_task_report
# [DEF:test_unknown_type_maps_to_unknown_profile:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure unknown plugin IDs map to unknown profile with populated summary and error context.
def test_unknown_type_maps_to_unknown_profile():
task = Task(
id="unknown-1",
@@ -34,8 +35,10 @@ def test_unknown_type_maps_to_unknown_profile():
# [/DEF:test_unknown_type_maps_to_unknown_profile:Function]
# [DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure missing result payload still yields visible report details with result placeholder.
def test_partial_payload_keeps_report_visible_with_placeholders():
task = Task(
id="partial-1",
@@ -56,8 +59,10 @@ def test_partial_payload_keeps_report_visible_with_placeholders():
# [/DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function]
# [DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure clean-release plugin ID maps to clean_release task profile and summary passthrough.
def test_clean_release_plugin_maps_to_clean_release_task_type():
task = Task(
id="clean-release-1",
@@ -75,4 +80,5 @@ def test_clean_release_plugin_maps_to_clean_release_task_type():
assert report.summary == "Clean release compliance passed"
# [/DEF:test_report_normalizer:Module]# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# [/DEF:test_report_normalizer:Module]

10
backend/src/services/reports/type_profiles.py
View File

@@ -3,7 +3,9 @@
# @SEMANTICS: reports, type_profiles, normalization, fallback
# @PURPOSE: Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[backend.src.models.report.TaskType:Function]
# @RELATION: [DEPENDS_ON] ->[TaskType]
# @RELATION: [CONTAINS] ->[resolve_task_type]
# @RELATION: [CONTAINS] ->[get_type_profile]
# @INVARIANT: Unknown input always resolves to TaskType.UNKNOWN with a single fallback profile.
# [SECTION: IMPORTS]
@@ -97,6 +99,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType:
if not normalized:
return TaskType.UNKNOWN
return PLUGIN_TO_TASK_TYPE.get(normalized, TaskType.UNKNOWN)
# [/DEF:resolve_task_type:Function]
@@ -118,6 +122,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType:
def get_type_profile(task_type: TaskType) -> Dict[str, Any]:
with belief_scope("get_type_profile"):
return TASK_TYPE_PROFILES.get(task_type, TASK_TYPE_PROFILES[TaskType.UNKNOWN])
# [/DEF:get_type_profile:Function]
# [/DEF:type_profiles:Module]
# [/DEF:type_profiles:Module]