busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: finalize semantic repair and test updates

Browse Source
This commit is contained in:
busya
2026-03-21 15:07:06 +03:00
parent c827c2f098
commit 2da548fd71
99 changed files with 2484 additions and 985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
backend/src/services/__tests__/test_llm_plugin_persistence.py
View File

@@ -1,5 +1,5 @@
# [DEF:test_llm_plugin_persistence:Module]
# @RELATION: VERIFIES ->[src.plugins.llm_analysis.plugin.DashboardValidationPlugin]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 3
# @PURPOSE: Regression test for ValidationRecord persistence fields populated from task context.
@@ -10,7 +10,7 @@ from src.plugins.llm_analysis import plugin as plugin_module
# [DEF:_DummyLogger:Class]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 1
# @PURPOSE: Minimal logger shim for TaskContext-like objects used in tests.
# @INVARIANT: Logging methods are no-ops and must not mutate test state.
@@ -35,7 +35,7 @@ class _DummyLogger:
# [DEF:_FakeDBSession:Class]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @COMPLEXITY: 2
# @PURPOSE: Captures persisted records for assertion and mimics SQLAlchemy session methods used by plugin.
# @INVARIANT: add/commit/close provide only persistence signals asserted by this test.
@@ -59,8 +59,11 @@ class _FakeDBSession:
# [DEF:test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @RELATION: BINDS_TO ->[test_llm_plugin_persistence]
# @RELATION: BINDS_TO -> [test_llm_plugin_persistence:Module]
# @RELATION: VERIFIES -> [DashboardValidationPlugin:Class]
# @COMPLEXITY: 2
# @PURPOSE: Ensure db ValidationRecord includes context.task_id and params.environment_id.
# @INVARIANT: Assertions remain restricted to persisted task/environment identity fields and session lifecycle signals.
@pytest.mark.asyncio
async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
tmp_path, monkeypatch
@@ -77,6 +80,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
is_active=True,
)
# [DEF:_FakeProviderService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: LLM provider service stub returning deterministic provider and decrypted API key for plugin tests.
# @INVARIANT: Returns same provider and key regardless of provider_id argument; no lookup logic.
class _FakeProviderService:
def __init__(self, _db):
return None
@@ -87,6 +95,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
def get_decrypted_api_key(self, _provider_id):
return "a" * 32
# [/DEF:_FakeProviderService:Class]
# [DEF:_FakeScreenshotService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Screenshot service stub that accepts capture_dashboard calls without side effects.
# @INVARIANT: capture_dashboard is intentionally permissive for this persistence-focused test and does not validate argument values.
class _FakeScreenshotService:
def __init__(self, _env):
return None
@@ -94,8 +109,10 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
async def capture_dashboard(self, _dashboard_id, _screenshot_path):
return None
# [/DEF:_FakeScreenshotService:Class]
# [DEF:_FakeLLMClient:Class]
# @RELATION: BINDS_TO ->[test_dashboard_validation_plugin_persists_task_and_environment_ids]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 2
# @PURPOSE: Deterministic LLM client double returning canonical analysis payload for persistence-path assertions.
# @INVARIANT: analyze_dashboard is side-effect free and returns schema-compatible PASS result.
@@ -112,6 +129,11 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
# [/DEF:_FakeLLMClient:Class]
# [DEF:_FakeNotificationService:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Notification service stub that accepts plugin dispatch_report payload without introducing side effects.
# @INVARIANT: dispatch_report accepts arbitrary keyword payloads because this test verifies persistence fields, not notification payload schema.
class _FakeNotificationService:
def __init__(self, *_args, **_kwargs):
return None
@@ -119,6 +141,13 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
async def dispatch_report(self, **_kwargs):
return None
# [/DEF:_FakeNotificationService:Class]
# [DEF:_FakeConfigManager:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Config manager stub providing storage root path and minimal settings for plugin execution path.
# @INVARIANT: Only storage.root_path and llm fields are safe to access; all other settings fields are absent.
class _FakeConfigManager:
def get_environment(self, _env_id):
return env
@@ -131,12 +160,21 @@ async def test_dashboard_validation_plugin_persists_task_and_environment_ids(
)
)
# [/DEF:_FakeConfigManager:Class]
# [DEF:_FakeSupersetClient:Class]
# @RELATION: BINDS_TO -> [test_dashboard_validation_plugin_persists_task_and_environment_ids:Function]
# @COMPLEXITY: 1
# @PURPOSE: Superset client stub exposing network.request as a lambda that returns empty result list.
# @INVARIANT: network.request intentionally accepts arbitrary keyword payloads because response shape, not request signature, is the persistence-path dependency.
class _FakeSupersetClient:
def __init__(self, _env):
self.network = types.SimpleNamespace(
request=lambda **_kwargs: {"result": []}
)
# [/DEF:_FakeSupersetClient:Class]
monkeypatch.setattr(plugin_module, "SessionLocal", lambda: fake_db)
monkeypatch.setattr(plugin_module, "LLMProviderService", _FakeProviderService)
monkeypatch.setattr(plugin_module, "ScreenshotService", _FakeScreenshotService)

30
backend/src/services/__tests__/test_llm_prompt_templates.py
View File

@@ -3,7 +3,7 @@
# @SEMANTICS: tests, llm, prompts, templates, settings
# @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates.
# @LAYER: Domain Tests
# @RELATION: DEPENDS_ON ->[backend.src.services.llm_prompt_templates:Function]
# @RELATION: DEPENDS_ON -> [backend.src.services.llm_prompt_templates:Function]
# @INVARIANT: All required prompt keys remain available after normalization.
from src.services.llm_prompt_templates import (
@@ -23,8 +23,6 @@ from src.services.llm_prompt_templates import (
# @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults.
# @PRE: Input llm settings do not contain complete prompts object.
# @POST: Returned structure includes required prompt templates with fallback defaults.
# [DEF:test_normalize_llm_settings_adds_default_prompts:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_normalize_llm_settings_adds_default_prompts():
normalized = normalize_llm_settings({"default_provider": "x"})
@@ -38,6 +36,8 @@ def test_normalize_llm_settings_adds_default_prompts():
assert key in normalized["provider_bindings"]
for key in DEFAULT_LLM_ASSISTANT_SETTINGS:
assert key in normalized
# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function]
@@ -47,17 +47,13 @@ def test_normalize_llm_settings_adds_default_prompts():
# @PURPOSE: Ensure user-customized prompt values are preserved during normalization.
# @PRE: Input llm settings contain custom prompt override.
# @POST: Custom prompt value remains unchanged in normalized output.
# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function]
# [DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_normalize_llm_settings_keeps_custom_prompt_values():
custom = "Doc for {dataset_name} using {columns_json}"
normalized = normalize_llm_settings(
{"prompts": {"documentation_prompt": custom}}
)
normalized = normalize_llm_settings({"prompts": {"documentation_prompt": custom}})
assert normalized["prompts"]["documentation_prompt"] == custom
# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
@@ -67,10 +63,6 @@ def test_normalize_llm_settings_keeps_custom_prompt_values():
# @PURPOSE: Ensure template placeholders are deterministically replaced.
# @PRE: Template contains placeholders matching provided variables.
# @POST: Rendered prompt string contains substituted values.
# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function]
# [DEF:test_render_prompt_replaces_known_placeholders:Function]
# @RELATION: BINDS_TO -> test_llm_prompt_templates
def test_render_prompt_replaces_known_placeholders():
rendered = render_prompt(
"Hello {name}, diff={diff}",
@@ -78,6 +70,8 @@ def test_render_prompt_replaces_known_placeholders():
)
assert rendered == "Hello bot, diff=A->B"
# [/DEF:test_render_prompt_replaces_known_placeholders:Function]
@@ -85,13 +79,13 @@ def test_render_prompt_replaces_known_placeholders():
# @RELATION: BINDS_TO -> test_llm_prompt_templates
# @COMPLEXITY: 2
# @PURPOSE: Ensure multimodal model detection recognizes common vision-capable model names.
# [/DEF:test_render_prompt_replaces_known_placeholders:Function]
def test_is_multimodal_model_detects_known_vision_models():
assert is_multimodal_model("gpt-4o") is True
assert is_multimodal_model("claude-3-5-sonnet") is True
assert is_multimodal_model("stepfun/step-3.5-flash:free", "openrouter") is False
assert is_multimodal_model("text-only-model") is False
# [/DEF:test_is_multimodal_model_detects_known_vision_models:Function]
@@ -106,6 +100,8 @@ def test_resolve_bound_provider_id_prefers_binding_then_default():
}
assert resolve_bound_provider_id(settings, "dashboard_validation") == "vision-1"
assert resolve_bound_provider_id(settings, "documentation") == "default-1"
# [/DEF:test_resolve_bound_provider_id_prefers_binding_then_default:Function]
@@ -122,6 +118,8 @@ def test_normalize_llm_settings_keeps_assistant_planner_settings():
)
assert normalized["assistant_planner_provider"] == "provider-a"
assert normalized["assistant_planner_model"] == "gpt-4.1-mini"
# [/DEF:test_normalize_llm_settings_keeps_assistant_planner_settings:Function]

76
backend/src/services/__tests__/test_llm_provider.py
View File

@@ -1,7 +1,9 @@
# [DEF:__tests__/test_llm_provider:Module]
# @RELATION: VERIFIES -> ../llm_provider.py
# [DEF:test_llm_provider:Module]
# @RELATION: VERIFIES -> [src.services.llm_provider:Module]
# @COMPLEXITY: 3
# @SEMANTICS: tests, llm-provider, encryption, contract
# @PURPOSE: Contract testing for LLMProviderService and EncryptionManager
# [/DEF:__tests__/test_llm_provider:Module]
# [/DEF:test_llm_provider:Module]
import pytest
import os
@@ -14,14 +16,16 @@ from src.plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType
# [DEF:_test_encryption_key_fixture:Global]
# @PURPOSE: Ensure encryption-dependent provider tests run with a valid Fernet key.
# @RELATION: DEPENDS_ON ->[pytest:Module]
# @RELATION: DEPENDS_ON -> [pytest:Module]
os.environ.setdefault("ENCRYPTION_KEY", Fernet.generate_key().decode())
# [/DEF:_test_encryption_key_fixture:Global]
# @TEST_CONTRACT: EncryptionManagerModel -> Invariants
# @TEST_INVARIANT: symmetric_encryption
# [DEF:test_encryption_cycle:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify EncryptionManager round-trip encryption/decryption invariant for non-empty secrets.
def test_encryption_cycle():
"""Verify encrypted data can be decrypted back to original string."""
manager = EncryptionManager()
@@ -30,49 +34,78 @@ def test_encryption_cycle():
assert encrypted != original
assert manager.decrypt(encrypted) == original
# @TEST_EDGE: empty_string_encryption
# [/DEF:test_encryption_cycle:Function]
# [DEF:test_empty_string_encryption:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify EncryptionManager preserves empty-string payloads through encrypt/decrypt cycle.
def test_empty_string_encryption():
manager = EncryptionManager()
original = ""
encrypted = manager.encrypt(original)
assert manager.decrypt(encrypted) == ""
# @TEST_EDGE: decrypt_invalid_data
# [/DEF:test_empty_string_encryption:Function]
# [DEF:test_decrypt_invalid_data:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure decrypt rejects invalid ciphertext input by raising an exception.
def test_decrypt_invalid_data():
manager = EncryptionManager()
with pytest.raises(Exception):
manager.decrypt("not-encrypted-string")
# @TEST_FIXTURE: mock_db_session
# [/DEF:test_decrypt_invalid_data:Function]
# [DEF:mock_db:Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: MagicMock(spec=Session) fixture providing a constrained DB session double for LLMProviderService tests.
# @INVARIANT: Chained calls beyond Session spec create unconstrained intermediate mocks; only top-level query/add/commit are spec-enforced.
@pytest.fixture
def mock_db():
return MagicMock(spec=Session)
# [/DEF:mock_db:Fixture]
# [DEF:service:Fixture]
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @COMPLEXITY: 1
# @PURPOSE: LLMProviderService fixture wired to mock_db for provider CRUD tests.
@pytest.fixture
def service(mock_db):
return LLMProviderService(db=mock_db)
# [/DEF:service:Fixture]
# [DEF:test_get_all_providers:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify provider list retrieval issues query/all calls on the backing DB session.
def test_get_all_providers(service, mock_db):
service.get_all_providers()
mock_db.query.assert_called()
mock_db.query().all.assert_called()
# [/DEF:test_get_all_providers:Function]
# [DEF:test_create_provider:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure provider creation persists entity and stores API key in encrypted form.
def test_create_provider(service, mock_db):
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
@@ -80,11 +113,11 @@ def test_create_provider(service, mock_db):
base_url="https://api.openai.com",
api_key="sk-test",
default_model="gpt-4",
is_active=True
is_active=True,
)
provider = service.create_provider(config)
mock_db.add.assert_called()
mock_db.commit.assert_called()
# Verify API key was encrypted
@@ -92,31 +125,40 @@ def test_create_provider(service, mock_db):
# Decrypt to verify it matches
assert EncryptionManager().decrypt(provider.api_key) == "sk-test"
# [/DEF:test_create_provider:Function]
# [DEF:test_get_decrypted_api_key:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify service decrypts stored provider API key for an existing provider record.
def test_get_decrypted_api_key(service, mock_db):
# Setup mock provider
encrypted_key = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(id="p1", api_key=encrypted_key)
mock_db.query().filter().first.return_value = mock_provider
key = service.get_decrypted_api_key("p1")
assert key == "secret-value"
# [/DEF:test_get_decrypted_api_key:Function]
# [DEF:test_get_decrypted_api_key_not_found:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Verify missing provider lookup returns None instead of attempting decryption.
def test_get_decrypted_api_key_not_found(service, mock_db):
mock_db.query().filter().first.return_value = None
assert service.get_decrypted_api_key("missing") is None
# [/DEF:test_get_decrypted_api_key_not_found:Function]
# [DEF:test_update_provider_ignores_masked_placeholder_api_key:Function]
# @RELATION: BINDS_TO -> __tests__/test_llm_provider
# @RELATION: BINDS_TO -> [test_llm_provider:Module]
# @PURPOSE: Ensure masked placeholder API keys do not overwrite previously encrypted provider secrets.
def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db):
existing_encrypted = EncryptionManager().encrypt("secret-value")
mock_provider = LLMProvider(
@@ -145,4 +187,6 @@ def test_update_provider_ignores_masked_placeholder_api_key(service, mock_db):
assert updated.api_key == existing_encrypted
assert EncryptionManager().decrypt(updated.api_key) == "secret-value"
assert updated.is_active is False
# [/DEF:test_update_provider_ignores_masked_placeholder_api_key:Function]

48
backend/src/services/auth_service.py
View File

@@ -3,11 +3,11 @@
# @SEMANTICS: auth, service, business-logic, login, jwt, adfs, jit-provisioning
# @PURPOSE: Orchestrates credential authentication and ADFS JIT user provisioning.
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.security.verify_password]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.jwt.create_access_token]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.User]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.auth.Role]
# @RELATION: DEPENDS_ON -> [AuthRepository]
# @RELATION: DEPENDS_ON -> [verify_password]
# @RELATION: DEPENDS_ON -> [create_access_token]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
# @INVARIANT: Authentication succeeds only for active users with valid credentials; issued sessions encode subject and scopes from assigned roles.
# @PRE: Core auth models and security utilities available.
# @POST: User identity verified and session tokens issued according to role scopes.
@@ -41,9 +41,10 @@ class AuthService:
def __init__(self, db: Session):
self.db = db
self.repo = AuthRepository(db)
# [/DEF:AuthService_init:Function]
# [DEF:authenticate_user:Function]
# [DEF:AuthService.authenticate_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Validates credentials and account state for local username/password authentication.
# @PRE: username and password are non-empty credential inputs.
@@ -58,19 +59,20 @@ class AuthService:
user = self.repo.get_user_by_username(username)
if not user or not user.is_active:
return None
if not verify_password(password, user.password_hash):
return None
# Update last login
user.last_login = datetime.utcnow()
self.db.commit()
self.db.refresh(user)
return user
# [/DEF:authenticate_user:Function]
# [DEF:create_session:Function]
return user
# [/DEF:AuthService.authenticate_user:Function]
# [DEF:AuthService.create_session:Function]
# @COMPLEXITY: 3
# @PURPOSE: Issues an access token payload for an already authenticated user.
# @PRE: user is a valid User entity containing username and iterable roles with role.name values.
@@ -86,9 +88,10 @@ class AuthService:
data={"sub": user.username, "scopes": roles}
)
return {"access_token": access_token, "token_type": "bearer"}
# [/DEF:create_session:Function]
# [DEF:provision_adfs_user:Function]
# [/DEF:AuthService.create_session:Function]
# [DEF:AuthService.provision_adfs_user:Function]
# @COMPLEXITY: 3
# @PURPOSE: Performs ADFS Just-In-Time provisioning and role synchronization from AD group mappings.
# @PRE: user_info contains identity claims where at least one of 'upn' or 'email' is present; 'groups' may be absent.
@@ -102,7 +105,7 @@ class AuthService:
username = user_info.get("upn") or user_info.get("email")
email = user_info.get("email")
groups = user_info.get("groups", [])
user = self.repo.get_user_by_username(username)
if not user:
user = User(
@@ -111,21 +114,24 @@ class AuthService:
full_name=user_info.get("name"),
auth_source="ADFS",
is_active=True,
is_ad_user=True
is_ad_user=True,
)
self.db.add(user)
log_security_event("USER_PROVISIONED", username, {"source": "ADFS"})
# Sync roles from AD groups
mapped_roles = self.repo.get_roles_by_ad_groups(groups)
user.roles = mapped_roles
user.last_login = datetime.utcnow()
self.db.commit()
self.db.refresh(user)
return user
# [/DEF:provision_adfs_user:Function]
# [/DEF:AuthService.provision_adfs_user:Function]
# [/DEF:AuthService:Class]
# [/DEF:auth_service:Module]
# [/DEF:auth_service:Module]

17
backend/src/services/dataset_review/orchestrator.py
View File

@@ -5,10 +5,10 @@
# @LAYER: Domain
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @RELATION: [DEPENDS_ON] ->[ClarificationEngine]
# @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor]
# @RELATION: [DEPENDS_ON] ->[SupersetCompilationAdapter]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [CONTAINS] ->[DatasetReviewOrchestrator]
# @PRE: session mutations must execute inside a persisted session boundary scoped to one authenticated user.
# @POST: state transitions are persisted atomically and emit observable progress for long-running steps.
# @SIDE_EFFECT: creates task records, updates session aggregates, triggers upstream Superset calls, persists audit artifacts.
@@ -158,8 +158,8 @@ class LaunchDatasetResult:
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[SupersetContextExtractor]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @PRE: constructor dependencies are valid and tied to the current request/task scope.
# @POST: orchestrator instance can execute session-scoped mutations for one authenticated user.
# @SIDE_EFFECT: downstream operations may persist session/profile/finding state and enqueue background tasks.
@@ -169,8 +169,13 @@ class DatasetReviewOrchestrator:
# [DEF:DatasetReviewOrchestrator_init:Function]
# @COMPLEXITY: 3
# @PURPOSE: Bind repository, config, and task dependencies required by the orchestration boundary.
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [DEPENDS_ON] ->[ConfigManager]
# @RELATION: [DEPENDS_ON] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[SemanticSourceResolver]
# @PRE: repository/config_manager are valid collaborators for the current request scope.
# @POST: Instance holds collaborator references used by start/preview/launch orchestration methods.
# @SIDE_EFFECT: Stores dependency references for later session lifecycle operations.
def __init__(
self,
repository: DatasetReviewSessionRepository,
@@ -188,9 +193,9 @@ class DatasetReviewOrchestrator:
# [DEF:start_session:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize a new session from a Superset link or dataset selection and trigger context recovery.
# @RELATION: [DEPENDS_ON] ->[SessionRepo]
# @RELATION: [DEPENDS_ON] ->[DatasetReviewSessionRepository]
# @RELATION: [CALLS] ->[SupersetContextExtractor.parse_superset_link]
# @RELATION: [CALLS] ->[create_task]
# @RELATION: [CALLS] ->[TaskManager.create_task]
# @PRE: source input is non-empty and environment is accessible.
# @POST: session exists in persisted storage with intake/recovery state and task linkage when async work is required.
# @SIDE_EFFECT: persists session and may enqueue recovery task.
@@ -1112,7 +1117,7 @@ class DatasetReviewOrchestrator:
# [DEF:_enqueue_recovery_task:Function]
# @COMPLEXITY: 4
# @PURPOSE: Link session start to observable async recovery when task infrastructure is available.
# @RELATION: [CALLS] ->[create_task]
# @RELATION: [CALLS] ->[TaskManager.create_task]
# @PRE: session is already persisted.
# @POST: returns task identifier when a task could be enqueued, otherwise None.
# @SIDE_EFFECT: may create one background task for progressive recovery.

169
backend/src/services/dataset_review/repositories/__tests__/test_session_repository.py
View File

@@ -16,34 +16,46 @@ from src.models.dataset_review import (
ReadinessState,
RecommendedAction,
SessionCollaborator,
SessionCollaboratorRole
SessionCollaboratorRole,
)
from src.services.dataset_review.repositories.session_repository import (
DatasetReviewSessionRepository,
)
from src.services.dataset_review.repositories.session_repository import DatasetReviewSessionRepository
# [DEF:SessionRepositoryTests:Module]
# @RELATION: BELONGS_TO -> SrcRoot
# @COMPLEXITY: 2
# @PURPOSE: Unit tests for DatasetReviewSessionRepository.
@pytest.fixture
def db_session():
# [DEF:db_session:Function]
# @COMPLEXITY: 2
# @PURPOSE: Build isolated in-memory SQLAlchemy session seeded with baseline user/environment rows.
# @RELATION: BINDS_TO -> [SessionRepositoryTests]
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
# Create test data
user = User(id="user1", username="testuser", email="test@example.com", password_hash="pw")
env = Environment(id="env1", name="Prod", url="http://superset", credentials_id="cred1")
user = User(
id="user1", username="testuser", email="test@example.com", password_hash="pw"
)
env = Environment(
id="env1", name="Prod", url="http://superset", credentials_id="cred1"
)
session.add_all([user, env])
session.commit()
yield session
session.close()
# [/DEF:db_session:Function]
# [DEF:test_create_session:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_create_session(db_session):
@@ -54,126 +66,156 @@ def test_create_session(db_session):
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1"
dataset_ref="dataset1",
)
repo.create_session(session)
assert session.session_id is not None
loaded = db_session.query(DatasetReviewSession).filter_by(session_id=session.session_id).first()
loaded = (
db_session.query(DatasetReviewSession)
.filter_by(session_id=session.session_id)
.first()
)
assert loaded.user_id == "user1"
# [/DEF:test_create_session:Function]
# [DEF:test_load_session_detail_ownership:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_load_session_detail_ownership(db_session):
# @PURPOSE: Verify ownership enforcement in detail loading.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
# Correct user
loaded = repo.load_session_detail(session.session_id, "user1")
assert loaded is not None
# Wrong user
loaded_wrong = repo.load_session_detail(session.session_id, "wrong_user")
assert loaded_wrong is None
# [/DEF:test_load_session_detail_ownership:Function]
# [DEF:test_load_session_detail_collaborator:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_load_session_detail_collaborator(db_session):
# @PURPOSE: Verify collaborator access in detail loading.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
# Add collaborator
collab_user = User(id="collab1", username="collab", email="c@e.com", password_hash="p")
collab_user = User(
id="collab1", username="collab", email="c@e.com", password_hash="p"
)
db_session.add(collab_user)
collaborator = SessionCollaborator(
session_id=session.session_id,
user_id="collab1",
role=SessionCollaboratorRole.REVIEWER
role=SessionCollaboratorRole.REVIEWER,
)
db_session.add(collaborator)
db_session.commit()
# Collaborator access
loaded = repo.load_session_detail(session.session_id, "collab1")
assert loaded is not None
assert loaded.session_id == session.session_id
# [/DEF:test_load_session_detail_collaborator:Function]
# [DEF:test_save_preview_marks_stale:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_preview_marks_stale(db_session):
# @PURPOSE: Verify that saving a new preview marks old ones as stale.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
p1 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f1")
p1 = CompiledPreview(
session_id=session.session_id, preview_status="ready", preview_fingerprint="f1"
)
repo.save_preview(session.session_id, "user1", p1)
p2 = CompiledPreview(session_id=session.session_id, preview_status="ready", preview_fingerprint="f2")
p2 = CompiledPreview(
session_id=session.session_id, preview_status="ready", preview_fingerprint="f2"
)
repo.save_preview(session.session_id, "user1", p2)
db_session.refresh(p1)
assert p1.preview_status == "stale"
assert p2.preview_status == "ready"
assert session.last_preview_id == p2.preview_id
# [/DEF:test_save_preview_marks_stale:Function]
# [DEF:test_save_profile_and_findings:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_profile_and_findings(db_session):
# @PURPOSE: Verify persistence of profile and findings.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
profile = DatasetProfile(
session_id=session.session_id,
dataset_name="Test DS",
business_summary="Summary",
business_summary_source=BusinessSummarySource.INFERRED,
confidence_state=ConfidenceState.UNRESOLVED
confidence_state=ConfidenceState.UNRESOLVED,
)
finding = ValidationFinding(
session_id=session.session_id,
area=FindingArea.SOURCE_INTAKE,
severity=FindingSeverity.BLOCKING,
code="ERR1",
title="Error",
message="Failure"
message="Failure",
)
repo.save_profile_and_findings(session.session_id, "user1", profile, [finding])
updated_session = repo.load_session_detail(session.session_id, "user1")
assert updated_session.profile.dataset_name == "Test DS"
assert len(updated_session.findings) == 1
assert updated_session.findings[0].code == "ERR1"
# Verify removal of old findings
new_finding = ValidationFinding(
session_id=session.session_id,
@@ -181,29 +223,34 @@ def test_save_profile_and_findings(db_session):
severity=FindingSeverity.WARNING,
code="WARN1",
title="Warning",
message="Something"
message="Something",
)
repo.save_profile_and_findings(session.session_id, "user1", profile, [new_finding])
db_session.expire_all()
final_session = repo.load_session_detail(session.session_id, "user1")
assert len(final_session.findings) == 1
assert final_session.findings[0].code == "WARN1"
# [/DEF:test_save_profile_and_findings:Function]
# [DEF:test_save_run_context:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_save_run_context(db_session):
# @PURPOSE: Verify saving of run context.
repo = DatasetReviewSessionRepository(db_session)
session = DatasetReviewSession(
user_id="user1", environment_id="env1", source_kind="superset_link",
source_input="http://link", dataset_ref="dataset1"
user_id="user1",
environment_id="env1",
source_kind="superset_link",
source_input="http://link",
dataset_ref="dataset1",
)
repo.create_session(session)
rc = DatasetRunContext(
session_id=session.session_id,
dataset_ref="ds1",
@@ -215,28 +262,50 @@ def test_save_run_context(db_session):
approved_mapping_ids=[],
semantic_decision_refs=[],
open_warning_refs=[],
launch_status="success"
launch_status="success",
)
repo.save_run_context(session.session_id, "user1", rc)
assert session.last_run_context_id == rc.run_context_id
# [/DEF:test_save_run_context:Function]
# [DEF:test_list_sessions_for_user:Function]
# @RELATION: BINDS_TO -> SessionRepositoryTests
def test_list_sessions_for_user(db_session):
# @PURPOSE: Verify listing of sessions by user.
repo = DatasetReviewSessionRepository(db_session)
s1 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r1")
s2 = DatasetReviewSession(user_id="user1", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r2")
s3 = DatasetReviewSession(user_id="other", environment_id="env1", source_kind="k", source_input="i", dataset_ref="r3")
s1 = DatasetReviewSession(
user_id="user1",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r1",
)
s2 = DatasetReviewSession(
user_id="user1",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r2",
)
s3 = DatasetReviewSession(
user_id="other",
environment_id="env1",
source_kind="k",
source_input="i",
dataset_ref="r3",
)
db_session.add_all([s1, s2, s3])
db_session.commit()
sessions = repo.list_sessions_for_user("user1")
assert len(sessions) == 2
assert all(s.user_id == "user1" for s in sessions)
# [/DEF:SessionRepositoryTests:Module]# [/DEF:test_list_sessions_for_user:Function]
# [/DEF:test_list_sessions_for_user:Function]
# [/DEF:SessionRepositoryTests:Module]

5
backend/src/services/dataset_review/repositories/session_repository.py
View File

@@ -34,13 +34,14 @@ from src.core.logger import belief_scope, logger
from src.services.dataset_review.event_logger import SessionEventLogger
# [DEF:SessionRepo:Class]
# [DEF:DatasetReviewSessionRepository:Class]
# @COMPLEXITY: 4
# @PURPOSE: Enforce ownership-scoped persistence and retrieval for dataset review session aggregates.
# @RELATION: [DEPENDS_ON] -> [DatasetReviewSession]
# @RELATION: [DEPENDS_ON] -> [DatasetProfile]
# @RELATION: [DEPENDS_ON] -> [ValidationFinding]
# @RELATION: [DEPENDS_ON] -> [CompiledPreview]
# @RELATION: [DEPENDS_ON] -> [SessionEventLogger]
# @PRE: constructor receives a live SQLAlchemy session and callers provide authenticated user scope for guarded reads and writes.
# @POST: repository methods return ownership-scoped aggregates or persisted child records without changing domain meaning.
# @SIDE_EFFECT: mutates and queries the persistence layer through the injected database session.
@@ -409,6 +410,6 @@ class DatasetReviewSessionRepository:
# [/DEF:list_user_sess:Function]
# [/DEF:SessionRepo:Class]
# [/DEF:DatasetReviewSessionRepository:Class]
# [/DEF:DatasetReviewSessionRepository:Module]

119
backend/src/services/health_service.py
View File

@@ -19,30 +19,44 @@ from ..core.superset_client import SupersetClient
from ..core.task_manager.cleanup import TaskCleanupService
from ..core.task_manager import TaskManager
# [DEF:HealthService:Class]
# @COMPLEXITY: 4
# @PURPOSE: Aggregate latest dashboard validation state and manage persisted health report lifecycle.
# @PRE: Service is constructed with a live SQLAlchemy session and optional config manager.
# @POST: Exposes health summary aggregation and validation report deletion operations.
# @SIDE_EFFECT: Maintains in-memory dashboard metadata caches and may coordinate cleanup through collaborators.
# @DATA_CONTRACT: Input[Session, Optional[Any]] -> Output[HealthSummaryResponse|bool]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.DashboardHealthItem]
# @RELATION: [DEPENDS_ON] ->[backend.src.schemas.health.HealthSummaryResponse]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService]
class HealthService:
_dashboard_summary_cache: Dict[str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]] = {}
_dashboard_summary_cache: Dict[
str, Tuple[float, Dict[str, Dict[str, Optional[str]]]]
] = {}
_dashboard_summary_cache_ttl_seconds = 60.0
"""
@PURPOSE: Service for managing and querying dashboard health data.
"""
# [DEF:HealthService_init:Function]
# @COMPLEXITY: 3
# @PURPOSE: Initialize health service with DB session and optional config access for dashboard metadata resolution.
# @PRE: db is a valid SQLAlchemy session.
# @POST: Service is ready to aggregate summaries and delete health reports.
def __init__(self, db: Session, config_manager = None):
# @SIDE_EFFECT: Initializes per-instance dashboard metadata cache.
# @DATA_CONTRACT: Input[db: Session, config_manager: Optional[Any]] -> Output[HealthService]
# @RELATION: [BINDS_TO] ->[backend.src.services.health_service.HealthService]
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
def __init__(self, db: Session, config_manager=None):
self.db = db
self.config_manager = config_manager
self._dashboard_meta_cache: Dict[Tuple[str, str], Dict[str, Optional[str]]] = {}
# [/DEF:HealthService_init:Function]
# [DEF:_prime_dashboard_meta_cache:Function]
@@ -51,6 +65,10 @@ class HealthService:
# @PRE: records may contain mixed numeric and slug dashboard identifiers.
# @POST: Numeric dashboard ids for known environments are cached when discoverable.
# @SIDE_EFFECT: May call Superset dashboard list API once per referenced environment.
# @DATA_CONTRACT: Input[records: List[ValidationRecord]] -> Output[None]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [CALLS] ->[config_manager.get_environments]
# @RELATION: [CALLS] ->[backend.src.core.superset_client.SupersetClient.get_dashboards_summary]
def _prime_dashboard_meta_cache(self, records: List[ValidationRecord]) -> None:
if not self.config_manager or not records:
@@ -87,10 +105,13 @@ class HealthService:
continue
try:
cached_meta = self.__class__._dashboard_summary_cache.get(environment_id)
cached_meta = self.__class__._dashboard_summary_cache.get(
environment_id
)
cache_is_fresh = (
cached_meta is not None
and (time.monotonic() - cached_meta[0]) < self.__class__._dashboard_summary_cache_ttl_seconds
and (time.monotonic() - cached_meta[0])
< self.__class__._dashboard_summary_cache_ttl_seconds
)
if cache_is_fresh:
dashboard_meta_map = cached_meta[1]
@@ -109,9 +130,11 @@ class HealthService:
dashboard_meta_map,
)
for dashboard_id in dashboard_ids:
self._dashboard_meta_cache[(environment_id, dashboard_id)] = dashboard_meta_map.get(
dashboard_id,
{"slug": None, "title": None},
self._dashboard_meta_cache[(environment_id, dashboard_id)] = (
dashboard_meta_map.get(
dashboard_id,
{"slug": None, "title": None},
)
)
except Exception as exc:
logger.warning(
@@ -124,6 +147,7 @@ class HealthService:
"slug": None,
"title": None,
}
# [/DEF:_prime_dashboard_meta_cache:Function]
# [DEF:_resolve_dashboard_meta:Function]
@@ -131,7 +155,10 @@ class HealthService:
# @PURPOSE: Resolve slug/title for a dashboard referenced by persisted validation record.
# @PRE: dashboard_id may be numeric or slug-like; environment_id may be empty.
# @POST: Returns dict with `slug` and `title` keys, using cache when possible.
def _resolve_dashboard_meta(self, dashboard_id: str, environment_id: Optional[str]) -> Dict[str, Optional[str]]:
# @SIDE_EFFECT: Writes default cache entries for unresolved numeric dashboard ids.
def _resolve_dashboard_meta(
self, dashboard_id: str, environment_id: Optional[str]
) -> Dict[str, Optional[str]]:
normalized_dashboard_id = str(dashboard_id or "").strip()
normalized_environment_id = str(environment_id or "").strip()
if not normalized_dashboard_id:
@@ -151,6 +178,7 @@ class HealthService:
meta = {"slug": None, "title": None}
self._dashboard_meta_cache[cache_key] = meta
return meta
# [/DEF:_resolve_dashboard_meta:Function]
# [DEF:get_health_summary:Function]
@@ -162,7 +190,9 @@ class HealthService:
# @DATA_CONTRACT: Input[environment_id: Optional[str]] -> Output[HealthSummaryResponse]
# @RELATION: [CALLS] ->[self._prime_dashboard_meta_cache]
# @RELATION: [CALLS] ->[self._resolve_dashboard_meta]
async def get_health_summary(self, environment_id: str = None) -> HealthSummaryResponse:
async def get_health_summary(
self, environment_id: str = None
) -> HealthSummaryResponse:
"""
@PURPOSE: Aggregates the latest validation status for all dashboards.
@PRE: environment_id (optional) to filter by environment.
@@ -170,23 +200,25 @@ class HealthService:
"""
# [REASON] We need the latest ValidationRecord for each unique dashboard_id.
# We use a subquery to find the max timestamp per dashboard_id.
subquery = self.db.query(
ValidationRecord.dashboard_id,
func.max(ValidationRecord.timestamp).label("max_ts")
func.max(ValidationRecord.timestamp).label("max_ts"),
)
if environment_id:
subquery = subquery.filter(ValidationRecord.environment_id == environment_id)
subquery = subquery.filter(
ValidationRecord.environment_id == environment_id
)
subquery = subquery.group_by(ValidationRecord.dashboard_id).subquery()
query = self.db.query(ValidationRecord).join(
subquery,
(ValidationRecord.dashboard_id == subquery.c.dashboard_id) &
(ValidationRecord.timestamp == subquery.c.max_ts)
(ValidationRecord.dashboard_id == subquery.c.dashboard_id)
& (ValidationRecord.timestamp == subquery.c.max_ts),
)
records = query.all()
self._prime_dashboard_meta_cache(records)
items = []
@@ -208,27 +240,32 @@ class HealthService:
status = "UNKNOWN"
meta = self._resolve_dashboard_meta(rec.dashboard_id, rec.environment_id)
items.append(DashboardHealthItem(
record_id=rec.id,
dashboard_id=rec.dashboard_id,
dashboard_slug=meta.get("slug"),
dashboard_title=meta.get("title"),
environment_id=rec.environment_id or "unknown",
status=status,
last_check=rec.timestamp,
task_id=rec.task_id,
summary=rec.summary
))
items.append(
DashboardHealthItem(
record_id=rec.id,
dashboard_id=rec.dashboard_id,
dashboard_slug=meta.get("slug"),
dashboard_title=meta.get("title"),
environment_id=rec.environment_id or "unknown",
status=status,
last_check=rec.timestamp,
task_id=rec.task_id,
summary=rec.summary,
)
)
logger.info(
f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records."
)
logger.info(f"[HealthService][get_health_summary] Aggregated {len(items)} dashboard health records.")
return HealthSummaryResponse(
items=items,
pass_count=pass_count,
warn_count=warn_count,
fail_count=fail_count,
unknown_count=unknown_count
unknown_count=unknown_count,
)
# [/DEF:get_health_summary:Function]
# [DEF:delete_validation_report:Function]
@@ -238,9 +275,19 @@ class HealthService:
# @POST: Returns True only when a matching record was deleted.
# @SIDE_EFFECT: Deletes DB rows, optional screenshot file, and optional task/log persistence.
# @DATA_CONTRACT: Input[record_id: str, task_manager: Optional[TaskManager]] -> Output[bool]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.llm.ValidationRecord]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.task_manager.TaskManager]
# @RELATION: [CALLS] ->[os.path.exists]
# @RELATION: [CALLS] ->[os.remove]
# @RELATION: [CALLS] ->[backend.src.core.task_manager.cleanup.TaskCleanupService.delete_task_with_logs]
def delete_validation_report(self, record_id: str, task_manager: Optional[TaskManager] = None) -> bool:
record = self.db.query(ValidationRecord).filter(ValidationRecord.id == record_id).first()
def delete_validation_report(
self, record_id: str, task_manager: Optional[TaskManager] = None
) -> bool:
record = (
self.db.query(ValidationRecord)
.filter(ValidationRecord.id == record_id)
.first()
)
if not record:
return False
@@ -250,7 +297,9 @@ class HealthService:
if record.environment_id is None:
peer_query = peer_query.filter(ValidationRecord.environment_id.is_(None))
else:
peer_query = peer_query.filter(ValidationRecord.environment_id == record.environment_id)
peer_query = peer_query.filter(
ValidationRecord.environment_id == record.environment_id
)
records_to_delete = peer_query.all()
screenshot_paths = [
@@ -306,8 +355,10 @@ class HealthService:
)
return True
# [/DEF:delete_validation_report:Function]
# [/DEF:HealthService:Class]
# [/DEF:health_service:Module]

44
backend/src/services/mapping_service.py
View File

@@ -2,9 +2,14 @@
#
# @SEMANTICS: service, mapping, fuzzy-matching, superset
# @PURPOSE: Orchestrates database fetching and fuzzy matching suggestions.
# @COMPLEXITY: 3
# @LAYER: Service
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
# @RELATION: DEPENDS_ON -> backend.src.core.utils.matching
# @PRE: source/target environment identifiers are provided by caller.
# @POST: Exposes stateless mapping suggestion orchestration over configured environments.
# @SIDE_EFFECT: Performs remote metadata reads through Superset API clients.
# @DATA_CONTRACT: Input[source_env_id: str, target_env_id: str] -> Output[List[Dict]]
# @RELATION: DEPENDS_ON -> SupersetClient
# @RELATION: DEPENDS_ON -> suggest_mappings
#
# @INVARIANT: Suggestions are based on database names.
@@ -15,57 +20,80 @@ from ..core.superset_client import SupersetClient
from ..core.utils.matching import suggest_mappings
# [/SECTION]
# [DEF:MappingService:Class]
# @PURPOSE: Service for handling database mapping logic.
# @COMPLEXITY: 3
# @PRE: config_manager exposes get_environments() with environment objects containing id.
# @POST: Provides client resolution and mapping suggestion methods.
# @SIDE_EFFECT: Instantiates Superset clients and performs upstream metadata reads.
# @DATA_CONTRACT: Input[config_manager] -> Output[List[Dict]]
# @RELATION: DEPENDS_ON -> SupersetClient
# @RELATION: DEPENDS_ON -> suggest_mappings
class MappingService:
# [DEF:init:Function]
# @PURPOSE: Initializes the mapping service with a config manager.
# @COMPLEXITY: 3
# @PRE: config_manager is provided.
# @PARAM: config_manager (ConfigManager) - The configuration manager.
# @POST: Service is initialized.
# @RELATION: DEPENDS_ON -> MappingService
def __init__(self, config_manager):
with belief_scope("MappingService.__init__"):
self.config_manager = config_manager
# [/DEF:init:Function]
# [DEF:_get_client:Function]
# @PURPOSE: Helper to get an initialized SupersetClient for an environment.
# @COMPLEXITY: 3
# @PARAM: env_id (str) - The ID of the environment.
# @PRE: environment must exist in config.
# @POST: Returns an initialized SupersetClient.
# @RETURN: SupersetClient - Initialized client.
# @RELATION: CALLS -> SupersetClient
def _get_client(self, env_id: str) -> SupersetClient:
with belief_scope("MappingService._get_client", f"env_id={env_id}"):
envs = self.config_manager.get_environments()
env = next((e for e in envs if e.id == env_id), None)
if not env:
raise ValueError(f"Environment {env_id} not found")
return SupersetClient(env)
# [/DEF:_get_client:Function]
# [DEF:get_suggestions:Function]
# @PURPOSE: Fetches databases from both environments and returns fuzzy matching suggestions.
# @COMPLEXITY: 3
# @PARAM: source_env_id (str) - Source environment ID.
# @PARAM: target_env_id (str) - Target environment ID.
# @PRE: Both environments must be accessible.
# @POST: Returns fuzzy-matched database suggestions.
# @RETURN: List[Dict] - Suggested mappings.
async def get_suggestions(self, source_env_id: str, target_env_id: str) -> List[Dict]:
with belief_scope("MappingService.get_suggestions", f"source={source_env_id}, target={target_env_id}"):
# @RELATION: CALLS -> _get_client
# @RELATION: CALLS -> suggest_mappings
async def get_suggestions(
self, source_env_id: str, target_env_id: str
) -> List[Dict]:
with belief_scope(
"MappingService.get_suggestions",
f"source={source_env_id}, target={target_env_id}",
):
"""
Get suggested mappings between two environments.
"""
source_client = self._get_client(source_env_id)
target_client = self._get_client(target_env_id)
source_dbs = source_client.get_databases_summary()
target_dbs = target_client.get_databases_summary()
return suggest_mappings(source_dbs, target_dbs)
# [/DEF:get_suggestions:Function]
# [/DEF:MappingService:Class]
# [/DEF:mapping_service:Module]

21
backend/src/services/notifications/providers.py
View File

@@ -3,8 +3,17 @@
# @COMPLEXITY: 5
# @SEMANTICS: notifications, providers, smtp, slack, telegram, abstraction
# @PURPOSE: Defines abstract base and concrete implementations for external notification delivery.
# @RELATION: IMPLEMENTS ->[NotificationService:Class]
# @RELATION: [DEPENDED_ON_BY] ->[NotificationService]
# @RELATION: [DEPENDS_ON] ->[NotificationProvider]
# @RELATION: [DEPENDS_ON] ->[SMTPProvider]
# @RELATION: [DEPENDS_ON] ->[TelegramProvider]
# @RELATION: [DEPENDS_ON] ->[SlackProvider]
# @LAYER: Infra
# @PRE: Provider configuration dictionaries are supplied by trusted configuration sources.
# @POST: Each provider exposes async send contract returning boolean delivery outcome.
# @SIDE_EFFECT: Performs outbound network I/O to SMTP or HTTP endpoints.
# @DATA_CONTRACT: Input[target, subject, body, context?] -> Output[bool]
# @INVARIANT: Concrete providers preserve boolean send contract and swallow transport exceptions into False.
#
# @INVARIANT: Providers must be stateless and resilient to network failures.
# @INVARIANT: Sensitive credentials must be handled via encrypted config.
@@ -20,7 +29,11 @@ from ...core.logger import logger
# [DEF:NotificationProvider:Class]
# @COMPLEXITY: 2
# @PURPOSE: Abstract base class for all notification providers.
# @RELATION: [DEPENDED_ON_BY] ->[SMTPProvider]
# @RELATION: [DEPENDED_ON_BY] ->[TelegramProvider]
# @RELATION: [DEPENDED_ON_BY] ->[SlackProvider]
class NotificationProvider(ABC):
@abstractmethod
async def send(
@@ -45,7 +58,9 @@ class NotificationProvider(ABC):
# [DEF:SMTPProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via SMTP.
# @RELATION: [INHERITS] ->[NotificationProvider]
class SMTPProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.host = config.get("host")
@@ -88,7 +103,9 @@ class SMTPProvider(NotificationProvider):
# [DEF:TelegramProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via Telegram Bot API.
# @RELATION: [INHERITS] ->[NotificationProvider]
class TelegramProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.bot_token = config.get("bot_token")
@@ -126,7 +143,9 @@ class TelegramProvider(NotificationProvider):
# [DEF:SlackProvider:Class]
# @COMPLEXITY: 3
# @PURPOSE: Delivers notifications via Slack Webhooks or API.
# @RELATION: [INHERITS] ->[NotificationProvider]
class SlackProvider(NotificationProvider):
def __init__(self, config: Dict[str, Any]):
self.webhook_url = config.get("webhook_url")

11
backend/src/services/profile_service.py
View File

@@ -86,9 +86,18 @@ class ProfileAuthorizationError(Exception):
# [DEF:ProfileService:Class]
# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session
# @RELATION: [DEPENDS_ON] ->[sqlalchemy.orm.Session]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.auth.repository.AuthRepository]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: [DEPENDS_ON] ->[backend.src.core.superset_profile_lookup.SupersetAccountLookupAdapter]
# @RELATION: [DEPENDS_ON] ->[backend.src.models.profile.UserDashboardPreference]
# @RELATION: [CALLS] ->[backend.src.services.rbac_permission_catalog.discover_declared_permissions]
# @COMPLEXITY: 5
# @PURPOSE: Implements profile preference read/update flow and Superset account lookup degradation strategy.
# @PRE: Caller provides authenticated User context for external service methods.
# @POST: Preference operations remain user-scoped and return normalized profile/lookup responses.
# @SIDE_EFFECT: Writes preference records and encrypted tokens; performs external account lookups when requested.
# @DATA_CONTRACT: Input[User,ProfilePreferenceUpdateRequest|SupersetAccountLookupRequest] -> Output[ProfilePreferenceResponse|SupersetAccountLookupResponse|bool]
class ProfileService:
# [DEF:init:Function]
# @RELATION: BINDS_TO -> ProfileService

8
backend/src/services/reports/__tests__/test_report_normalizer.py
View File

@@ -14,6 +14,7 @@ from src.services.reports.normalizer import normalize_task_report
# [DEF:test_unknown_type_maps_to_unknown_profile:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure unknown plugin IDs map to unknown profile with populated summary and error context.
def test_unknown_type_maps_to_unknown_profile():
task = Task(
id="unknown-1",
@@ -34,8 +35,10 @@ def test_unknown_type_maps_to_unknown_profile():
# [/DEF:test_unknown_type_maps_to_unknown_profile:Function]
# [DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure missing result payload still yields visible report details with result placeholder.
def test_partial_payload_keeps_report_visible_with_placeholders():
task = Task(
id="partial-1",
@@ -56,8 +59,10 @@ def test_partial_payload_keeps_report_visible_with_placeholders():
# [/DEF:test_partial_payload_keeps_report_visible_with_placeholders:Function]
# [DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# @RELATION: BINDS_TO -> test_report_normalizer
# @PURPOSE: Ensure clean-release plugin ID maps to clean_release task profile and summary passthrough.
def test_clean_release_plugin_maps_to_clean_release_task_type():
task = Task(
id="clean-release-1",
@@ -75,4 +80,5 @@ def test_clean_release_plugin_maps_to_clean_release_task_type():
assert report.summary == "Clean release compliance passed"
# [/DEF:test_report_normalizer:Module]# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# [/DEF:test_clean_release_plugin_maps_to_clean_release_task_type:Function]
# [/DEF:test_report_normalizer:Module]

10
backend/src/services/reports/type_profiles.py
View File

@@ -3,7 +3,9 @@
# @SEMANTICS: reports, type_profiles, normalization, fallback
# @PURPOSE: Deterministic mapping of plugin/task identifiers to canonical report task types and fallback profile metadata.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[backend.src.models.report.TaskType:Function]
# @RELATION: [DEPENDS_ON] ->[TaskType]
# @RELATION: [CONTAINS] ->[resolve_task_type]
# @RELATION: [CONTAINS] ->[get_type_profile]
# @INVARIANT: Unknown input always resolves to TaskType.UNKNOWN with a single fallback profile.
# [SECTION: IMPORTS]
@@ -97,6 +99,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType:
if not normalized:
return TaskType.UNKNOWN
return PLUGIN_TO_TASK_TYPE.get(normalized, TaskType.UNKNOWN)
# [/DEF:resolve_task_type:Function]
@@ -118,6 +122,8 @@ def resolve_task_type(plugin_id: Optional[str]) -> TaskType:
def get_type_profile(task_type: TaskType) -> Dict[str, Any]:
with belief_scope("get_type_profile"):
return TASK_TYPE_PROFILES.get(task_type, TASK_TYPE_PROFILES[TaskType.UNKNOWN])
# [/DEF:get_type_profile:Function]
# [/DEF:type_profiles:Module]
# [/DEF:type_profiles:Module]