busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: finalize semantic repair and test updates

Browse Source
This commit is contained in:
busya
2026-03-21 15:07:06 +03:00
parent c827c2f098
commit 2da548fd71
99 changed files with 2484 additions and 985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
backend/src/core/__tests__/test_config_manager_compat.py
View File

@@ -23,6 +23,8 @@ def test_get_payload_preserves_legacy_sections():
assert payload["settings"]["migration_sync_cron"] == "0 2 * * *"
assert payload["notifications"]["smtp"]["host"] == "mail.local"
# [/DEF:test_get_payload_preserves_legacy_sections:Function]
@@ -71,10 +73,22 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
credentials_id="legacy-user",
)
# [DEF:_FakeQuery:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal query stub returning hardcoded existing environment record list for sync tests.
# @INVARIANT: all() always returns [existing_record]; no parameterization or filtering.
class _FakeQuery:
def all(self):
return [existing_record]
# [/DEF:_FakeQuery:Class]
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_save_config_syncs_environment_records_for_fk_backed_flows]
# @COMPLEXITY: 1
# @PURPOSE: Minimal SQLAlchemy session stub that captures add/delete calls for environment sync assertions.
# @INVARIANT: query() always returns _FakeQuery; no real DB interaction.
class _FakeSession:
def query(self, model):
return _FakeQuery()
@@ -85,6 +99,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
def delete(self, value):
deleted_records.append(value)
# [/DEF:_FakeSession:Class]
session = _FakeSession()
config = AppConfig(
environments=[
@@ -107,6 +123,8 @@ def test_save_config_syncs_environment_records_for_fk_backed_flows():
assert added_records[0].url == "http://superset.local"
assert added_records[0].credentials_id == "demo"
assert deleted_records == [existing_record]
# [/DEF:test_save_config_syncs_environment_records_for_fk_backed_flows:Function]
@@ -123,6 +141,11 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
closed = {"value": False}
committed = {"value": False}
# [DEF:_FakeSession:Class]
# @RELATION: BINDS_TO -> [test_load_config_syncs_environment_records_from_existing_db_payload]
# @COMPLEXITY: 1
# @PURPOSE: Minimal session stub tracking commit/close signals for config load lifecycle assertions.
# @INVARIANT: No query or add semantics; only lifecycle signal tracking.
class _FakeSession:
def commit(self):
committed["value"] = True
@@ -130,6 +153,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
def close(self):
closed["value"] = True
# [/DEF:_FakeSession:Class]
fake_session = _FakeSession()
fake_record = SimpleNamespace(
id="global",
@@ -163,6 +188,8 @@ def test_load_config_syncs_environment_records_from_existing_db_payload(monkeypa
assert sync_calls[0][1].environments[0].id == "dev"
assert committed["value"] is True
assert closed["value"] is True
# [/DEF:test_load_config_syncs_environment_records_from_existing_db_payload:Function]
# [/DEF:TestConfigManagerCompat:Module]

73
backend/src/core/__tests__/test_throttled_scheduler.py
View File

@@ -7,8 +7,10 @@ from src.core.scheduler import ThrottledSchedulerConfigurator
# @COMPLEXITY: 3
# @PURPOSE: Unit tests for ThrottledSchedulerConfigurator distribution logic.
# [DEF:test_calculate_schedule_even_distribution:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate even spacing across a two-hour scheduling window for three tasks.
def test_calculate_schedule_even_distribution():
"""
@TEST_SCENARIO: 3 tasks in a 2-hour window should be spaced 1 hour apart.
@@ -17,18 +19,23 @@ def test_calculate_schedule_even_distribution():
end = time(3, 0)
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
assert schedule[1] == datetime(2024, 1, 1, 2, 0)
assert schedule[2] == datetime(2024, 1, 1, 3, 0)
# [/DEF:test_calculate_schedule_even_distribution:Function]
# [DEF:test_calculate_schedule_midnight_crossing:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate scheduler correctly rolls timestamps into the next day across midnight.
def test_calculate_schedule_midnight_crossing():
"""
@TEST_SCENARIO: Window from 23:00 to 01:00 (next day).
@@ -37,18 +44,23 @@ def test_calculate_schedule_midnight_crossing():
end = time(1, 0)
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 23, 0)
assert schedule[1] == datetime(2024, 1, 2, 0, 0)
assert schedule[2] == datetime(2024, 1, 2, 1, 0)
# [/DEF:test_calculate_schedule_midnight_crossing:Function]
# [DEF:test_calculate_schedule_single_task:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate single-task schedule returns only the window start timestamp.
def test_calculate_schedule_single_task():
"""
@TEST_SCENARIO: Single task should be scheduled at start time.
@@ -57,16 +69,21 @@ def test_calculate_schedule_single_task():
end = time(2, 0)
dashboards = ["d1"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 1
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
# [/DEF:test_calculate_schedule_single_task:Function]
# [DEF:test_calculate_schedule_empty_list:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate empty dashboard list produces an empty schedule.
def test_calculate_schedule_empty_list():
"""
@TEST_SCENARIO: Empty dashboard list returns empty schedule.
@@ -75,15 +92,20 @@ def test_calculate_schedule_empty_list():
end = time(2, 0)
dashboards = []
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert schedule == []
# [/DEF:test_calculate_schedule_empty_list:Function]
# [DEF:test_calculate_schedule_zero_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate zero-length window schedules all tasks at identical start timestamp.
def test_calculate_schedule_zero_window():
"""
@TEST_SCENARIO: Window start == end. All tasks at start time.
@@ -92,31 +114,40 @@ def test_calculate_schedule_zero_window():
end = time(1, 0)
dashboards = ["d1", "d2"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 2
assert schedule[0] == datetime(2024, 1, 1, 1, 0)
assert schedule[1] == datetime(2024, 1, 1, 1, 0)
# [/DEF:test_calculate_schedule_zero_window:Function]
# [DEF:test_calculate_schedule_very_small_window:Function]
# @RELATION: BINDS_TO -> test_throttled_scheduler
# @PURPOSE: Validate sub-second interpolation when task count exceeds near-zero window granularity.
def test_calculate_schedule_very_small_window():
"""
@TEST_SCENARIO: Window smaller than number of tasks (in seconds).
"""
start = time(1, 0, 0)
end = time(1, 0, 1) # 1 second window
end = time(1, 0, 1) # 1 second window
dashboards = ["d1", "d2", "d3"]
today = date(2024, 1, 1)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(start, end, dashboards, today)
schedule = ThrottledSchedulerConfigurator.calculate_schedule(
start, end, dashboards, today
)
assert len(schedule) == 3
assert schedule[0] == datetime(2024, 1, 1, 1, 0, 0)
assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s
assert schedule[1] == datetime(2024, 1, 1, 1, 0, 0, 500000) # 0.5s
assert schedule[2] == datetime(2024, 1, 1, 1, 0, 1)
# [/DEF:test_throttled_scheduler:Module]# [/DEF:test_calculate_schedule_very_small_window:Function]
# [/DEF:test_calculate_schedule_very_small_window:Function]
# [/DEF:test_throttled_scheduler:Module]

1
backend/src/core/auth/config.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: auth, config, settings, jwt, adfs
# @PURPOSE: Centralized configuration for authentication and authorization.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> pydantic
#

30
backend/src/core/auth/jwt.py
View File

@@ -4,8 +4,8 @@
# @SEMANTICS: jwt, token, session, auth
# @PURPOSE: JWT token generation and validation logic.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> jose
# @RELATION: USES -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
# @RELATION: USES -> [auth_config]
#
# @INVARIANT: Tokens must include expiration time and user identifier.
@@ -17,11 +17,12 @@ from .config import auth_config
from ..logger import belief_scope
# [/SECTION]
# [DEF:create_access_token:Function]
# @PURPOSE: Generates a new JWT access token.
# @PRE: data dict contains 'sub' (user_id) and optional 'scopes' (roles).
# @POST: Returns a signed JWT string.
# @RELATION: DEPENDS_ON -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
#
# @PARAM: data (dict) - Payload data for the token.
# @PARAM: expires_delta (Optional[timedelta]) - Custom expiration time.
@@ -32,26 +33,37 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES)
expire = datetime.utcnow() + timedelta(
minutes=auth_config.ACCESS_TOKEN_EXPIRE_MINUTES
)
to_encode.update({"exp": expire})
encoded_jwt = jwt.encode(to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM)
encoded_jwt = jwt.encode(
to_encode, auth_config.SECRET_KEY, algorithm=auth_config.ALGORITHM
)
return encoded_jwt
# [/DEF:create_access_token:Function]
# [DEF:decode_token:Function]
# @PURPOSE: Decodes and validates a JWT token.
# @PRE: token is a signed JWT string.
# @POST: Returns the decoded payload if valid.
# @RELATION: DEPENDS_ON -> auth_config
# @RELATION: DEPENDS_ON -> [auth_config]
#
# @PARAM: token (str) - The JWT to decode.
# @RETURN: dict - The decoded payload.
# @THROW: jose.JWTError - If token is invalid or expired.
def decode_token(token: str) -> dict:
with belief_scope("decode_token"):
payload = jwt.decode(token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM])
payload = jwt.decode(
token, auth_config.SECRET_KEY, algorithms=[auth_config.ALGORITHM]
)
return payload
# [/DEF:decode_token:Function]
# [/DEF:AuthJwtModule:Module]
# [/DEF:AuthJwtModule:Module]

1
backend/src/core/auth/oauth.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: auth, oauth, oidc, adfs
# @PURPOSE: ADFS OIDC configuration and client using Authlib.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> authlib
# @RELATION: USES -> auth_config

92
backend/src/core/auth/repository.py
View File

@@ -4,12 +4,12 @@
# @SEMANTICS: auth, repository, database, user, role, permission
# @PURPOSE: Data access layer for authentication and user preference entities.
# @LAYER: Domain
# @RELATION: DEPENDS_ON ->[sqlalchemy.orm.Session]
# @RELATION: DEPENDS_ON ->[User:Class]
# @RELATION: DEPENDS_ON ->[Role:Class]
# @RELATION: DEPENDS_ON ->[Permission:Class]
# @RELATION: DEPENDS_ON ->[UserDashboardPreference:Class]
# @RELATION: DEPENDS_ON ->[belief_scope:Function]
# @RELATION: DEPENDS_ON -> [Session]
# @RELATION: DEPENDS_ON -> [User]
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
# @RELATION: DEPENDS_ON -> [UserDashboardPreference]
# @RELATION: DEPENDS_ON -> [belief_scope]
# @INVARIANT: All database read/write operations must execute via the injected SQLAlchemy session boundary.
# @DATA_CONTRACT: Session -> [User | Role | Permission | UserDashboardPreference]
# @PRE: Database connection is active.
@@ -24,12 +24,13 @@ from ...models.profile import UserDashboardPreference
from ..logger import belief_scope, logger
# [/SECTION]
# [DEF:AuthRepository:Class]
# @PURPOSE: Provides low-level CRUD operations for identity and authorization records.
# @PRE: Database session is bound.
# @POST: Entity instances returned safely.
# @SIDE_EFFECT: Performs database reads.
# @RELATION: DEPENDS_ON -> sqlalchemy.orm.Session
# @RELATION: DEPENDS_ON -> [Session]
class AuthRepository:
# @PURPOSE: Initialize repository with database session.
def __init__(self, db: Session):
@@ -39,98 +40,125 @@ class AuthRepository:
# @PURPOSE: Retrieve user by UUID.
# @PRE: user_id is a valid UUID string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> User
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_id(self, user_id: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_id"):
logger.reason(f"Fetching user by id: {user_id}")
result = self.db.query(User).filter(User.id == user_id).first()
logger.reflect(f"User found: {result is not None}")
return result
# [/DEF:get_user_by_id:Function]
# [DEF:get_user_by_username:Function]
# @PURPOSE: Retrieve user by username.
# @PRE: username is a non-empty string.
# @POST: Returns User object if found, else None.
# @RELATION: DEPENDS_ON -> User
# @RELATION: DEPENDS_ON -> [User]
def get_user_by_username(self, username: str) -> Optional[User]:
with belief_scope("AuthRepository.get_user_by_username"):
logger.reason(f"Fetching user by username: {username}")
result = self.db.query(User).filter(User.username == username).first()
logger.reflect(f"User found: {result is not None}")
return result
# [/DEF:get_user_by_username:Function]
# [DEF:get_role_by_id:Function]
# @PURPOSE: Retrieve role by UUID with permissions preloaded.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [Permission]
def get_role_by_id(self, role_id: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_id"):
return self.db.query(Role).options(selectinload(Role.permissions)).filter(Role.id == role_id).first()
return (
self.db.query(Role)
.options(selectinload(Role.permissions))
.filter(Role.id == role_id)
.first()
)
# [/DEF:get_role_by_id:Function]
# [DEF:get_role_by_name:Function]
# @PURPOSE: Retrieve role by unique name.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> [Role]
def get_role_by_name(self, name: str) -> Optional[Role]:
with belief_scope("AuthRepository.get_role_by_name"):
return self.db.query(Role).filter(Role.name == name).first()
# [/DEF:get_role_by_name:Function]
# [DEF:get_permission_by_id:Function]
# @PURPOSE: Retrieve permission by UUID.
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_id(self, permission_id: str) -> Optional[Permission]:
with belief_scope("AuthRepository.get_permission_by_id"):
return self.db.query(Permission).filter(Permission.id == permission_id).first()
return (
self.db.query(Permission).filter(Permission.id == permission_id).first()
)
# [/DEF:get_permission_by_id:Function]
# [DEF:get_permission_by_resource_action:Function]
# @PURPOSE: Retrieve permission by resource and action tuple.
# @RELATION: DEPENDS_ON -> Permission
def get_permission_by_resource_action(self, resource: str, action: str) -> Optional[Permission]:
# @RELATION: DEPENDS_ON -> [Permission]
def get_permission_by_resource_action(
self, resource: str, action: str
) -> Optional[Permission]:
with belief_scope("AuthRepository.get_permission_by_resource_action"):
return self.db.query(Permission).filter(
Permission.resource == resource,
Permission.action == action
).first()
return (
self.db.query(Permission)
.filter(Permission.resource == resource, Permission.action == action)
.first()
)
# [/DEF:get_permission_by_resource_action:Function]
# [DEF:list_permissions:Function]
# @PURPOSE: List all system permissions.
# @RELATION: DEPENDS_ON -> Permission
# @RELATION: DEPENDS_ON -> [Permission]
def list_permissions(self) -> List[Permission]:
with belief_scope("AuthRepository.list_permissions"):
return self.db.query(Permission).all()
# [/DEF:list_permissions:Function]
# [DEF:get_user_dashboard_preference:Function]
# @PURPOSE: Retrieve dashboard filters/preferences for a user.
# @RELATION: DEPENDS_ON -> UserDashboardPreference
def get_user_dashboard_preference(self, user_id: str) -> Optional[UserDashboardPreference]:
# @RELATION: DEPENDS_ON -> [UserDashboardPreference]
def get_user_dashboard_preference(
self, user_id: str
) -> Optional[UserDashboardPreference]:
with belief_scope("AuthRepository.get_user_dashboard_preference"):
return self.db.query(UserDashboardPreference).filter(
UserDashboardPreference.user_id == user_id
).first()
return (
self.db.query(UserDashboardPreference)
.filter(UserDashboardPreference.user_id == user_id)
.first()
)
# [/DEF:get_user_dashboard_preference:Function]
# [DEF:get_roles_by_ad_groups:Function]
# @PURPOSE: Retrieve roles that match a list of AD group names.
# @PRE: groups is a list of strings representing AD group identifiers.
# @POST: Returns a list of Role objects mapped to the provided AD groups.
# @RELATION: DEPENDS_ON -> Role
# @RELATION: DEPENDS_ON -> ADGroupMapping
# @RELATION: DEPENDS_ON -> [Role]
# @RELATION: DEPENDS_ON -> [ADGroupMapping]
def get_roles_by_ad_groups(self, groups: List[str]) -> List[Role]:
with belief_scope("AuthRepository.get_roles_by_ad_groups"):
logger.reason(f"Fetching roles for AD groups: {groups}")
if not groups:
return []
return self.db.query(Role).join(ADGroupMapping).filter(
ADGroupMapping.ad_group.in_(groups)
).all()
return (
self.db.query(Role)
.join(ADGroupMapping)
.filter(ADGroupMapping.ad_group.in_(groups))
.all()
)
# [/DEF:get_roles_by_ad_groups:Function]
# [/DEF:AuthRepository:Class]
# [/DEF:AuthRepositoryModule:Module]

1
backend/src/core/auth/security.py
View File

@@ -2,6 +2,7 @@
#
# @SEMANTICS: security, password, hashing, bcrypt
# @PURPOSE: Utility for password hashing and verification using Passlib.
# @COMPLEXITY: 2
# @LAYER: Core
# @RELATION: DEPENDS_ON -> bcrypt
#

12
backend/src/core/encryption_key.py
View File

@@ -1,10 +1,14 @@
# [DEF:backend.src.core.encryption_key:Module]
# [DEF:EncryptionKeyModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: encryption, key, bootstrap, environment, startup
# @PURPOSE: Resolve and persist the Fernet encryption key required by runtime services.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Runtime key resolution never falls back to an ephemeral secret.
# @PRE: Runtime environment can read process variables and target .env path is writable when key generation is required.
# @POST: A valid Fernet key is available to runtime services via ENCRYPTION_KEY.
# @SIDE_EFFECT: May append ENCRYPTION_KEY entry into backend .env file and set process environment variable.
# @DATA_CONTRACT: Input[env_file_path] -> Output[encryption_key]
from __future__ import annotations
@@ -51,6 +55,8 @@ def ensure_encryption_key(env_file_path: Path = DEFAULT_ENV_FILE_PATH) -> str:
logger.reason(f"Generated ENCRYPTION_KEY and persisted it to {env_file_path}.")
logger.reflect("Encryption key is available for runtime services.")
return generated_key
# [/DEF:ensure_encryption_key:Function]
# [/DEF:backend.src.core.encryption_key:Module]
# [/DEF:EncryptionKeyModule:Module]

187
backend/src/core/mapping_service.py
View File

@@ -1,11 +1,15 @@
# [DEF:backend.src.core.mapping_service:Module]
# [DEF:IdMappingServiceModule:Module]
#
# @COMPLEXITY: 5
# @SEMANTICS: mapping, ids, synchronization, environments, cross-filters
# @PURPOSE: Service for tracking and synchronizing Superset Resource IDs (UUID <-> Integer ID)
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.models.mapping (ResourceMapping, ResourceType)
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [MappingModels]
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @PRE: Database session is valid and Superset client factory returns authenticated clients for requested environments.
# @POST: Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT: Reads/writes ResourceMapping rows, emits logs, and schedules periodic sync jobs.
# @DATA_CONTRACT: Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
#
# @INVARIANT: sync_environment must handle remote API failures gracefully.
@@ -20,9 +24,17 @@ from src.models.mapping import ResourceMapping, ResourceType
from src.core.logger import logger, belief_scope
# [/SECTION]
# [DEF:IdMappingService:Class]
# @COMPLEXITY: 5
# @PURPOSE: Service handling the cataloging and retrieval of remote Superset Integer IDs.
# @PRE: db_session is an active SQLAlchemy Session bound to mapping tables.
# @POST: Service instance provides scheduler control and environment-scoped mapping synchronization APIs.
# @RELATION: DEPENDS_ON -> [MappingModels]
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: self.db remains the authoritative session for all mapping operations.
# @SIDE_EFFECT: Instantiates an in-process scheduler and performs database writes during sync cycles.
# @DATA_CONTRACT: Input[db_session] -> Output[IdMappingService]
#
# @TEST_CONTRACT: IdMappingServiceModel ->
# {
@@ -39,13 +51,13 @@ from src.core.logger import logger, belief_scope
# @TEST_EDGE: get_batch_empty_list -> returns empty dict
# @TEST_INVARIANT: resilient_fetching -> verifies: [sync_api_failure]
class IdMappingService:
# [DEF:__init__:Function]
# @PURPOSE: Initializes the mapping service.
def __init__(self, db_session: Session):
self.db = db_session
self.scheduler = BackgroundScheduler()
self._sync_job = None
# [/DEF:__init__:Function]
# [DEF:start_scheduler:Function]
@@ -53,30 +65,39 @@ class IdMappingService:
# @PARAM: cron_string (str) - Cron expression for the sync interval.
# @PARAM: environments (List[str]) - List of environment IDs to sync.
# @PARAM: superset_client_factory - Function to get a client for an environment.
def start_scheduler(self, cron_string: str, environments: List[str], superset_client_factory):
def start_scheduler(
self, cron_string: str, environments: List[str], superset_client_factory
):
with belief_scope("IdMappingService.start_scheduler"):
if self._sync_job:
self.scheduler.remove_job(self._sync_job.id)
logger.info("[IdMappingService.start_scheduler][Reflect] Removed existing sync job.")
logger.info(
"[IdMappingService.start_scheduler][Reflect] Removed existing sync job."
)
def sync_all():
for env_id in environments:
client = superset_client_factory(env_id)
if client:
self.sync_environment(env_id, client)
self._sync_job = self.scheduler.add_job(
sync_all,
CronTrigger.from_crontab(cron_string),
id='id_mapping_sync_job',
replace_existing=True
id="id_mapping_sync_job",
replace_existing=True,
)
if not self.scheduler.running:
self.scheduler.start()
logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Started background scheduler with cron: {cron_string}"
)
else:
logger.info(f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}")
logger.info(
f"[IdMappingService.start_scheduler][Coherence:OK] Updated background scheduler with cron: {cron_string}"
)
# [/DEF:start_scheduler:Function]
# [DEF:sync_environment:Function]
@@ -85,54 +106,74 @@ class IdMappingService:
# @PARAM: superset_client - Instance capable of hitting the Superset API.
# @PRE: environment_id exists in the database.
# @POST: ResourceMapping records for the environment are created or updated.
def sync_environment(self, environment_id: str, superset_client, incremental: bool = False) -> None:
def sync_environment(
self, environment_id: str, superset_client, incremental: bool = False
) -> None:
"""
Polls the Superset APIs for the target environment and updates the local mapping table.
If incremental=True, only fetches items changed since the max last_synced_at date.
"""
with belief_scope("IdMappingService.sync_environment"):
logger.info(f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})")
logger.info(
f"[IdMappingService.sync_environment][Action] Starting sync for environment {environment_id} (incremental={incremental})"
)
# Implementation Note: In a real scenario, superset_client needs to be an instance
# capable of auth & iteration over /api/v1/chart/, /api/v1/dataset/, /api/v1/dashboard/
# Here we structure the logic according to the spec.
types_to_poll = [
(ResourceType.CHART, "chart", "slice_name"),
(ResourceType.DATASET, "dataset", "table_name"),
(ResourceType.DASHBOARD, "dashboard", "slug") # Note: dashboard slug or dashboard_title
(
ResourceType.DASHBOARD,
"dashboard",
"slug",
), # Note: dashboard slug or dashboard_title
]
total_synced = 0
total_deleted = 0
try:
for res_enum, endpoint, name_field in types_to_poll:
logger.debug(f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint")
logger.debug(
f"[IdMappingService.sync_environment][Explore] Polling {endpoint} endpoint"
)
# Simulated API Fetch (Would be: superset_client.get(f"/api/v1/{endpoint}/")... )
# This relies on the superset API structure, e.g. { "result": [{"id": 1, "uuid": "...", name_field: "..."}] }
# We assume superset_client provides a generic method to fetch all pages.
try:
since_dttm = None
if incremental:
from sqlalchemy.sql import func
max_date = self.db.query(func.max(ResourceMapping.last_synced_at)).filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == res_enum
).scalar()
max_date = (
self.db.query(func.max(ResourceMapping.last_synced_at))
.filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == res_enum,
)
.scalar()
)
if max_date:
# We subtract a bit for safety overlap
from datetime import timedelta
since_dttm = max_date - timedelta(minutes=5)
logger.debug(f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}")
resources = superset_client.get_all_resources(endpoint, since_dttm=since_dttm)
since_dttm = max_date - timedelta(minutes=5)
logger.debug(
f"[IdMappingService.sync_environment] Incremental sync since {since_dttm}"
)
resources = superset_client.get_all_resources(
endpoint, since_dttm=since_dttm
)
# Track which UUIDs we see in this sync cycle
synced_uuids = set()
for res in resources:
res_uuid = res.get("uuid")
raw_id = res.get("id")
@@ -140,16 +181,20 @@ class IdMappingService:
if not res_uuid or raw_id is None:
continue
synced_uuids.add(res_uuid)
res_id = str(raw_id) # Store as string
res_id = str(raw_id) # Store as string
# Upsert Logic
mapping = self.db.query(ResourceMapping).filter_by(
environment_id=environment_id,
resource_type=res_enum,
uuid=res_uuid
).first()
mapping = (
self.db.query(ResourceMapping)
.filter_by(
environment_id=environment_id,
resource_type=res_enum,
uuid=res_uuid,
)
.first()
)
if mapping:
mapping.remote_integer_id = res_id
@@ -162,10 +207,10 @@ class IdMappingService:
uuid=res_uuid,
remote_integer_id=res_id,
resource_name=res_name,
last_synced_at=datetime.now(timezone.utc)
last_synced_at=datetime.now(timezone.utc),
)
self.db.add(new_mapping)
total_synced += 1
# Delete stale mappings: rows for this env+type whose UUID
@@ -183,19 +228,28 @@ class IdMappingService:
deleted = stale_query.delete(synchronize_session="fetch")
if deleted:
total_deleted += deleted
logger.info(f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}")
logger.info(
f"[IdMappingService.sync_environment][Action] Removed {deleted} stale {endpoint} mapping(s) for {environment_id}"
)
except Exception as loop_e:
logger.error(f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}")
logger.error(
f"[IdMappingService.sync_environment][Reason] Error polling {endpoint}: {loop_e}"
)
# Continue to next resource type instead of blowing up the whole sync
self.db.commit()
logger.info(f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items.")
logger.info(
f"[IdMappingService.sync_environment][Coherence:OK] Successfully synced {total_synced} items and deleted {total_deleted} stale items."
)
except Exception as e:
self.db.rollback()
logger.error(f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}")
logger.error(
f"[IdMappingService.sync_environment][Coherence:Failed] Critical sync failure: {e}"
)
raise
# [/DEF:sync_environment:Function]
# [DEF:get_remote_id:Function]
@@ -204,19 +258,24 @@ class IdMappingService:
# @PARAM: resource_type (ResourceType)
# @PARAM: uuid (str)
# @RETURN: Optional[int]
def get_remote_id(self, environment_id: str, resource_type: ResourceType, uuid: str) -> Optional[int]:
mapping = self.db.query(ResourceMapping).filter_by(
environment_id=environment_id,
resource_type=resource_type,
uuid=uuid
).first()
def get_remote_id(
self, environment_id: str, resource_type: ResourceType, uuid: str
) -> Optional[int]:
mapping = (
self.db.query(ResourceMapping)
.filter_by(
environment_id=environment_id, resource_type=resource_type, uuid=uuid
)
.first()
)
if mapping:
try:
return int(mapping.remote_integer_id)
except ValueError:
return None
return None
# [/DEF:get_remote_id:Function]
# [DEF:get_remote_ids_batch:Function]
@@ -225,15 +284,21 @@ class IdMappingService:
# @PARAM: resource_type (ResourceType)
# @PARAM: uuids (List[str])
# @RETURN: Dict[str, int] - Mapping of UUID -> Integer ID
def get_remote_ids_batch(self, environment_id: str, resource_type: ResourceType, uuids: List[str]) -> Dict[str, int]:
def get_remote_ids_batch(
self, environment_id: str, resource_type: ResourceType, uuids: List[str]
) -> Dict[str, int]:
if not uuids:
return {}
mappings = self.db.query(ResourceMapping).filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == resource_type,
ResourceMapping.uuid.in_(uuids)
).all()
mappings = (
self.db.query(ResourceMapping)
.filter(
ResourceMapping.environment_id == environment_id,
ResourceMapping.resource_type == resource_type,
ResourceMapping.uuid.in_(uuids),
)
.all()
)
result = {}
for m in mappings:
@@ -241,9 +306,11 @@ class IdMappingService:
result[m.uuid] = int(m.remote_integer_id)
except ValueError:
pass
return result
# [/DEF:get_remote_ids_batch:Function]
# [/DEF:IdMappingService:Class]
# [/DEF:backend.src.core.mapping_service:Module]
# [/DEF:IdMappingServiceModule:Module]

4
backend/src/core/migration/__init__.py
View File

@@ -1,4 +1,4 @@
# [DEF:backend.src.core.migration.__init__:Module]
# [DEF:MigrationPackage:Module]
# @COMPLEXITY: 1
# @SEMANTICS: migration, package, exports
# @PURPOSE: Namespace package for migration pre-flight orchestration components.
@@ -9,4 +9,4 @@ from .archive_parser import MigrationArchiveParser
__all__ = ["MigrationDryRunService", "MigrationArchiveParser"]
# [/DEF:backend.src.core.migration.__init__:Module]
# [/DEF:MigrationPackage:Module]

14
backend/src/core/migration/archive_parser.py
View File

@@ -1,9 +1,9 @@
# [DEF:backend.src.core.migration.archive_parser:Module]
# [DEF:MigrationArchiveParserModule:Module]
# @COMPLEXITY: 3
# @SEMANTICS: migration, zip, parser, yaml, metadata
# @PURPOSE: Parse Superset export ZIP archives into normalized object catalogs for diffing.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> backend.src.core.logger
# @RELATION: DEPENDS_ON -> [LoggerModule]
# @INVARIANT: Parsing is read-only and never mutates archive files.
import json
@@ -19,11 +19,13 @@ from ..logger import logger, belief_scope
# [DEF:MigrationArchiveParser:Class]
# @PURPOSE: Extract normalized dashboards/charts/datasets metadata from ZIP archives.
# @RELATION: CONTAINS -> [extract_objects_from_zip, _collect_yaml_objects, _normalize_object_payload]
# @RELATION: CONTAINS -> [extract_objects_from_zip]
# @RELATION: CONTAINS -> [_collect_yaml_objects]
# @RELATION: CONTAINS -> [_normalize_object_payload]
class MigrationArchiveParser:
# [DEF:extract_objects_from_zip:Function]
# @PURPOSE: Extract object catalogs from Superset archive.
# @RELATION: DEPENDS_ON -> _collect_yaml_objects
# @RELATION: DEPENDS_ON -> [_collect_yaml_objects]
# @PRE: zip_path points to a valid readable ZIP.
# @POST: Returns object lists grouped by resource type.
# @RETURN: Dict[str, List[Dict[str, Any]]]
@@ -53,7 +55,7 @@ class MigrationArchiveParser:
# [DEF:_collect_yaml_objects:Function]
# @PURPOSE: Read and normalize YAML manifests for one object type.
# @RELATION: DEPENDS_ON -> _normalize_object_payload
# @RELATION: DEPENDS_ON -> [_normalize_object_payload]
# @PRE: object_type is one of dashboards/charts/datasets.
# @POST: Returns only valid normalized objects.
def _collect_yaml_objects(
@@ -153,4 +155,4 @@ class MigrationArchiveParser:
# [/DEF:MigrationArchiveParser:Class]
# [/DEF:backend.src.core.migration.archive_parser:Module]
# [/DEF:MigrationArchiveParserModule:Module]

29
backend/src/core/migration/dry_run_orchestrator.py
View File

@@ -1,12 +1,12 @@
# [DEF:backend.src.core.migration.dry_run_orchestrator:Module]
# [DEF:MigrationDryRunOrchestratorModule:Module]
# @COMPLEXITY: 3
# @SEMANTICS: migration, dry_run, diff, risk, superset
# @PURPOSE: Compute pre-flight migration diff and risk scoring without apply.
# @LAYER: Core
# @RELATION: DEPENDS_ON ->[backend.src.core.superset_client.SupersetClient]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration_engine.MigrationEngine]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.archive_parser.MigrationArchiveParser]
# @RELATION: DEPENDS_ON ->[backend.src.core.migration.risk_assessor]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: DEPENDS_ON -> [MigrationEngine]
# @RELATION: DEPENDS_ON -> [MigrationArchiveParser]
# @RELATION: DEPENDS_ON -> [RiskAssessorModule]
# @INVARIANT: Dry run is informative only and must not mutate target environment.
from datetime import datetime, timezone
@@ -27,7 +27,14 @@ from ..utils.fileio import create_temp_file
# [DEF:MigrationDryRunService:Class]
# @PURPOSE: Build deterministic diff/risk payload for migration pre-flight.
# @RELATION: CONTAINS -> [__init__, run, _load_db_mapping, _accumulate_objects, _index_by_uuid, _build_object_diff, _build_target_signatures, _build_risks]
# @RELATION: CONTAINS -> [__init__]
# @RELATION: CONTAINS -> [run]
# @RELATION: CONTAINS -> [_load_db_mapping]
# @RELATION: CONTAINS -> [_accumulate_objects]
# @RELATION: CONTAINS -> [_index_by_uuid]
# @RELATION: CONTAINS -> [_build_object_diff]
# @RELATION: CONTAINS -> [_build_target_signatures]
# @RELATION: CONTAINS -> [_build_risks]
class MigrationDryRunService:
# [DEF:__init__:Function]
# @PURPOSE: Wire parser dependency for archive object extraction.
@@ -40,7 +47,11 @@ class MigrationDryRunService:
# [DEF:run:Function]
# @PURPOSE: Execute full dry-run computation for selected dashboards.
# @RELATION: DEPENDS_ON -> [_load_db_mapping, _accumulate_objects, _build_target_signatures, _build_object_diff, _build_risks]
# @RELATION: DEPENDS_ON -> [_load_db_mapping]
# @RELATION: DEPENDS_ON -> [_accumulate_objects]
# @RELATION: DEPENDS_ON -> [_build_target_signatures]
# @RELATION: DEPENDS_ON -> [_build_object_diff]
# @RELATION: DEPENDS_ON -> [_build_risks]
# @PRE: source/target clients are authenticated and selection validated by caller.
# @POST: Returns JSON-serializable pre-flight payload with summary, diff and risk.
# @SIDE_EFFECT: Reads source export archives and target metadata via network.
@@ -195,7 +206,7 @@ class MigrationDryRunService:
# [DEF:_build_object_diff:Function]
# @PURPOSE: Compute create/update/delete buckets by UUID+signature.
# @RELATION: DEPENDS_ON -> _index_by_uuid
# @RELATION: DEPENDS_ON -> [_index_by_uuid]
def _build_object_diff(
self, source_objects: List[Dict[str, Any]], target_objects: List[Dict[str, Any]]
) -> Dict[str, List[Dict[str, Any]]]:
@@ -349,4 +360,4 @@ class MigrationDryRunService:
# [/DEF:MigrationDryRunService:Class]
# [/DEF:backend.src.core.migration.dry_run_orchestrator:Module]
# [/DEF:MigrationDryRunOrchestratorModule:Module]

19
backend/src/core/migration/risk_assessor.py
View File

@@ -1,12 +1,18 @@
# [DEF:backend.src.core.migration.risk_assessor:Module]
# [DEF:RiskAssessorModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: migration, dry_run, risk, scoring, preflight
# @PURPOSE: Compute deterministic migration risk items and aggregate score for dry-run reporting.
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> [backend.src.core.superset_client.SupersetClient]
# @RELATION: DISPATCHED_BY -> [backend.src.core.migration.dry_run_orchestrator.MigrationDryRunService.run]
# @RELATION: CONTAINS -> [index_by_uuid, extract_owner_identifiers, build_risks, score_risks]
# @RELATION: DEPENDS_ON -> [SupersetClient]
# @RELATION: CONTAINS -> [index_by_uuid]
# @RELATION: CONTAINS -> [extract_owner_identifiers]
# @RELATION: CONTAINS -> [build_risks]
# @RELATION: CONTAINS -> [score_risks]
# @INVARIANT: Risk scoring must remain bounded to [0,100] and preserve severity-to-weight mapping.
# @PRE: Risk assessor functions receive normalized migration object collections from dry-run orchestration.
# @POST: Risk scoring output preserves item list and provides bounded score with derived level.
# @SIDE_EFFECT: Emits diagnostic logs and performs read-only metadata requests via Superset client.
# @DATA_CONTRACT: Module[build_risks, score_risks]
# @TEST_CONTRACT: [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
# @TEST_SCENARIO: [overwrite_update_objects] -> [medium overwrite_existing risk is emitted for each update diff item]
# @TEST_SCENARIO: [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
@@ -80,7 +86,8 @@ def extract_owner_identifiers(owners: Any) -> List[str]:
# [DEF:build_risks:Function]
# @PURPOSE: Build risk list from computed diffs and target catalog state.
# @RELATION: DEPENDS_ON -> [index_by_uuid, extract_owner_identifiers]
# @RELATION: DEPENDS_ON -> [index_by_uuid]
# @RELATION: DEPENDS_ON -> [extract_owner_identifiers]
# @PRE: source_objects/target_objects/diff contain dashboards/charts/datasets keys with expected list structures.
# @PRE: target_client is authenticated/usable for database list retrieval.
# @POST: Returns list of deterministic risk items derived from overwrite, missing datasource, reference, and owner mismatch checks.
@@ -192,4 +199,4 @@ def score_risks(risk_items: List[Dict[str, Any]]) -> Dict[str, Any]:
# [/DEF:score_risks:Function]
# [/DEF:backend.src.core.migration.risk_assessor:Module]
# [/DEF:RiskAssessorModule:Module]

79
backend/src/core/scheduler.py
View File

@@ -3,20 +3,18 @@
# @SEMANTICS: scheduler, apscheduler, cron, backup
# @PURPOSE: Manages scheduled tasks using APScheduler.
# @LAYER: Core
# @RELATION: Uses TaskManager to run scheduled backups.
# @RELATION: DEPENDS_ON -> TaskManager
# [SECTION: IMPORTS]
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.cron import CronTrigger
from apscheduler.triggers.date import DateTrigger
from .logger import logger, belief_scope
from .config_manager import ConfigManager
from .database import SessionLocal
from ..models.llm import ValidationPolicy
import asyncio
from datetime import datetime, time, timedelta, date
# [/SECTION]
# [DEF:SchedulerService:Class]
# @COMPLEXITY: 3
# @SEMANTICS: scheduler, service, apscheduler
@@ -32,6 +30,7 @@ class SchedulerService:
self.config_manager = config_manager
self.scheduler = BackgroundScheduler()
self.loop = asyncio.get_event_loop()
# [/DEF:__init__:Function]
# [DEF:start:Function]
@@ -44,6 +43,7 @@ class SchedulerService:
self.scheduler.start()
logger.info("Scheduler started.")
self.load_schedules()
# [/DEF:start:Function]
# [DEF:stop:Function]
@@ -55,6 +55,7 @@ class SchedulerService:
if self.scheduler.running:
self.scheduler.shutdown()
logger.info("Scheduler stopped.")
# [/DEF:stop:Function]
# [DEF:load_schedules:Function]
@@ -65,11 +66,12 @@ class SchedulerService:
with belief_scope("SchedulerService.load_schedules"):
# Clear existing jobs
self.scheduler.remove_all_jobs()
config = self.config_manager.get_config()
for env in config.environments:
if env.backup_schedule and env.backup_schedule.enabled:
self.add_backup_job(env.id, env.backup_schedule.cron_expression)
# [/DEF:load_schedules:Function]
# [DEF:add_backup_job:Function]
@@ -79,7 +81,10 @@ class SchedulerService:
# @PARAM: env_id (str) - The ID of the environment.
# @PARAM: cron_expression (str) - The cron expression for the schedule.
def add_backup_job(self, env_id: str, cron_expression: str):
with belief_scope("SchedulerService.add_backup_job", f"env_id={env_id}, cron={cron_expression}"):
with belief_scope(
"SchedulerService.add_backup_job",
f"env_id={env_id}, cron={cron_expression}",
):
job_id = f"backup_{env_id}"
try:
self.scheduler.add_job(
@@ -87,11 +92,14 @@ class SchedulerService:
CronTrigger.from_crontab(cron_expression),
id=job_id,
args=[env_id],
replace_existing=True
replace_existing=True,
)
logger.info(
f"Scheduled backup job added for environment {env_id}: {cron_expression}"
)
logger.info(f"Scheduled backup job added for environment {env_id}: {cron_expression}")
except Exception as e:
logger.error(f"Failed to add backup job for environment {env_id}: {e}")
# [/DEF:add_backup_job:Function]
# [DEF:_trigger_backup:Function]
@@ -102,30 +110,45 @@ class SchedulerService:
def _trigger_backup(self, env_id: str):
with belief_scope("SchedulerService._trigger_backup", f"env_id={env_id}"):
logger.info(f"Triggering scheduled backup for environment {env_id}")
# Check if a backup is already running for this environment
active_tasks = self.task_manager.get_tasks(limit=100)
for task in active_tasks:
if (task.plugin_id == "superset-backup" and
task.status in ["PENDING", "RUNNING"] and
task.params.get("environment_id") == env_id):
logger.warning(f"Backup already running for environment {env_id}. Skipping scheduled run.")
if (
task.plugin_id == "superset-backup"
and task.status in ["PENDING", "RUNNING"]
and task.params.get("environment_id") == env_id
):
logger.warning(
f"Backup already running for environment {env_id}. Skipping scheduled run."
)
return
# Run the backup task
# We need to run this in the event loop since create_task is async
asyncio.run_coroutine_threadsafe(
self.task_manager.create_task("superset-backup", {"environment_id": env_id}),
self.loop
self.task_manager.create_task(
"superset-backup", {"environment_id": env_id}
),
self.loop,
)
# [/DEF:_trigger_backup:Function]
# [/DEF:SchedulerService:Class]
# [DEF:ThrottledSchedulerConfigurator:Class]
# @COMPLEXITY: 5
# @SEMANTICS: scheduler, throttling, distribution
# @PURPOSE: Distributes validation tasks evenly within an execution window.
# @PRE: Validation policies provide a finite dashboard list and a valid execution window.
# @POST: Produces deterministic per-dashboard run timestamps within the configured window.
# @RELATION: DEPENDS_ON -> SchedulerModule
# @INVARIANT: Returned schedule size always matches number of dashboard IDs.
# @SIDE_EFFECT: Emits warning logs for degenerate or near-zero scheduling windows.
# @DATA_CONTRACT: Input[window_start, window_end, dashboard_ids, current_date] -> Output[List[datetime]]
class ThrottledSchedulerConfigurator:
# [DEF:calculate_schedule:Function]
# @PURPOSE: Calculates execution times for N tasks within a window.
@@ -134,10 +157,7 @@ class ThrottledSchedulerConfigurator:
# @INVARIANT: Tasks are distributed with near-even spacing.
@staticmethod
def calculate_schedule(
window_start: time,
window_end: time,
dashboard_ids: list,
current_date: date
window_start: time, window_end: time, dashboard_ids: list, current_date: date
) -> list:
with belief_scope("ThrottledSchedulerConfigurator.calculate_schedule"):
n = len(dashboard_ids)
@@ -152,32 +172,39 @@ class ThrottledSchedulerConfigurator:
end_dt += timedelta(days=1)
total_seconds = (end_dt - start_dt).total_seconds()
# Minimum interval of 1 second to avoid division by zero or negative
if total_seconds <= 0:
logger.warning(f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks.")
logger.warning(
f"[calculate_schedule] Window size is zero or negative. Falling back to start time for all {n} tasks."
)
return [start_dt] * n
# If window is too small for even distribution (e.g. 10 tasks in 5 seconds),
# we still distribute them but they might be very close.
# The requirement says "near-even spacing".
if n == 1:
return [start_dt]
interval = total_seconds / (n - 1) if n > 1 else 0
# If interval is too small (e.g. < 1s), we might want a fallback,
# but the spec says "handle too-small windows with explicit fallback/warning".
if interval < 1:
logger.warning(f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated.")
logger.warning(
f"[calculate_schedule] Window too small for {n} tasks (interval {interval:.2f}s). Tasks will be highly concentrated."
)
scheduled_times = []
for i in range(n):
scheduled_times.append(start_dt + timedelta(seconds=i * interval))
return scheduled_times
# [/DEF:calculate_schedule:Function]
# [/DEF:ThrottledSchedulerConfigurator:Class]
# [/DEF:SchedulerModule:Module]
# [/DEF:SchedulerModule:Module]

39
backend/src/core/task_manager/context.py
View File

@@ -2,9 +2,14 @@
# @SEMANTICS: task, context, plugin, execution, logger
# @PURPOSE: Provides execution context passed to plugins during task execution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins
# @RELATION: DEPENDS_ON -> [TaskLoggerModule]
# @RELATION: USED_BY -> [TaskManager]
# @COMPLEXITY: 5
# @INVARIANT: Each TaskContext is bound to a single task execution.
# @PRE: Task execution pipeline provides valid task identifiers, logging callbacks, and parameter dictionaries.
# @POST: Plugins receive context instances with stable logger and parameter accessors.
# @SIDE_EFFECT: Creates task-scoped logger wrappers and carries optional background task handles across sub-contexts.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# [SECTION: IMPORTS]
# [SECTION: IMPORTS]
@@ -13,11 +18,17 @@ from .task_logger import TaskLogger
from ..logger import belief_scope
# [/SECTION]
# [DEF:TaskContext:Class]
# @SEMANTICS: context, task, execution, plugin
# @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities.
# @COMPLEXITY: 5
# @INVARIANT: logger is always a valid TaskLogger instance.
# @PRE: Constructor receives non-empty task_id, callable add_log_fn, and params mapping.
# @POST: Instance exposes immutable task identity with logger, params, and optional background task access.
# @RELATION: DEPENDS_ON -> [TaskLogger]
# @SIDE_EFFECT: Emits structured task logs through TaskLogger on plugin interactions.
# @DATA_CONTRACT: Input[task_id, add_log_fn, params, default_source, background_tasks] -> Output[TaskContext]
# @UX_STATE: Idle -> Active -> Complete
#
# @TEST_CONTRACT: TaskContextInit ->
@@ -36,7 +47,7 @@ from ..logger import belief_scope
class TaskContext:
"""
Execution context provided to plugins during task execution.
Usage:
def execute(params: dict, context: TaskContext = None):
if context:
@@ -44,7 +55,7 @@ class TaskContext:
context.logger.progress("Processing items", percent=50)
# ... plugin logic
"""
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskContext with task-specific resources.
# @PRE: task_id is a valid task identifier, add_log_fn is callable.
@@ -66,12 +77,11 @@ class TaskContext:
self._params = params
self._background_tasks = background_tasks
self._logger = TaskLogger(
task_id=task_id,
add_log_fn=add_log_fn,
source=default_source
task_id=task_id, add_log_fn=add_log_fn, source=default_source
)
# [/DEF:__init__:Function]
# [DEF:task_id:Function]
# @PURPOSE: Get the task ID.
# @PRE: TaskContext must be initialized.
@@ -81,8 +91,9 @@ class TaskContext:
def task_id(self) -> str:
with belief_scope("task_id"):
return self._task_id
# [/DEF:task_id:Function]
# [DEF:logger:Function]
# @PURPOSE: Get the TaskLogger instance for this context.
# @PRE: TaskContext must be initialized.
@@ -92,8 +103,9 @@ class TaskContext:
def logger(self) -> TaskLogger:
with belief_scope("logger"):
return self._logger
# [/DEF:logger:Function]
# [DEF:params:Function]
# @PURPOSE: Get the task parameters.
# @PRE: TaskContext must be initialized.
@@ -103,6 +115,7 @@ class TaskContext:
def params(self) -> Dict[str, Any]:
with belief_scope("params"):
return self._params
# [/DEF:params:Function]
# [DEF:background_tasks:Function]
@@ -113,8 +126,9 @@ class TaskContext:
def background_tasks(self) -> Optional[Any]:
with belief_scope("background_tasks"):
return self._background_tasks
# [/DEF:background_tasks:Function]
# [DEF:get_param:Function]
# @PURPOSE: Get a specific parameter value with optional default.
# @PRE: TaskContext must be initialized.
@@ -125,8 +139,9 @@ class TaskContext:
def get_param(self, key: str, default: Any = None) -> Any:
with belief_scope("get_param"):
return self._params.get(key, default)
# [/DEF:get_param:Function]
# [DEF:create_sub_context:Function]
# @PURPOSE: Create a sub-context with a different default source.
# @PRE: source is a non-empty string.
@@ -143,8 +158,10 @@ class TaskContext:
default_source=source,
background_tasks=self._background_tasks,
)
# [/DEF:create_sub_context:Function]
# [/DEF:TaskContext:Class]
# [/DEF:TaskContextModule:Module]

242
backend/src/core/task_manager/manager.py
View File

@@ -8,8 +8,9 @@
# @POST: Orchestrates task execution and persistence.
# @SIDE_EFFECT: Spawns worker threads and flushes logs to DB.
# @DATA_CONTRACT: Input[plugin_id, params] -> Model[Task, LogEntry]
# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceModule:Module]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @INVARIANT: Task IDs are unique.
# @CONSTRAINT: Must use belief_scope for logging.
# @TEST_CONTRACT: TaskManagerModule -> {
@@ -38,15 +39,16 @@ from .context import TaskContext
from ..logger import logger, belief_scope, should_log_task_level
# [/SECTION]
# [DEF:TaskManager:Class]
# @TIER: CRITICAL
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
# @LAYER: Core
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService:Class]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService:Class]
# @RELATION: [DEPENDS_ON] ->[PluginLoader:Class]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
@@ -56,42 +58,49 @@ class TaskManager:
"""
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
"""
# Log flush interval in seconds
LOG_FLUSH_INTERVAL = 2.0
# [DEF:__init__:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
# @PRE: plugin_loader is initialized.
# @POST: TaskManager is ready to accept tasks.
# @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @PARAM: plugin_loader - The plugin loader instance.
def __init__(self, plugin_loader):
with belief_scope("TaskManager.__init__"):
self.plugin_loader = plugin_loader
self.tasks: Dict[str, Task] = {}
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
self.executor = ThreadPoolExecutor(
max_workers=5
) # For CPU-bound plugin execution
self.persistence_service = TaskPersistenceService()
self.log_persistence_service = TaskLogPersistenceService()
# Log buffer: task_id -> List[LogEntry]
self._log_buffer: Dict[str, List[LogEntry]] = {}
self._log_buffer_lock = threading.Lock()
# Flusher thread for batch writing logs
self._flusher_stop_event = threading.Event()
self._flusher_thread = threading.Thread(target=self._flusher_loop, daemon=True)
self._flusher_thread = threading.Thread(
target=self._flusher_loop, daemon=True
)
self._flusher_thread.start()
try:
self.loop = asyncio.get_running_loop()
except RuntimeError:
self.loop = asyncio.get_event_loop()
self.task_futures: Dict[str, asyncio.Future] = {}
# Load persisted tasks on startup
self.load_persisted_tasks()
# [/DEF:__init__:Function]
# [DEF:_flusher_loop:Function]
@@ -99,11 +108,13 @@ class TaskManager:
# @PURPOSE: Background thread that periodically flushes log buffer to database.
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION: [CALLS] ->[TaskManager._flush_logs]
def _flusher_loop(self):
"""Background thread that flushes log buffer to database."""
while not self._flusher_stop_event.is_set():
self._flush_logs()
self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)
# [/DEF:_flusher_loop:Function]
# [DEF:_flush_logs:Function]
@@ -111,15 +122,16 @@ class TaskManager:
# @PURPOSE: Flush all buffered logs to the database.
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
def _flush_logs(self):
"""Flush all buffered logs to the database."""
with self._log_buffer_lock:
task_ids = list(self._log_buffer.keys())
for task_id in task_ids:
with self._log_buffer_lock:
logs = self._log_buffer.pop(task_id, [])
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
@@ -131,6 +143,7 @@ class TaskManager:
if task_id not in self._log_buffer:
self._log_buffer[task_id] = []
self._log_buffer[task_id].extend(logs)
# [/DEF:_flush_logs:Function]
# [DEF:_flush_task_logs:Function]
@@ -139,17 +152,19 @@ class TaskManager:
# @PRE: task_id exists.
# @POST: Task's buffered logs are written to database.
# @PARAM: task_id (str) - The task ID.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
def _flush_task_logs(self, task_id: str):
"""Flush logs for a specific task immediately."""
with belief_scope("_flush_task_logs"):
with self._log_buffer_lock:
logs = self._log_buffer.pop(task_id, [])
if logs:
try:
self.log_persistence_service.add_logs(task_id, logs)
except Exception as e:
logger.error(f"Failed to flush logs for task {task_id}: {e}")
# [/DEF:_flush_task_logs:Function]
# [DEF:create_task:Function]
@@ -162,24 +177,30 @@ class TaskManager:
# @PARAM: user_id (Optional[str]) - ID of the user requesting the task.
# @RETURN: Task - The created task instance.
# @THROWS: ValueError if plugin not found or params invalid.
async def create_task(self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None) -> Task:
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def create_task(
self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None
) -> Task:
with belief_scope("TaskManager.create_task", f"plugin_id={plugin_id}"):
if not self.plugin_loader.has_plugin(plugin_id):
logger.error(f"Plugin with ID '{plugin_id}' not found.")
raise ValueError(f"Plugin with ID '{plugin_id}' not found.")
self.plugin_loader.get_plugin(plugin_id)
if not isinstance(params, dict):
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
self.tasks[task.id] = task
self.persistence_service.persist_task(task)
logger.info(f"Task {task.id} created and scheduled for execution")
self.loop.create_task(self._run_task(task.id)) # Schedule task for execution
self.loop.create_task(
self._run_task(task.id)
) # Schedule task for execution
return task
# [/DEF:create_task:Function]
# [DEF:_run_task:Function]
@@ -188,25 +209,33 @@ class TaskManager:
# @PRE: Task exists in registry.
# @POST: Task is executed, status updated to SUCCESS or FAILED.
# @PARAM: task_id (str) - The ID of the task to run.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def _run_task(self, task_id: str):
with belief_scope("TaskManager._run_task", f"task_id={task_id}"):
task = self.tasks[task_id]
plugin = self.plugin_loader.get_plugin(task.plugin_id)
logger.info(f"Starting execution of task {task_id} for plugin '{plugin.name}'")
logger.info(
f"Starting execution of task {task_id} for plugin '{plugin.name}'"
)
task.status = TaskStatus.RUNNING
task.started_at = datetime.utcnow()
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'", source="system")
self._add_log(
task_id,
"INFO",
f"Task started for plugin '{plugin.name}'",
source="system",
)
try:
# Prepare params and check if plugin supports new TaskContext
params = {**task.params, "_task_id": task_id}
# Check if plugin's execute method accepts 'context' parameter
sig = inspect.signature(plugin.execute)
accepts_context = 'context' in sig.parameters
accepts_context = "context" in sig.parameters
if accepts_context:
# Create TaskContext for new-style plugins
context = TaskContext(
@@ -216,13 +245,13 @@ class TaskManager:
default_source="plugin",
background_tasks=None,
)
if asyncio.iscoroutinefunction(plugin.execute):
task.result = await plugin.execute(params, context=context)
else:
task.result = await self.loop.run_in_executor(
self.executor,
lambda: plugin.execute(params, context=context)
lambda: plugin.execute(params, context=context),
)
else:
# Backward compatibility: old-style plugins without context
@@ -230,24 +259,36 @@ class TaskManager:
task.result = await plugin.execute(params)
else:
task.result = await self.loop.run_in_executor(
self.executor,
plugin.execute,
params
self.executor, plugin.execute, params
)
logger.info(f"Task {task_id} completed successfully")
task.status = TaskStatus.SUCCESS
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'", source="system")
self._add_log(
task_id,
"INFO",
f"Task completed successfully for plugin '{plugin.name}'",
source="system",
)
except Exception as e:
logger.error(f"Task {task_id} failed: {e}")
task.status = TaskStatus.FAILED
self._add_log(task_id, "ERROR", f"Task failed: {e}", source="system", metadata={"error_type": type(e).__name__})
self._add_log(
task_id,
"ERROR",
f"Task failed: {e}",
source="system",
metadata={"error_type": type(e).__name__},
)
finally:
task.finished_at = datetime.utcnow()
# Flush any remaining buffered logs before persisting task
self._flush_task_logs(task_id)
self.persistence_service.persist_task(task)
logger.info(f"Task {task_id} execution finished with status: {task.status}")
logger.info(
f"Task {task_id} execution finished with status: {task.status}"
)
# [/DEF:_run_task:Function]
# [DEF:resolve_task:Function]
@@ -258,21 +299,23 @@ class TaskManager:
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.
# @THROWS: ValueError if task not found or not awaiting mapping.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):
with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task or task.status != TaskStatus.AWAITING_MAPPING:
raise ValueError("Task is not awaiting mapping.")
# Update task params with resolution
task.params.update(resolution_params)
task.status = TaskStatus.RUNNING
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task resumed after mapping resolution.")
# Signal the future to continue
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# [/DEF:resolve_task:Function]
# [DEF:wait_for_resolution:Function]
@@ -281,21 +324,23 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
async def wait_for_resolution(self, task_id: str):
with belief_scope("TaskManager.wait_for_resolution", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
return
task.status = TaskStatus.AWAITING_MAPPING
self.persistence_service.persist_task(task)
self.task_futures[task_id] = self.loop.create_future()
try:
await self.task_futures[task_id]
finally:
if task_id in self.task_futures:
del self.task_futures[task_id]
# [/DEF:wait_for_resolution:Function]
# [DEF:wait_for_input:Function]
@@ -304,20 +349,22 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[asyncio.AbstractEventLoop.create_future]
async def wait_for_input(self, task_id: str):
with belief_scope("TaskManager.wait_for_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
return
# Status is already set to AWAITING_INPUT by await_input()
self.task_futures[task_id] = self.loop.create_future()
try:
await self.task_futures[task_id]
finally:
if task_id in self.task_futures:
del self.task_futures[task_id]
# [/DEF:wait_for_input:Function]
# [DEF:get_task:Function]
@@ -327,9 +374,11 @@ class TaskManager:
# @POST: Returns Task object or None.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: Optional[Task] - The task or None.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_task(self, task_id: str) -> Optional[Task]:
with belief_scope("TaskManager.get_task", f"task_id={task_id}"):
return self.tasks.get(task_id)
# [/DEF:get_task:Function]
# [DEF:get_all_tasks:Function]
@@ -338,9 +387,11 @@ class TaskManager:
# @PRE: None.
# @POST: Returns list of all Task objects.
# @RETURN: List[Task] - All tasks.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_all_tasks(self) -> List[Task]:
with belief_scope("TaskManager.get_all_tasks"):
return list(self.tasks.values())
# [/DEF:get_all_tasks:Function]
# [DEF:get_tasks:Function]
@@ -352,13 +403,14 @@ class TaskManager:
# @PARAM: offset (int) - Number of tasks to skip.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: List[Task] - List of tasks matching criteria.
# @RELATION: [READS] ->[TaskManager.tasks]
def get_tasks(
self,
limit: int = 10,
offset: int = 0,
status: Optional[TaskStatus] = None,
plugin_ids: Optional[List[str]] = None,
completed_only: bool = False
completed_only: bool = False,
) -> List[Task]:
with belief_scope("TaskManager.get_tasks"):
tasks = list(self.tasks.values())
@@ -368,7 +420,10 @@ class TaskManager:
plugin_id_set = set(plugin_ids)
tasks = [t for t in tasks if t.plugin_id in plugin_id_set]
if completed_only:
tasks = [t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]]
tasks = [
t for t in tasks if t.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]
]
# Sort by started_at descending with tolerant handling of mixed tz-aware/naive values.
def sort_key(task: Task) -> float:
started_at = task.started_at
@@ -381,7 +436,8 @@ class TaskManager:
return started_at.timestamp()
tasks.sort(key=sort_key, reverse=True)
return tasks[offset:offset + limit]
return tasks[offset : offset + limit]
# [/DEF:get_tasks:Function]
# [DEF:get_task_logs:Function]
@@ -392,10 +448,13 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
# @RETURN: List[LogEntry] - List of log entries.
def get_task_logs(self, task_id: str, log_filter: Optional[LogFilter] = None) -> List[LogEntry]:
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]
def get_task_logs(
self, task_id: str, log_filter: Optional[LogFilter] = None
) -> List[LogEntry]:
with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"):
task = self.tasks.get(task_id)
# For completed tasks, fetch from persistence
if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:
if log_filter is None:
@@ -408,15 +467,16 @@ class TaskManager:
level=log.level,
message=log.message,
source=log.source,
metadata=log.metadata
metadata=log.metadata,
)
for log in task_logs
]
# For running/pending tasks, return from memory
return task.logs if task else []
# [/DEF:get_task_logs:Function]
# [DEF:get_task_log_stats:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get statistics about logs for a task.
@@ -424,11 +484,13 @@ class TaskManager:
# @POST: Returns LogStats with counts by level and source.
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]
def get_task_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
return self.log_persistence_service.get_log_stats(task_id)
# [/DEF:get_task_log_stats:Function]
# [DEF:get_task_log_sources:Function]
# @COMPLEXITY: 3
# @PURPOSE: Get unique sources for a task's logs.
@@ -436,9 +498,11 @@ class TaskManager:
# @POST: Returns list of unique source strings.
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]
def get_task_log_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
return self.log_persistence_service.get_sources(task_id)
# [/DEF:get_task_log_sources:Function]
# [DEF:_add_log:Function]
@@ -452,6 +516,7 @@ class TaskManager:
# @PARAM: source (str) - Source component (default: "system").
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
# @RELATION: [CALLS] ->[should_log_task_level]
def _add_log(
self,
task_id: str,
@@ -459,7 +524,7 @@ class TaskManager:
message: str,
source: str = "system",
metadata: Optional[Dict[str, Any]] = None,
context: Optional[Dict[str, Any]] = None
context: Optional[Dict[str, Any]] = None,
):
with belief_scope("TaskManager._add_log", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -476,12 +541,12 @@ class TaskManager:
message=message,
source=source,
metadata=metadata,
context=context # Keep for backward compatibility
context=context, # Keep for backward compatibility
)
# Add to in-memory logs (for backward compatibility with legacy JSON field)
task.logs.append(log_entry)
# Add to buffer for batch persistence
with self._log_buffer_lock:
if task_id not in self._log_buffer:
@@ -492,6 +557,7 @@ class TaskManager:
if task_id in self.subscribers:
for queue in self.subscribers[task_id]:
self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)
# [/DEF:_add_log:Function]
# [DEF:subscribe_logs:Function]
@@ -501,6 +567,7 @@ class TaskManager:
# @POST: Returns an asyncio.Queue for log entries.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: asyncio.Queue - Queue for log entries.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
with belief_scope("TaskManager.subscribe_logs", f"task_id={task_id}"):
queue = asyncio.Queue()
@@ -508,6 +575,7 @@ class TaskManager:
self.subscribers[task_id] = []
self.subscribers[task_id].append(queue)
return queue
# [/DEF:subscribe_logs:Function]
# [DEF:unsubscribe_logs:Function]
@@ -517,6 +585,7 @@ class TaskManager:
# @POST: Queue removed from subscribers.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: queue (asyncio.Queue) - Queue to remove.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
with belief_scope("TaskManager.unsubscribe_logs", f"task_id={task_id}"):
if task_id in self.subscribers:
@@ -524,6 +593,7 @@ class TaskManager:
self.subscribers[task_id].remove(queue)
if not self.subscribers[task_id]:
del self.subscribers[task_id]
# [/DEF:unsubscribe_logs:Function]
# [DEF:load_persisted_tasks:Function]
@@ -531,12 +601,14 @@ class TaskManager:
# @PURPOSE: Load persisted tasks using persistence service.
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
def load_persisted_tasks(self) -> None:
with belief_scope("TaskManager.load_persisted_tasks"):
loaded_tasks = self.persistence_service.load_tasks(limit=100)
for task in loaded_tasks:
if task.id not in self.tasks:
self.tasks[task.id] = task
# [/DEF:load_persisted_tasks:Function]
# [DEF:await_input:Function]
@@ -547,19 +619,28 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: input_request (Dict) - Details about required input.
# @THROWS: ValueError if task not found or not RUNNING.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:
with belief_scope("TaskManager.await_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
if not task:
raise ValueError(f"Task {task_id} not found")
if task.status != TaskStatus.RUNNING:
raise ValueError(f"Task {task_id} is not RUNNING (current: {task.status})")
raise ValueError(
f"Task {task_id} is not RUNNING (current: {task.status})"
)
task.status = TaskStatus.AWAITING_INPUT
task.input_required = True
task.input_request = input_request
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task paused for user input", {"input_request": input_request})
self._add_log(
task_id,
"INFO",
"Task paused for user input",
{"input_request": input_request},
)
# [/DEF:await_input:Function]
# [DEF:resume_task_with_password:Function]
@@ -570,26 +651,39 @@ class TaskManager:
# @PARAM: task_id (str) - ID of the task.
# @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.
# @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.
def resume_task_with_password(self, task_id: str, passwords: Dict[str, str]) -> None:
with belief_scope("TaskManager.resume_task_with_password", f"task_id={task_id}"):
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
def resume_task_with_password(
self, task_id: str, passwords: Dict[str, str]
) -> None:
with belief_scope(
"TaskManager.resume_task_with_password", f"task_id={task_id}"
):
task = self.tasks.get(task_id)
if not task:
raise ValueError(f"Task {task_id} not found")
if task.status != TaskStatus.AWAITING_INPUT:
raise ValueError(f"Task {task_id} is not AWAITING_INPUT (current: {task.status})")
raise ValueError(
f"Task {task_id} is not AWAITING_INPUT (current: {task.status})"
)
if not isinstance(passwords, dict) or not passwords:
raise ValueError("Passwords must be a non-empty dictionary")
task.params["passwords"] = passwords
task.input_required = False
task.input_request = None
task.status = TaskStatus.RUNNING
self.persistence_service.persist_task(task)
self._add_log(task_id, "INFO", "Task resumed with passwords", {"databases": list(passwords.keys())})
self._add_log(
task_id,
"INFO",
"Task resumed with passwords",
{"databases": list(passwords.keys())},
)
if task_id in self.task_futures:
self.task_futures[task_id].set_result(True)
# [/DEF:resume_task_with_password:Function]
# [DEF:clear_tasks:Function]
@@ -599,6 +693,7 @@ class TaskManager:
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: int - Number of tasks cleared.
# @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]
def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:
with belief_scope("TaskManager.clear_tasks"):
tasks_to_remove = []
@@ -607,16 +702,20 @@ class TaskManager:
# If status is None, match everything EXCEPT RUNNING (unless they are awaiting input/mapping which are technically running but paused?)
# Actually, AWAITING_INPUT and AWAITING_MAPPING are distinct statuses in TaskStatus enum.
# RUNNING is active execution.
should_remove = False
if status:
if task.status == status:
should_remove = True
else:
# Clear all non-active tasks (keep RUNNING, AWAITING_INPUT, AWAITING_MAPPING)
if task.status not in [TaskStatus.RUNNING, TaskStatus.AWAITING_INPUT, TaskStatus.AWAITING_MAPPING]:
if task.status not in [
TaskStatus.RUNNING,
TaskStatus.AWAITING_INPUT,
TaskStatus.AWAITING_MAPPING,
]:
should_remove = True
if should_remove:
tasks_to_remove.append(task_id)
@@ -625,18 +724,21 @@ class TaskManager:
if tid in self.task_futures:
self.task_futures[tid].cancel()
del self.task_futures[tid]
del self.tasks[tid]
# Remove from persistence (task_records and task_logs via CASCADE)
self.persistence_service.delete_tasks(tasks_to_remove)
# Also explicitly delete logs (in case CASCADE is not set up)
if tasks_to_remove:
self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)
logger.info(f"Cleared {len(tasks_to_remove)} tasks.")
return len(tasks_to_remove)
# [/DEF:clear_tasks:Function]
# [/DEF:TaskManager:Class]
# [/DEF:TaskManager:Module]

56
backend/src/core/task_manager/models.py
View File

@@ -3,7 +3,8 @@
# @SEMANTICS: task, models, pydantic, enum, state
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
# @LAYER: Core
# @RELATION: Used by TaskManager and API routes.
# @RELATION: [USED_BY] -> [TaskManager]
# @RELATION: [USED_BY] -> [ApiRoutes]
# @INVARIANT: Task IDs are immutable once created.
# @CONSTRAINT: Must use Pydantic for data validation.
@@ -16,6 +17,7 @@ from typing import Dict, Any, List, Optional
from pydantic import BaseModel, Field
# [/SECTION]
# [DEF:TaskStatus:Enum]
# @COMPLEXITY: 1
# @SEMANTICS: task, status, state, enum
@@ -27,23 +29,33 @@ class TaskStatus(str, Enum):
FAILED = "FAILED"
AWAITING_MAPPING = "AWAITING_MAPPING"
AWAITING_INPUT = "AWAITING_INPUT"
# [/DEF:TaskStatus:Enum]
# [DEF:LogLevel:Enum]
# @SEMANTICS: log, level, severity, enum
# @PURPOSE: Defines the possible log levels for task logging.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskLogger]
class LogLevel(str, Enum):
DEBUG = "DEBUG"
INFO = "INFO"
WARNING = "WARNING"
ERROR = "ERROR"
# [/DEF:LogLevel:Enum]
# [DEF:LogEntry:Class]
# @SEMANTICS: log, entry, record, pydantic
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
# @COMPLEXITY: 5
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [LogLevel]
# @RELATION: [PART_OF] -> [Task]
# @INVARIANT: Each log entry has a unique timestamp and source.
#
# @TEST_CONTRACT: LogEntryModel ->
@@ -57,16 +69,25 @@ class LogEntry(BaseModel):
timestamp: datetime = Field(default_factory=datetime.utcnow)
level: str = Field(default="INFO")
message: str
source: str = Field(default="system") # Component attribution: plugin, superset_api, git, etc.
context: Optional[Dict[str, Any]] = None # Legacy field, kept for backward compatibility
metadata: Optional[Dict[str, Any]] = None # Structured metadata (e.g., dashboard_id, progress)
source: str = Field(
default="system"
) # Component attribution: plugin, superset_api, git, etc.
context: Optional[Dict[str, Any]] = (
None # Legacy field, kept for backward compatibility
)
metadata: Optional[Dict[str, Any]] = (
None # Structured metadata (e.g., dashboard_id, progress)
)
# [/DEF:LogEntry:Class]
# [DEF:TaskLog:Class]
# @SEMANTICS: task, log, persistent, pydantic
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
# @COMPLEXITY: 3
# @RELATION: MAPS_TO -> TaskLogRecord
# @RELATION: [MAPS_TO] -> [TaskLogRecord]
class TaskLog(BaseModel):
id: int
task_id: str
@@ -78,34 +99,48 @@ class TaskLog(BaseModel):
class Config:
from_attributes = True
# [/DEF:TaskLog:Class]
# [DEF:LogFilter:Class]
# @SEMANTICS: log, filter, query, pydantic
# @PURPOSE: Filter parameters for querying task logs.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [USED_BY] -> [TaskManager]
class LogFilter(BaseModel):
level: Optional[str] = None # Filter by log level
source: Optional[str] = None # Filter by source component
search: Optional[str] = None # Text search in message
offset: int = Field(default=0, ge=0)
limit: int = Field(default=100, ge=1, le=1000)
# [/DEF:LogFilter:Class]
# [DEF:LogStats:Class]
# @SEMANTICS: log, stats, aggregation, pydantic
# @PURPOSE: Statistics about log entries for a task.
# @COMPLEXITY: 3
# @COMPLEXITY: 1
# @RELATION: [COMPUTED_FROM] -> [TaskLog]
class LogStats(BaseModel):
total_count: int
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7}
# [/DEF:LogStats:Class]
# [DEF:Task:Class]
# @COMPLEXITY: 3
# @SEMANTICS: task, job, execution, state, pydantic
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
# @RELATION: [DEPENDS_ON] -> [TaskStatus]
# @RELATION: [CONTAINS] -> [LogEntry]
# @RELATION: [USED_BY] -> [TaskManager]
class Task(BaseModel):
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
plugin_id: str
@@ -129,7 +164,10 @@ class Task(BaseModel):
super().__init__(**data)
if self.status == TaskStatus.AWAITING_INPUT and not self.input_request:
raise ValueError("input_request is required when status is AWAITING_INPUT")
# [/DEF:__init__:Function]
# [/DEF:Task:Class]
# [/DEF:TaskManagerModels:Module]

17
backend/src/core/task_manager/persistence.py
View File

@@ -7,7 +7,7 @@
# @POST: Provides reliable storage and retrieval for task metadata and logs.
# @SIDE_EFFECT: Performs database I/O on tasks.db.
# @DATA_CONTRACT: Input[Task, LogEntry] -> Model[TaskRecord, TaskLogRecord]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @INVARIANT: Database schema must match the TaskRecord model structure.
@@ -36,7 +36,7 @@ from ..logger import logger, belief_scope
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
# @RELATION: [DEPENDS_ON] ->[Environment]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @INVARIANT: Persistence must handle potentially missing task fields natively.
#
# @TEST_CONTRACT: TaskPersistenceService ->
@@ -100,6 +100,7 @@ class TaskPersistenceService:
# @PRE: Session is active
# @POST: Returns existing environments.id or None when unresolved.
# @DATA_CONTRACT: Input[env_id: Optional[str]] -> Output[Optional[str]]
# @RELATION: [DEPENDS_ON] ->[Environment]
@staticmethod
def _resolve_environment_id(session: Session, env_id: Optional[str]) -> Optional[str]:
with belief_scope("_resolve_environment_id"):
@@ -287,6 +288,7 @@ class TaskPersistenceService:
# @POST: Specified task records deleted from database.
# @PARAM: task_ids (List[str]) - List of task IDs to delete.
# @SIDE_EFFECT: Deletes rows from task_records table.
# @RELATION: [DEPENDS_ON] ->[TaskRecord]
def delete_tasks(self, task_ids: List[str]) -> None:
if not task_ids:
return
@@ -313,7 +315,7 @@ class TaskPersistenceService:
# @DATA_CONTRACT: Input[task_id:str, logs:List[LogEntry], log_filter:LogFilter, task_ids:List[str]] -> Model[TaskLogRecord] -> Output[None | List[TaskLog] | LogStats | List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TasksSessionLocal]
# @RELATION: [USED_BY] ->[backend.src.core.task_manager.manager.TaskManager]
# @RELATION: [USED_BY] ->[TaskManager]
# @INVARIANT: Log entries are batch-inserted for performance.
#
# @TEST_CONTRACT: TaskLogPersistenceService ->
@@ -352,6 +354,7 @@ class TaskLogPersistenceService:
# @PARAM: logs (List[LogEntry]) - Log entries to insert.
# @SIDE_EFFECT: Writes to task_logs table.
# @DATA_CONTRACT: Input[List[LogEntry]] -> Model[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:
if not logs:
return
@@ -385,6 +388,9 @@ class TaskLogPersistenceService:
# @PARAM: log_filter (LogFilter) - Filter parameters.
# @RETURN: List[TaskLog] - Filtered log entries.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[TaskLog]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[LogFilter]
# @RELATION: [DEPENDS_ON] ->[TaskLog]
def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -438,6 +444,8 @@ class TaskLogPersistenceService:
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[LogStats]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
# @RELATION: [DEPENDS_ON] ->[LogStats]
def get_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -485,6 +493,7 @@ class TaskLogPersistenceService:
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @DATA_CONTRACT: Model[TaskLogRecord] -> Output[List[str]]
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def get_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -505,6 +514,7 @@ class TaskLogPersistenceService:
# @POST: All logs for the task are deleted.
# @PARAM: task_id (str) - The task ID.
# @SIDE_EFFECT: Deletes from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def delete_logs_for_task(self, task_id: str) -> None:
with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"):
session: Session = TasksSessionLocal()
@@ -527,6 +537,7 @@ class TaskLogPersistenceService:
# @POST: All logs for the tasks are deleted.
# @PARAM: task_ids (List[str]) - List of task IDs.
# @SIDE_EFFECT: Deletes rows from task_logs table.
# @RELATION: [DEPENDS_ON] ->[TaskLogRecord]
def delete_logs_for_tasks(self, task_ids: List[str]) -> None:
if not task_ids:
return

90
backend/src/core/task_manager/task_logger.py
View File

@@ -2,18 +2,23 @@
# @SEMANTICS: task, logger, context, plugin, attribution
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
# @LAYER: Core
# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log
# @COMPLEXITY: 5
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @COMPLEXITY: 2
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
# [SECTION: IMPORTS]
from typing import Dict, Any, Optional, Callable
# [/SECTION]
# [DEF:TaskLogger:Class]
# @SEMANTICS: logger, task, source, attribution
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
# @COMPLEXITY: 5
# @COMPLEXITY: 2
# @RELATION: [DEPENDS_ON] -> [TaskManager]
# @RELATION: [CALLS] -> [_add_log]
# @RELATION: [USED_BY] -> [TaskManager]
# @INVARIANT: All log calls include the task_id and source.
# @UX_STATE: Idle -> Logging -> (system records log)
#
@@ -33,17 +38,17 @@ from typing import Dict, Any, Optional, Callable
class TaskLogger:
"""
A dedicated logger for tasks that automatically tags logs with source attribution.
Usage:
logger = TaskLogger(task_id="abc123", add_log_fn=task_manager._add_log, source="plugin")
logger.info("Starting backup process")
logger.error("Failed to connect", metadata={"error_code": 500})
# Create sub-logger with different source
api_logger = logger.with_source("superset_api")
api_logger.info("Fetching dashboards")
"""
# [DEF:__init__:Function]
# @PURPOSE: Initialize the TaskLogger with task context.
# @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
@@ -51,17 +56,13 @@ class TaskLogger:
# @PARAM: task_id (str) - The ID of the task.
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
# @PARAM: source (str) - Default source for logs (default: "plugin").
def __init__(
self,
task_id: str,
add_log_fn: Callable,
source: str = "plugin"
):
def __init__(self, task_id: str, add_log_fn: Callable, source: str = "plugin"):
self._task_id = task_id
self._add_log = add_log_fn
self._default_source = source
# [/DEF:__init__:Function]
# [DEF:with_source:Function]
# @PURPOSE: Create a sub-logger with a different default source.
# @PRE: source is a non-empty string.
@@ -71,12 +72,11 @@ class TaskLogger:
def with_source(self, source: str) -> "TaskLogger":
"""Create a sub-logger with a different source context."""
return TaskLogger(
task_id=self._task_id,
add_log_fn=self._add_log,
source=source
task_id=self._task_id, add_log_fn=self._add_log, source=source
)
# [/DEF:with_source:Function]
# [DEF:_log:Function]
# @PURPOSE: Internal method to log a message at a given level.
# @PRE: level is a valid log level string.
@@ -87,11 +87,11 @@ class TaskLogger:
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @UX_STATE: Logging -> (writing internal log)
def _log(
self,
level: str,
message: str,
self,
level: str,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Internal logging method."""
self._add_log(
@@ -99,10 +99,11 @@ class TaskLogger:
level=level,
message=message,
source=source or self._default_source,
metadata=metadata
metadata=metadata,
)
# [/DEF:_log:Function]
# [DEF:debug:Function]
# @PURPOSE: Log a DEBUG level message.
# @PRE: message is a string.
@@ -111,14 +112,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def debug(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("DEBUG", message, source, metadata)
# [/DEF:debug:Function]
# [DEF:info:Function]
# @PURPOSE: Log an INFO level message.
# @PRE: message is a string.
@@ -127,14 +129,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def info(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("INFO", message, source, metadata)
# [/DEF:info:Function]
# [DEF:warning:Function]
# @PURPOSE: Log a WARNING level message.
# @PRE: message is a string.
@@ -143,14 +146,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def warning(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("WARNING", message, source, metadata)
# [/DEF:warning:Function]
# [DEF:error:Function]
# @PURPOSE: Log an ERROR level message.
# @PRE: message is a string.
@@ -159,14 +163,15 @@ class TaskLogger:
# @PARAM: source (Optional[str]) - Override source.
# @PARAM: metadata (Optional[Dict]) - Additional data.
def error(
self,
message: str,
self,
message: str,
source: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None,
) -> None:
self._log("ERROR", message, source, metadata)
# [/DEF:error:Function]
# [DEF:progress:Function]
# @PURPOSE: Log a progress update with percentage.
# @PRE: percent is between 0 and 100.
@@ -175,16 +180,15 @@ class TaskLogger:
# @PARAM: percent (float) - Progress percentage (0-100).
# @PARAM: source (Optional[str]) - Override source.
def progress(
self,
message: str,
percent: float,
source: Optional[str] = None
self, message: str, percent: float, source: Optional[str] = None
) -> None:
"""Log a progress update with percentage."""
metadata = {"progress": min(100, max(0, percent))}
self._log("INFO", message, source, metadata)
# [/DEF:progress:Function]
# [/DEF:TaskLogger:Class]
# [/DEF:TaskLoggerModule:Module]