busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

semantics

Browse Source
This commit is contained in:
busya
2026-03-27 21:27:31 +03:00
parent 202823aba5
commit 7901ce0f39
182 changed files with 21186 additions and 10254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
backend/src/core/task_manager/manager.py
View File

@@ -1,4 +1,4 @@
# [DEF:TaskManager:Module]
# [DEF:TaskManagerModule:Module]
# @COMPLEXITY: 5
# @SEMANTICS: task, manager, lifecycle, execution, state
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
@@ -10,9 +10,13 @@
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @INVARIANT: Task IDs are unique.
# @CONSTRAINT: Must use belief_scope for logging.
# @TEST_CONTRACT: TaskManagerModule -> {
# @TEST_CONTRACT: TaskManagerRuntime -> {
# required_fields: {plugin_loader: PluginLoader},
# optional_fields: {},
# invariants: ["Must use belief_scope for logging"]
@@ -47,6 +51,12 @@ from ..logger import logger, belief_scope, should_log_task_level
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @PRE: Plugin loader resolves plugin ids and persistence services are available for task state hydration.
# @POST: In-memory task graph, lifecycle scheduler, and log event bus stay consistent with persisted task state.
# @INVARIANT: Task IDs are unique within the registry.
# @INVARIANT: Each task has exactly one status at any time.
# @INVARIANT: Log entries are never deleted after being added to a task.
@@ -60,6 +70,44 @@ class TaskManager:
# Log flush interval in seconds
LOG_FLUSH_INTERVAL = 2.0
# [DEF:TaskGraph:Block]
# @COMPLEXITY: 5
# @PURPOSE: Represents the in-memory task dependency graph spanning task registry nodes, paused futures, and persistence-backed hydration.
# @RELATION: [DEPENDS_ON] ->[Task]
# @RELATION: [DEPENDS_ON] ->[TaskPersistenceService]
# @PRE: Task ids are generated before insertion and persisted tasks can be reconstructed into Task models.
# @POST: Each live task id resolves to at most one active Task node and optional pause future.
# @SIDE_EFFECT: Mutates the in-memory task registry and loads persisted state during manager startup.
# @DATA_CONTRACT: Input[Task lifecycle events] -> Output[tasks:Dict[str, Task], task_futures:Dict[str, asyncio.Future]]
# @INVARIANT: Registry membership is keyed by unique task id and survives log streaming side channels.
# [/DEF:TaskGraph:Block]
# [DEF:EventBus:Block]
# @COMPLEXITY: 5
# @PURPOSE: Coordinates task-scoped log buffering, persistence flushes, and subscriber fan-out for real-time observers.
# @RELATION: [DEPENDS_ON] ->[LogEntry]
# @RELATION: [DEPENDS_ON] ->[TaskLogPersistenceService]
# @PRE: Task log records accept structured LogEntry payloads and subscribers consume asyncio queues.
# @POST: Accepted task logs are buffered, optionally persisted, and dispatched to active subscribers in emission order.
# @SIDE_EFFECT: Writes task logs to persistence, mutates in-memory buffers, and pushes log entries into subscriber queues.
# @DATA_CONTRACT: Input[task_id, LogEntry, Queue subscribers] -> Output[persisted log rows, streamed log events]
# @INVARIANT: Buffered logs are retried on persistence failure and every subscriber receives only task-scoped events.
# [/DEF:EventBus:Block]
# [DEF:JobLifecycle:Block]
# @COMPLEXITY: 5
# @PURPOSE: Encodes task creation, execution, pause/resume, and completion transitions for plugin-backed jobs.
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[PluginLoader]
# @PRE: Requested plugin ids resolve to executable plugins and task state transitions target known TaskStatus values.
# @POST: Every scheduled task transitions through a valid lifecycle path ending in persisted terminal or waiting state.
# @SIDE_EFFECT: Schedules async execution, pauses on input/mapping requests, and mutates persisted task lifecycle markers.
# @DATA_CONTRACT: Input[plugin_id, params, task_id, resolution payloads] -> Output[Task status transitions, task result]
# @INVARIANT: A task cannot be resumed from a waiting state unless a matching future exists or a new wait future is created.
# [/DEF:JobLifecycle:Block]
# [DEF:__init__:Function]
# @COMPLEXITY: 5
# @PURPOSE: Initialize the TaskManager with dependencies.
@@ -67,9 +115,12 @@ class TaskManager:
# @POST: TaskManager is ready to accept tasks.
# @SIDE_EFFECT: Starts background flusher thread and loads persisted task state into memory.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
# @PARAM: plugin_loader - The plugin loader instance.
def __init__(self, plugin_loader):
with belief_scope("TaskManager.__init__"):
logger.reason("Initializing task manager runtime services")
self.plugin_loader = plugin_loader
self.tasks: Dict[str, Task] = {}
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
@@ -98,6 +149,10 @@ class TaskManager:
# Load persisted tasks on startup
self.load_persisted_tasks()
logger.reflect(
"Task manager runtime initialized",
extra={"task_count": len(self.tasks)},
)
# [/DEF:__init__:Function]
@@ -107,6 +162,7 @@ class TaskManager:
# @PRE: TaskManager is initialized.
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
# @RELATION: [CALLS] ->[TaskManager._flush_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flusher_loop(self):
"""Background thread that flushes log buffer to database."""
while not self._flusher_stop_event.is_set():
@@ -121,6 +177,7 @@ class TaskManager:
# @PRE: None.
# @POST: All buffered logs are written to task_logs table.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flush_logs(self):
"""Flush all buffered logs to the database."""
with self._log_buffer_lock:
@@ -151,6 +208,7 @@ class TaskManager:
# @POST: Task's buffered logs are written to database.
# @PARAM: task_id (str) - The task ID.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.add_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def _flush_task_logs(self, task_id: str):
"""Flush logs for a specific task immediately."""
with belief_scope("_flush_task_logs"):
@@ -176,6 +234,8 @@ class TaskManager:
# @RETURN: Task - The created task instance.
# @THROWS: ValueError if plugin not found or params invalid.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
async def create_task(
self, plugin_id: str, params: Dict[str, Any], user_id: Optional[str] = None
) -> Task:
@@ -190,6 +250,7 @@ class TaskManager:
logger.error("Task parameters must be a dictionary.")
raise ValueError("Task parameters must be a dictionary.")
logger.reason("Creating task node and scheduling execution")
task = Task(plugin_id=plugin_id, params=params, user_id=user_id)
self.tasks[task.id] = task
self.persistence_service.persist_task(task)
@@ -197,6 +258,10 @@ class TaskManager:
self.loop.create_task(
self._run_task(task.id)
) # Schedule task for execution
logger.reflect(
"Task creation persisted and execution scheduled",
extra={"task_id": task.id, "plugin_id": plugin_id},
)
return task
# [/DEF:create_task:Function]
@@ -208,11 +273,18 @@ class TaskManager:
# @POST: Task is executed, status updated to SUCCESS or FAILED.
# @PARAM: task_id (str) - The ID of the task to run.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
# @RELATION: [DEPENDS_ON] ->[TaskContext]
# @RELATION: [DEPENDS_ON] ->[EventBus]
async def _run_task(self, task_id: str):
with belief_scope("TaskManager._run_task", f"task_id={task_id}"):
task = self.tasks[task_id]
plugin = self.plugin_loader.get_plugin(task.plugin_id)
logger.reason(
"Transitioning task to running state",
extra={"task_id": task_id, "plugin_id": task.plugin_id},
)
logger.info(
f"Starting execution of task {task_id} for plugin '{plugin.name}'"
)
@@ -286,6 +358,10 @@ class TaskManager:
logger.info(
f"Task {task_id} execution finished with status: {task.status}"
)
logger.reflect(
"Task lifecycle reached persisted terminal state",
extra={"task_id": task_id, "status": str(task.status)},
)
# [/DEF:_run_task:Function]
@@ -298,6 +374,7 @@ class TaskManager:
# @PARAM: resolution_params (Dict[str, Any]) - Params to resolve the wait.
# @THROWS: ValueError if task not found or not awaiting mapping.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def resolve_task(self, task_id: str, resolution_params: Dict[str, Any]):
with belief_scope("TaskManager.resolve_task", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -323,6 +400,7 @@ class TaskManager:
# @POST: Execution pauses until future is set.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def wait_for_resolution(self, task_id: str):
with belief_scope("TaskManager.wait_for_resolution", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -347,7 +425,7 @@ class TaskManager:
# @PRE: Task exists.
# @POST: Execution pauses until future is set via resume_task_with_password.
# @PARAM: task_id (str) - The ID of the task.
# @RELATION: [CALLS] ->[asyncio.AbstractEventLoop.create_future]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
async def wait_for_input(self, task_id: str):
with belief_scope("TaskManager.wait_for_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -372,7 +450,7 @@ class TaskManager:
# @POST: Returns Task object or None.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: Optional[Task] - The task or None.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_task(self, task_id: str) -> Optional[Task]:
with belief_scope("TaskManager.get_task", f"task_id={task_id}"):
return self.tasks.get(task_id)
@@ -385,7 +463,7 @@ class TaskManager:
# @PRE: None.
# @POST: Returns list of all Task objects.
# @RETURN: List[Task] - All tasks.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_all_tasks(self) -> List[Task]:
with belief_scope("TaskManager.get_all_tasks"):
return list(self.tasks.values())
@@ -401,7 +479,7 @@ class TaskManager:
# @PARAM: offset (int) - Number of tasks to skip.
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: List[Task] - List of tasks matching criteria.
# @RELATION: [READS] ->[TaskManager.tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def get_tasks(
self,
limit: int = 10,
@@ -447,6 +525,7 @@ class TaskManager:
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
# @RETURN: List[LogEntry] - List of log entries.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_logs]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_logs(
self, task_id: str, log_filter: Optional[LogFilter] = None
) -> List[LogEntry]:
@@ -483,6 +562,7 @@ class TaskManager:
# @PARAM: task_id (str) - The task ID.
# @RETURN: LogStats - Statistics about task logs.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_log_stats]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_log_stats(self, task_id: str) -> LogStats:
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
return self.log_persistence_service.get_log_stats(task_id)
@@ -497,6 +577,7 @@ class TaskManager:
# @PARAM: task_id (str) - The task ID.
# @RETURN: List[str] - Unique source names.
# @RELATION: [CALLS] ->[TaskLogPersistenceService.get_sources]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def get_task_log_sources(self, task_id: str) -> List[str]:
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
return self.log_persistence_service.get_sources(task_id)
@@ -515,6 +596,7 @@ class TaskManager:
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
# @RELATION: [CALLS] ->[should_log_task_level]
# @RELATION: [DISPATCHES] ->[EventBus]
def _add_log(
self,
task_id: str,
@@ -565,7 +647,7 @@ class TaskManager:
# @POST: Returns an asyncio.Queue for log entries.
# @PARAM: task_id (str) - ID of the task.
# @RETURN: asyncio.Queue - Queue for log entries.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
# @RELATION: [DEPENDS_ON] ->[EventBus]
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
with belief_scope("TaskManager.subscribe_logs", f"task_id={task_id}"):
queue = asyncio.Queue()
@@ -583,7 +665,7 @@ class TaskManager:
# @POST: Queue removed from subscribers.
# @PARAM: task_id (str) - ID of the task.
# @PARAM: queue (asyncio.Queue) - Queue to remove.
# @RELATION: [MUTATES] ->[TaskManager.subscribers]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
with belief_scope("TaskManager.unsubscribe_logs", f"task_id={task_id}"):
if task_id in self.subscribers:
@@ -600,6 +682,7 @@ class TaskManager:
# @PRE: None.
# @POST: Persisted tasks loaded into self.tasks.
# @RELATION: [CALLS] ->[TaskPersistenceService.load_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
def load_persisted_tasks(self) -> None:
with belief_scope("TaskManager.load_persisted_tasks"):
loaded_tasks = self.persistence_service.load_tasks(limit=100)
@@ -618,6 +701,7 @@ class TaskManager:
# @PARAM: input_request (Dict) - Details about required input.
# @THROWS: ValueError if task not found or not RUNNING.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
def await_input(self, task_id: str, input_request: Dict[str, Any]) -> None:
with belief_scope("TaskManager.await_input", f"task_id={task_id}"):
task = self.tasks.get(task_id)
@@ -636,7 +720,7 @@ class TaskManager:
task_id,
"INFO",
"Task paused for user input",
{"input_request": input_request},
metadata={"input_request": input_request},
)
# [/DEF:await_input:Function]
@@ -650,6 +734,7 @@ class TaskManager:
# @PARAM: passwords (Dict[str, str]) - Mapping of database name to password.
# @THROWS: ValueError if task not found, not awaiting input, or passwords invalid.
# @RELATION: [CALLS] ->[TaskPersistenceService.persist_task]
# @RELATION: [DEPENDS_ON] ->[JobLifecycle]
def resume_task_with_password(
self, task_id: str, passwords: Dict[str, str]
) -> None:
@@ -676,7 +761,7 @@ class TaskManager:
task_id,
"INFO",
"Task resumed with passwords",
{"databases": list(passwords.keys())},
metadata={"databases": list(passwords.keys())},
)
if task_id in self.task_futures:
@@ -692,6 +777,8 @@ class TaskManager:
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
# @RETURN: int - Number of tasks cleared.
# @RELATION: [CALLS] ->[TaskPersistenceService.delete_tasks]
# @RELATION: [DEPENDS_ON] ->[TaskGraph]
# @RELATION: [DEPENDS_ON] ->[EventBus]
def clear_tasks(self, status: Optional[TaskStatus] = None) -> int:
with belief_scope("TaskManager.clear_tasks"):
tasks_to_remove = []
@@ -739,4 +826,4 @@ class TaskManager:
# [/DEF:TaskManager:Class]
# [/DEF:TaskManager:Module]
# [/DEF:TaskManagerModule:Module]