444 lines
18 KiB
Python
444 lines
18 KiB
Python
# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, schedule, cron, job]
|
|
# @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService.
|
|
# @LAYER Domain
|
|
# @RELATION DEPENDS_ON -> [TranslationSchedule]
|
|
# @RELATION DEPENDS_ON -> [SchedulerService]
|
|
# @RELATION DEPENDS_ON -> [TranslationOrchestrator]
|
|
# @RELATION DEPENDS_ON -> [TranslationEventLog]
|
|
# @PRE Database session and SchedulerService are available.
|
|
# @POST TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed.
|
|
# @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events.
|
|
# @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance.
|
|
# @REJECTED Separate scheduler instance would create resource contention.
|
|
# @REJECTED Polling-based approach — event-driven APScheduler is more precise.
|
|
|
|
from datetime import UTC, datetime, timedelta
|
|
import uuid
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ...core.config_manager import ConfigManager
|
|
from ...core.cot_logger import seed_trace_id
|
|
from ...core.logger import belief_scope, logger
|
|
from ...models.translate import TranslationJob, TranslationRun, TranslationSchedule
|
|
from ...services.notifications.service import NotificationService
|
|
from .events import TranslationEventLog
|
|
|
|
|
|
# #region TranslationScheduler [C:4] [TYPE Class]
|
|
# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
|
|
class TranslationScheduler:
|
|
|
|
def __init__(self, db: Session, config_manager: ConfigManager, current_user: str | None = None):
|
|
self.db = db
|
|
self.config_manager = config_manager
|
|
self.current_user = current_user
|
|
self.event_log = TranslationEventLog(db)
|
|
|
|
# region create_schedule [TYPE Function]
|
|
# @PURPOSE: Create a new schedule for a job.
|
|
# @PRE job_id exists. cron_expression is valid.
|
|
# @POST TranslationSchedule row created.
|
|
def create_schedule(
|
|
self,
|
|
job_id: str,
|
|
cron_expression: str,
|
|
timezone: str = "UTC",
|
|
is_active: bool = True,
|
|
execution_mode: str = "full",
|
|
) -> TranslationSchedule:
|
|
with belief_scope("TranslationScheduler.create_schedule"):
|
|
# Verify job exists
|
|
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
|
if not job:
|
|
raise ValueError(f"Translation job '{job_id}' not found")
|
|
|
|
schedule = TranslationSchedule(
|
|
id=str(uuid.uuid4()),
|
|
job_id=job_id,
|
|
cron_expression=cron_expression,
|
|
timezone=timezone,
|
|
is_active=is_active,
|
|
execution_mode=execution_mode,
|
|
created_by=self.current_user,
|
|
)
|
|
self.db.add(schedule)
|
|
self.db.commit()
|
|
self.db.refresh(schedule)
|
|
|
|
self.event_log.log_event(
|
|
job_id=job_id,
|
|
event_type="SCHEDULE_CREATED",
|
|
payload={
|
|
"schedule_id": schedule.id,
|
|
"cron_expression": cron_expression,
|
|
"timezone": timezone,
|
|
},
|
|
created_by=self.current_user,
|
|
)
|
|
|
|
logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id})
|
|
return schedule
|
|
# endregion create_schedule
|
|
|
|
# region update_schedule [TYPE Function]
|
|
# @PURPOSE: Update an existing schedule.
|
|
# @PRE job_id has an existing schedule.
|
|
# @POST Schedule updated.
|
|
def update_schedule(
|
|
self,
|
|
job_id: str,
|
|
cron_expression: str | None = None,
|
|
timezone_str: str | None = None,
|
|
is_active: bool | None = None,
|
|
execution_mode: str | None = None,
|
|
) -> TranslationSchedule:
|
|
with belief_scope("TranslationScheduler.update_schedule"):
|
|
schedule = self.db.query(TranslationSchedule).filter(
|
|
TranslationSchedule.job_id == job_id
|
|
).first()
|
|
if not schedule:
|
|
raise ValueError(f"No schedule found for job '{job_id}'")
|
|
|
|
if cron_expression is not None:
|
|
schedule.cron_expression = cron_expression
|
|
if timezone_str is not None:
|
|
schedule.timezone = timezone_str
|
|
if is_active is not None:
|
|
schedule.is_active = is_active
|
|
if execution_mode is not None:
|
|
schedule.execution_mode = execution_mode
|
|
schedule.updated_at = datetime.now(UTC)
|
|
self.db.commit()
|
|
self.db.refresh(schedule)
|
|
|
|
self.event_log.log_event(
|
|
job_id=job_id,
|
|
event_type="SCHEDULE_UPDATED",
|
|
payload={
|
|
"schedule_id": schedule.id,
|
|
"cron_expression": schedule.cron_expression,
|
|
"timezone": schedule.timezone,
|
|
"is_active": schedule.is_active,
|
|
},
|
|
created_by=self.current_user,
|
|
)
|
|
|
|
logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id})
|
|
return schedule
|
|
# endregion update_schedule
|
|
|
|
# region delete_schedule [TYPE Function]
|
|
# @PURPOSE: Delete a schedule for a job.
|
|
# @PRE job_id has an existing schedule.
|
|
# @POST Schedule deleted.
|
|
def delete_schedule(self, job_id: str) -> None:
|
|
with belief_scope("TranslationScheduler.delete_schedule"):
|
|
schedule = self.db.query(TranslationSchedule).filter(
|
|
TranslationSchedule.job_id == job_id
|
|
).first()
|
|
if not schedule:
|
|
raise ValueError(f"No schedule found for job '{job_id}'")
|
|
|
|
schedule_id = schedule.id
|
|
self.db.delete(schedule)
|
|
self.db.commit()
|
|
|
|
self.event_log.log_event(
|
|
job_id=job_id,
|
|
event_type="SCHEDULE_DELETED",
|
|
payload={"schedule_id": schedule_id},
|
|
created_by=self.current_user,
|
|
)
|
|
|
|
logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id})
|
|
# endregion delete_schedule
|
|
|
|
# region enable_disable_schedule [TYPE Function]
|
|
# @PURPOSE: Enable or disable a schedule.
|
|
# @PRE job_id has an existing schedule.
|
|
# @POST Schedule is_active updated.
|
|
def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule:
|
|
with belief_scope("TranslationScheduler.set_schedule_active"):
|
|
schedule = self.db.query(TranslationSchedule).filter(
|
|
TranslationSchedule.job_id == job_id
|
|
).first()
|
|
if not schedule:
|
|
raise ValueError(f"No schedule found for job '{job_id}'")
|
|
|
|
schedule.is_active = is_active
|
|
schedule.updated_at = datetime.now(UTC)
|
|
self.db.commit()
|
|
self.db.refresh(schedule)
|
|
|
|
logger.reflect("Schedule active state set", {
|
|
"schedule_id": schedule.id,
|
|
"job_id": job_id,
|
|
"is_active": is_active,
|
|
})
|
|
return schedule
|
|
# endregion enable_disable_schedule
|
|
|
|
# region get_schedule [TYPE Function]
|
|
# @PURPOSE: Get schedule for a job.
|
|
# @PRE job_id exists.
|
|
# @POST Returns TranslationSchedule or raises ValueError.
|
|
def get_schedule(self, job_id: str) -> TranslationSchedule:
|
|
with belief_scope("TranslationScheduler.get_schedule"):
|
|
schedule = self.db.query(TranslationSchedule).filter(
|
|
TranslationSchedule.job_id == job_id
|
|
).first()
|
|
if not schedule:
|
|
raise ValueError(f"No schedule found for job '{job_id}'")
|
|
return schedule
|
|
# endregion get_schedule
|
|
|
|
# region list_active_schedules [TYPE Function]
|
|
# @PURPOSE: List all active schedules.
|
|
# @POST Returns list of active TranslationSchedule rows.
|
|
@staticmethod
|
|
def list_active_schedules(db: Session) -> list[TranslationSchedule]:
|
|
return (
|
|
db.query(TranslationSchedule)
|
|
.filter(TranslationSchedule.is_active)
|
|
.all()
|
|
)
|
|
# endregion list_active_schedules
|
|
|
|
# region get_next_executions [TYPE Function]
|
|
# @PURPOSE: Compute next N execution times from cron expression.
|
|
# @PRE cron_expression is valid.
|
|
# @POST Returns list of ISO datetime strings.
|
|
@staticmethod
|
|
def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> list[str]:
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
try:
|
|
tz = ZoneInfo(timezone_str)
|
|
trigger = CronTrigger.from_crontab(cron_expression, timezone=tz)
|
|
except (ValueError, KeyError) as e:
|
|
logger.warning(f"[get_next_executions] Invalid cron: {e}")
|
|
return []
|
|
|
|
now = datetime.now(tz)
|
|
results = []
|
|
next_time = now
|
|
prev = None
|
|
for _ in range(n):
|
|
ft = trigger.get_next_fire_time(prev, next_time)
|
|
if ft is None:
|
|
break
|
|
results.append(ft.isoformat())
|
|
prev = ft
|
|
next_time = ft
|
|
return results
|
|
# endregion get_next_executions
|
|
|
|
|
|
# #endregion TranslationScheduler
|
|
|
|
|
|
# #region execute_scheduled_translation [C:4] [TYPE Function]
|
|
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
|
|
# @PRE schedule_id is valid.
|
|
# @POST Translation run created and executed if no concurrent run exists.
|
|
# @SIDE_EFFECT DB writes; LLM calls; Superset API calls.
|
|
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
|
|
# @COMPLEXITY 4
|
|
|
|
def execute_scheduled_translation(
|
|
schedule_id: str,
|
|
job_id: str,
|
|
db_session_maker,
|
|
config_manager: ConfigManager,
|
|
execution_mode: str = "full",
|
|
) -> None:
|
|
"""APScheduler job callback for scheduled translations."""
|
|
seed_trace_id()
|
|
db: Session = db_session_maker()
|
|
try:
|
|
with belief_scope("execute_scheduled_translation"):
|
|
# Load schedule
|
|
schedule = db.query(TranslationSchedule).filter(
|
|
TranslationSchedule.id == schedule_id,
|
|
TranslationSchedule.job_id == job_id,
|
|
).first()
|
|
if not schedule or not schedule.is_active:
|
|
logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}")
|
|
return
|
|
|
|
logger.reason("Scheduled translation triggered", {
|
|
"schedule_id": schedule_id,
|
|
"job_id": job_id,
|
|
})
|
|
|
|
# Concurrency check: max 1 pending/running run per job
|
|
stale_threshold = timedelta(hours=1)
|
|
now = datetime.now(UTC)
|
|
|
|
active_run = (
|
|
db.query(TranslationRun)
|
|
.filter(
|
|
TranslationRun.job_id == job_id,
|
|
TranslationRun.status.in_(["PENDING", "RUNNING"]),
|
|
)
|
|
.order_by(TranslationRun.created_at.asc()) # oldest first
|
|
.first()
|
|
)
|
|
if active_run:
|
|
is_stale = (
|
|
active_run.status == "PENDING"
|
|
and active_run.created_at
|
|
and (now - active_run.created_at) > stale_threshold
|
|
)
|
|
if is_stale:
|
|
# Mark ALL stale PENDING runs as FAILED
|
|
stale_runs = (
|
|
db.query(TranslationRun)
|
|
.filter(
|
|
TranslationRun.job_id == job_id,
|
|
TranslationRun.status == "PENDING",
|
|
TranslationRun.created_at < (now - stale_threshold),
|
|
)
|
|
.all()
|
|
)
|
|
for sr in stale_runs:
|
|
sr.status = "FAILED"
|
|
sr.error_message = "Stale: concurrency deadlock — marked by scheduled trigger"
|
|
sr.completed_at = now
|
|
db.commit()
|
|
logger.explore(f"Cleared {len(stale_runs)} stale PENDING run(s), proceeding with scheduled trigger", {
|
|
"job_id": job_id,
|
|
"stale_run_ids": [sr.id for sr in stale_runs],
|
|
})
|
|
# Proceed to execute the scheduled run (do NOT return)
|
|
else:
|
|
logger.explore("Skipping scheduled run — concurrent run in progress", {
|
|
"job_id": job_id,
|
|
"existing_run_id": active_run.id,
|
|
})
|
|
event_log = TranslationEventLog(db)
|
|
event_log.log_event(
|
|
job_id=job_id,
|
|
event_type="RUN_STARTED",
|
|
payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id},
|
|
)
|
|
return
|
|
|
|
# Check baseline expiry
|
|
baseline_expired = False
|
|
most_recent = (
|
|
db.query(TranslationRun)
|
|
.filter(
|
|
TranslationRun.job_id == job_id,
|
|
TranslationRun.insert_status == "succeeded",
|
|
)
|
|
.order_by(TranslationRun.created_at.desc())
|
|
.first()
|
|
)
|
|
if most_recent:
|
|
age = datetime.now(UTC) - most_recent.created_at
|
|
if age > timedelta(days=90):
|
|
baseline_expired = True
|
|
logger.reason("Baseline expired — full translation", {
|
|
"job_id": job_id,
|
|
"last_run": most_recent.created_at.isoformat(),
|
|
"age_days": age.days,
|
|
})
|
|
|
|
# Import and execute
|
|
from .orchestrator import TranslationOrchestrator
|
|
|
|
orch = TranslationOrchestrator(db, config_manager, current_user="scheduler")
|
|
run = orch.start_run(
|
|
job_id=job_id,
|
|
is_scheduled=True,
|
|
)
|
|
|
|
# Determine trigger type based on execution_mode and baseline expiry
|
|
if execution_mode == "new_key_only" and not baseline_expired:
|
|
run.trigger_type = "new_key_only"
|
|
elif execution_mode == "new_key_only" and baseline_expired:
|
|
logger.reason("Baseline expired — falling back to full translation", {
|
|
"job_id": job_id, "schedule_id": schedule_id, "age_days": age.days,
|
|
})
|
|
run.trigger_type = "scheduled"
|
|
else:
|
|
run.trigger_type = "scheduled"
|
|
db.flush()
|
|
|
|
if baseline_expired:
|
|
event_log = TranslationEventLog(db)
|
|
event_log.log_event(
|
|
job_id=job_id,
|
|
run_id=run.id,
|
|
event_type="RUN_STARTED",
|
|
payload={"reason": "baseline_expired", "age_days": age.days},
|
|
)
|
|
|
|
# Execute in same thread (APScheduler runs in background thread pool)
|
|
try:
|
|
orch.execute_run(run)
|
|
if run.status == "FAILED":
|
|
logger.explore("Scheduled translation completed with all records failed", {
|
|
"run_id": run.id,
|
|
"error": run.error_message,
|
|
})
|
|
run.insert_status = run.insert_status or None
|
|
else:
|
|
run.insert_status = run.insert_status or "succeeded"
|
|
except Exception as exec_err:
|
|
logger.explore("Scheduled translation execution failed", {
|
|
"run_id": run.id,
|
|
"error": str(exec_err),
|
|
})
|
|
# Send notification on scheduled-run failure (FR-041, FR-048)
|
|
try:
|
|
notify_service = NotificationService(db, config_manager)
|
|
notify_service._initialize_providers()
|
|
subject = f"Scheduled translation failed: job={job_id}"
|
|
body = (
|
|
f"Scheduled translation run failed for job '{job_id}'.\n"
|
|
f"Schedule: {schedule_id}\n"
|
|
f"Error: {exec_err}\n"
|
|
f"Time: {datetime.now(UTC).isoformat()}"
|
|
)
|
|
import asyncio
|
|
for provider in notify_service._providers.values():
|
|
asyncio.run(provider.send(recipient="admin", subject=subject, body=body))
|
|
except Exception as notify_err:
|
|
logger.warning(f"Failed to send failure notification: {notify_err}")
|
|
|
|
# Leave schedule enabled — schedule continues on failure
|
|
run.status = "FAILED"
|
|
run.error_message = str(exec_err)
|
|
run.completed_at = datetime.now(UTC)
|
|
|
|
# Update schedule tracking
|
|
schedule.last_run_at = datetime.now(UTC)
|
|
db.commit()
|
|
|
|
logger.reflect("Scheduled translation complete", {
|
|
"schedule_id": schedule_id,
|
|
"run_id": run.id,
|
|
"status": run.status,
|
|
})
|
|
except Exception as e:
|
|
logger.error(f"[scheduled_translation] Unexpected error: {e}")
|
|
try:
|
|
notify_service = NotificationService(db, config_manager)
|
|
notify_service._initialize_providers()
|
|
subject = f"Scheduled translation CRASHED: job={job_id}"
|
|
body = f"Scheduled translation for job '{job_id}' crashed unexpectedly.\nError: {e}\nTime: {datetime.now(UTC).isoformat()}"
|
|
import asyncio
|
|
for provider in notify_service._providers.values():
|
|
asyncio.run(provider.send(recipient="admin", subject=subject, body=body))
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
db.close()
|
|
# #endregion execute_scheduled_translation
|
|
# #endregion TranslationScheduler
|