# #region TranslationScheduler [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, schedule, cron, job] # @BRIEF Manage TranslationSchedule rows and register them with core SchedulerService. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationSchedule] # @RELATION DEPENDS_ON -> [SchedulerService] # @RELATION DEPENDS_ON -> [TranslationOrchestrator] # @RELATION DEPENDS_ON -> [TranslationEventLog] # @PRE Database session and SchedulerService are available. # @POST TranslationSchedule CRUD persisted; APScheduler jobs registered/updated/removed. # @SIDE_EFFECT Registers APScheduler jobs; runs translations on trigger; creates events. # @RATIONALE Uses existing SchedulerService (APScheduler) to avoid creating a second scheduler instance. # @REJECTED Separate scheduler instance would create resource contention. # @REJECTED Polling-based approach — event-driven APScheduler is more precise. from datetime import UTC, datetime, timedelta import uuid from sqlalchemy.orm import Session from ...core.config_manager import ConfigManager from ...core.cot_logger import seed_trace_id from ...core.logger import belief_scope, logger from ...models.translate import TranslationJob, TranslationRun, TranslationSchedule from ...services.notifications.service import NotificationService from .events import TranslationEventLog # #region TranslationScheduler [C:4] [TYPE Class] # @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers. class TranslationScheduler: def __init__(self, db: Session, config_manager: ConfigManager, current_user: str | None = None): self.db = db self.config_manager = config_manager self.current_user = current_user self.event_log = TranslationEventLog(db) # region create_schedule [TYPE Function] # @PURPOSE: Create a new schedule for a job. # @PRE job_id exists. cron_expression is valid. # @POST TranslationSchedule row created. def create_schedule( self, job_id: str, cron_expression: str, timezone: str = "UTC", is_active: bool = True, execution_mode: str = "full", ) -> TranslationSchedule: with belief_scope("TranslationScheduler.create_schedule"): # Verify job exists job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first() if not job: raise ValueError(f"Translation job '{job_id}' not found") schedule = TranslationSchedule( id=str(uuid.uuid4()), job_id=job_id, cron_expression=cron_expression, timezone=timezone, is_active=is_active, execution_mode=execution_mode, created_by=self.current_user, ) self.db.add(schedule) self.db.commit() self.db.refresh(schedule) self.event_log.log_event( job_id=job_id, event_type="SCHEDULE_CREATED", payload={ "schedule_id": schedule.id, "cron_expression": cron_expression, "timezone": timezone, }, created_by=self.current_user, ) logger.reflect("Schedule created", {"schedule_id": schedule.id, "job_id": job_id}) return schedule # endregion create_schedule # region update_schedule [TYPE Function] # @PURPOSE: Update an existing schedule. # @PRE job_id has an existing schedule. # @POST Schedule updated. def update_schedule( self, job_id: str, cron_expression: str | None = None, timezone_str: str | None = None, is_active: bool | None = None, execution_mode: str | None = None, ) -> TranslationSchedule: with belief_scope("TranslationScheduler.update_schedule"): schedule = self.db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") if cron_expression is not None: schedule.cron_expression = cron_expression if timezone_str is not None: schedule.timezone = timezone_str if is_active is not None: schedule.is_active = is_active if execution_mode is not None: schedule.execution_mode = execution_mode schedule.updated_at = datetime.now(UTC) self.db.commit() self.db.refresh(schedule) self.event_log.log_event( job_id=job_id, event_type="SCHEDULE_UPDATED", payload={ "schedule_id": schedule.id, "cron_expression": schedule.cron_expression, "timezone": schedule.timezone, "is_active": schedule.is_active, }, created_by=self.current_user, ) logger.reflect("Schedule updated", {"schedule_id": schedule.id, "job_id": job_id}) return schedule # endregion update_schedule # region delete_schedule [TYPE Function] # @PURPOSE: Delete a schedule for a job. # @PRE job_id has an existing schedule. # @POST Schedule deleted. def delete_schedule(self, job_id: str) -> None: with belief_scope("TranslationScheduler.delete_schedule"): schedule = self.db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") schedule_id = schedule.id self.db.delete(schedule) self.db.commit() self.event_log.log_event( job_id=job_id, event_type="SCHEDULE_DELETED", payload={"schedule_id": schedule_id}, created_by=self.current_user, ) logger.reflect("Schedule deleted", {"schedule_id": schedule_id, "job_id": job_id}) # endregion delete_schedule # region enable_disable_schedule [TYPE Function] # @PURPOSE: Enable or disable a schedule. # @PRE job_id has an existing schedule. # @POST Schedule is_active updated. def set_schedule_active(self, job_id: str, is_active: bool) -> TranslationSchedule: with belief_scope("TranslationScheduler.set_schedule_active"): schedule = self.db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") schedule.is_active = is_active schedule.updated_at = datetime.now(UTC) self.db.commit() self.db.refresh(schedule) logger.reflect("Schedule active state set", { "schedule_id": schedule.id, "job_id": job_id, "is_active": is_active, }) return schedule # endregion enable_disable_schedule # region get_schedule [TYPE Function] # @PURPOSE: Get schedule for a job. # @PRE job_id exists. # @POST Returns TranslationSchedule or raises ValueError. def get_schedule(self, job_id: str) -> TranslationSchedule: with belief_scope("TranslationScheduler.get_schedule"): schedule = self.db.query(TranslationSchedule).filter( TranslationSchedule.job_id == job_id ).first() if not schedule: raise ValueError(f"No schedule found for job '{job_id}'") return schedule # endregion get_schedule # region list_active_schedules [TYPE Function] # @PURPOSE: List all active schedules. # @POST Returns list of active TranslationSchedule rows. @staticmethod def list_active_schedules(db: Session) -> list[TranslationSchedule]: return ( db.query(TranslationSchedule) .filter(TranslationSchedule.is_active) .all() ) # endregion list_active_schedules # region get_next_executions [TYPE Function] # @PURPOSE: Compute next N execution times from cron expression. # @PRE cron_expression is valid. # @POST Returns list of ISO datetime strings. @staticmethod def get_next_executions(cron_expression: str, timezone_str: str = "UTC", n: int = 3) -> list[str]: from zoneinfo import ZoneInfo from apscheduler.triggers.cron import CronTrigger try: tz = ZoneInfo(timezone_str) trigger = CronTrigger.from_crontab(cron_expression, timezone=tz) except (ValueError, KeyError) as e: logger.warning(f"[get_next_executions] Invalid cron: {e}") return [] now = datetime.now(tz) results = [] next_time = now prev = None for _ in range(n): ft = trigger.get_next_fire_time(prev, next_time) if ft is None: break results.append(ft.isoformat()) prev = ft next_time = ft return results # endregion get_next_executions # #endregion TranslationScheduler # #region execute_scheduled_translation [C:4] [TYPE Function] # @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback. # @PRE schedule_id is valid. # @POST Translation run created and executed if no concurrent run exists. # @SIDE_EFFECT DB writes; LLM calls; Superset API calls. # @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full. # @COMPLEXITY 4 def execute_scheduled_translation( schedule_id: str, job_id: str, db_session_maker, config_manager: ConfigManager, execution_mode: str = "full", ) -> None: """APScheduler job callback for scheduled translations.""" seed_trace_id() db: Session = db_session_maker() try: with belief_scope("execute_scheduled_translation"): # Load schedule schedule = db.query(TranslationSchedule).filter( TranslationSchedule.id == schedule_id, TranslationSchedule.job_id == job_id, ).first() if not schedule or not schedule.is_active: logger.info(f"[scheduled_translation] Schedule inactive/missing: {schedule_id}") return logger.reason("Scheduled translation triggered", { "schedule_id": schedule_id, "job_id": job_id, }) # Concurrency check: max 1 pending/running run per job stale_threshold = timedelta(hours=1) now = datetime.now(UTC) active_run = ( db.query(TranslationRun) .filter( TranslationRun.job_id == job_id, TranslationRun.status.in_(["PENDING", "RUNNING"]), ) .order_by(TranslationRun.created_at.asc()) # oldest first .first() ) if active_run: is_stale = ( active_run.status == "PENDING" and active_run.created_at and (now - active_run.created_at) > stale_threshold ) if is_stale: # Mark ALL stale PENDING runs as FAILED stale_runs = ( db.query(TranslationRun) .filter( TranslationRun.job_id == job_id, TranslationRun.status == "PENDING", TranslationRun.created_at < (now - stale_threshold), ) .all() ) for sr in stale_runs: sr.status = "FAILED" sr.error_message = "Stale: concurrency deadlock — marked by scheduled trigger" sr.completed_at = now db.commit() logger.explore(f"Cleared {len(stale_runs)} stale PENDING run(s), proceeding with scheduled trigger", { "job_id": job_id, "stale_run_ids": [sr.id for sr in stale_runs], }) # Proceed to execute the scheduled run (do NOT return) else: logger.explore("Skipping scheduled run — concurrent run in progress", { "job_id": job_id, "existing_run_id": active_run.id, }) event_log = TranslationEventLog(db) event_log.log_event( job_id=job_id, event_type="RUN_STARTED", payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id}, ) return # Check baseline expiry baseline_expired = False most_recent = ( db.query(TranslationRun) .filter( TranslationRun.job_id == job_id, TranslationRun.insert_status == "succeeded", ) .order_by(TranslationRun.created_at.desc()) .first() ) if most_recent: age = datetime.now(UTC) - most_recent.created_at if age > timedelta(days=90): baseline_expired = True logger.reason("Baseline expired — full translation", { "job_id": job_id, "last_run": most_recent.created_at.isoformat(), "age_days": age.days, }) # Import and execute from .orchestrator import TranslationOrchestrator orch = TranslationOrchestrator(db, config_manager, current_user="scheduler") run = orch.start_run( job_id=job_id, is_scheduled=True, ) # Determine trigger type based on execution_mode and baseline expiry if execution_mode == "new_key_only" and not baseline_expired: run.trigger_type = "new_key_only" elif execution_mode == "new_key_only" and baseline_expired: logger.reason("Baseline expired — falling back to full translation", { "job_id": job_id, "schedule_id": schedule_id, "age_days": age.days, }) run.trigger_type = "scheduled" else: run.trigger_type = "scheduled" db.flush() if baseline_expired: event_log = TranslationEventLog(db) event_log.log_event( job_id=job_id, run_id=run.id, event_type="RUN_STARTED", payload={"reason": "baseline_expired", "age_days": age.days}, ) # Execute in same thread (APScheduler runs in background thread pool) try: orch.execute_run(run) if run.status == "FAILED": logger.explore("Scheduled translation completed with all records failed", { "run_id": run.id, "error": run.error_message, }) run.insert_status = run.insert_status or None else: run.insert_status = run.insert_status or "succeeded" except Exception as exec_err: logger.explore("Scheduled translation execution failed", { "run_id": run.id, "error": str(exec_err), }) # Send notification on scheduled-run failure (FR-041, FR-048) try: notify_service = NotificationService(db, config_manager) notify_service._initialize_providers() subject = f"Scheduled translation failed: job={job_id}" body = ( f"Scheduled translation run failed for job '{job_id}'.\n" f"Schedule: {schedule_id}\n" f"Error: {exec_err}\n" f"Time: {datetime.now(UTC).isoformat()}" ) import asyncio for provider in notify_service._providers.values(): asyncio.run(provider.send(recipient="admin", subject=subject, body=body)) except Exception as notify_err: logger.warning(f"Failed to send failure notification: {notify_err}") # Leave schedule enabled — schedule continues on failure run.status = "FAILED" run.error_message = str(exec_err) run.completed_at = datetime.now(UTC) # Update schedule tracking schedule.last_run_at = datetime.now(UTC) db.commit() logger.reflect("Scheduled translation complete", { "schedule_id": schedule_id, "run_id": run.id, "status": run.status, }) except Exception as e: logger.error(f"[scheduled_translation] Unexpected error: {e}") try: notify_service = NotificationService(db, config_manager) notify_service._initialize_providers() subject = f"Scheduled translation CRASHED: job={job_id}" body = f"Scheduled translation for job '{job_id}' crashed unexpectedly.\nError: {e}\nTime: {datetime.now(UTC).isoformat()}" import asyncio for provider in notify_service._providers.values(): asyncio.run(provider.send(recipient="admin", subject=subject, body=body)) except Exception: pass finally: db.close() # #endregion execute_scheduled_translation # #endregion TranslationScheduler