From c8e81747fc4cadb8ee38f7d6a25ad01f2608d7bc Mon Sep 17 00:00:00 2001 From: busya Date: Sun, 31 May 2026 23:00:35 +0300 Subject: [PATCH] fix(validation): auto-cleanup stuck validation runs on backend startup When the backend restarts (deploy, crash, reload), the in-memory TaskManager loses its queue. ValidationRun records left 'running' in the DB would hang forever, blocking re-runs. Fix: during lifespan startup, query all ValidationRun with status='running' and force-stop them as FAIL with a clear summary. This prevents the 'already has a running run' error after restarts. --- backend/src/app.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/src/app.py b/backend/src/app.py index 496415b8..8eba40f6 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -84,6 +84,27 @@ async def lifespan(app: FastAPI): init_db() logger.info("👤 Bootstrapping admin user...") ensure_initial_admin_user() + # Clean up stuck validation runs from previous backend lifetime + # (runs left "running" in DB when the in-memory task queue was lost) + try: + from sqlalchemy.orm import Session as _Ses + from src.core.database import SessionLocal as _Db + from src.models.llm import ValidationRun as _VR + from datetime import datetime, timezone + _s: _Ses = _Db() + _stuck = _s.query(_VR).filter(_VR.status == "running").all() + for _r in _stuck: + _r.status = "FAIL" + _r.finished_at = datetime.now(timezone.utc) + _r.summary = "Force-stopped: backend restarted while run was in progress" + logger.reason( + f"Force-stopped stuck run {_r.id} (policy={_r.policy_id})", + extra={"src": "app.startup", "run_id": _r.id}, + ) + _s.commit() + _s.close() + except Exception as _e: + logger.warning(f"Failed to clean up stuck validation runs: {_e}") logger.info("⏰ Starting scheduler...") scheduler = get_scheduler_service() scheduler.start()