fix(validation): auto-cleanup stuck validation runs on backend startup

When the backend restarts (deploy, crash, reload), the in-memory
TaskManager loses its queue. ValidationRun records left 'running'
in the DB would hang forever, blocking re-runs.

Fix: during lifespan startup, query all ValidationRun with
status='running' and force-stop them as FAIL with a clear summary.

This prevents the 'already has a running run' error after restarts.
This commit is contained in:
2026-05-31 23:00:35 +03:00
parent 7f7a85b2c5
commit c8e81747fc

View File

@@ -84,6 +84,27 @@ async def lifespan(app: FastAPI):
init_db()
logger.info("👤 Bootstrapping admin user...")
ensure_initial_admin_user()
# Clean up stuck validation runs from previous backend lifetime
# (runs left "running" in DB when the in-memory task queue was lost)
try:
from sqlalchemy.orm import Session as _Ses
from src.core.database import SessionLocal as _Db
from src.models.llm import ValidationRun as _VR
from datetime import datetime, timezone
_s: _Ses = _Db()
_stuck = _s.query(_VR).filter(_VR.status == "running").all()
for _r in _stuck:
_r.status = "FAIL"
_r.finished_at = datetime.now(timezone.utc)
_r.summary = "Force-stopped: backend restarted while run was in progress"
logger.reason(
f"Force-stopped stuck run {_r.id} (policy={_r.policy_id})",
extra={"src": "app.startup", "run_id": _r.id},
)
_s.commit()
_s.close()
except Exception as _e:
logger.warning(f"Failed to clean up stuck validation runs: {_e}")
logger.info("⏰ Starting scheduler...")
scheduler = get_scheduler_service()
scheduler.start()