fix(validation): auto-cleanup stuck validation runs on backend startup
When the backend restarts (deploy, crash, reload), the in-memory TaskManager loses its queue. ValidationRun records left 'running' in the DB would hang forever, blocking re-runs. Fix: during lifespan startup, query all ValidationRun with status='running' and force-stop them as FAIL with a clear summary. This prevents the 'already has a running run' error after restarts.
This commit is contained in:
@@ -84,6 +84,27 @@ async def lifespan(app: FastAPI):
|
||||
init_db()
|
||||
logger.info("👤 Bootstrapping admin user...")
|
||||
ensure_initial_admin_user()
|
||||
# Clean up stuck validation runs from previous backend lifetime
|
||||
# (runs left "running" in DB when the in-memory task queue was lost)
|
||||
try:
|
||||
from sqlalchemy.orm import Session as _Ses
|
||||
from src.core.database import SessionLocal as _Db
|
||||
from src.models.llm import ValidationRun as _VR
|
||||
from datetime import datetime, timezone
|
||||
_s: _Ses = _Db()
|
||||
_stuck = _s.query(_VR).filter(_VR.status == "running").all()
|
||||
for _r in _stuck:
|
||||
_r.status = "FAIL"
|
||||
_r.finished_at = datetime.now(timezone.utc)
|
||||
_r.summary = "Force-stopped: backend restarted while run was in progress"
|
||||
logger.reason(
|
||||
f"Force-stopped stuck run {_r.id} (policy={_r.policy_id})",
|
||||
extra={"src": "app.startup", "run_id": _r.id},
|
||||
)
|
||||
_s.commit()
|
||||
_s.close()
|
||||
except Exception as _e:
|
||||
logger.warning(f"Failed to clean up stuck validation runs: {_e}")
|
||||
logger.info("⏰ Starting scheduler...")
|
||||
scheduler = get_scheduler_service()
|
||||
scheduler.start()
|
||||
|
||||
Reference in New Issue
Block a user