fix(frontend): add WS reconnect + timeout safety nets for translate runs
QA found critical regression: after removing HTTP polling, a WS disconnect left the UI permanently stuck (uxState='running' forever). Fixes: - WS reconnect: up to 3 attempts with 10s backoff on close (non-1000/1005) - App timeout: 600s (10 min) max — transitions to failed if stuck - Stale log message: removed 'falling back to polling' (no longer exists) - Contract header: updated @BRIEF, restored @UX_STATE/@UX_FEEDBACK/@UX_RECOVERY - cleanup(): unified WS close + timer clear (used by stop, timeout, terminal) - onclose handler: triggers reconnect via _handleWsFailure() - onerror handler: defers to onclose (fires after error) - onopen handler: resets _reconnectCount on successful connect - Terminal states: call _cleanup() to stop reconnect timer + timeout Build ✅ 698 tests ✅
This commit is contained in:
@@ -1,8 +1,16 @@
|
||||
// #region TranslationRunStore [C:4] [TYPE Store] [SEMANTICS translate, run, progress, websocket, polling, store]
|
||||
// @BRIEF Global store for active translation run progress — uses WebSocket for real-time
|
||||
// progress + polling fallback for structured status. Survives page navigation.
|
||||
// #region TranslationRunStore [C:4] [TYPE Store] [SEMANTICS translate, run, progress, websocket, store]
|
||||
// @BRIEF Global store for active translation run progress — WebSocket-only real-time
|
||||
// streaming with reconnect + timeout safety nets. Survives page navigation.
|
||||
// @LAYER UI
|
||||
// @RELATION DEPENDS_ON -> [ApiModule.getTranslateRunWsUrl]
|
||||
// @UX_STATE idle -> No active run
|
||||
// @UX_STATE running -> WebSocket active, progress bar updates in real-time
|
||||
// @UX_STATE completed/partial/failed/insert_failed/cancelled -> Terminal states
|
||||
// @UX_FEEDBACK WebSocket streams batch completion + insert phase events
|
||||
// @UX_RECOVERY WebSocket reconnect: up to 3 attempts with 10s backoff on disconnect
|
||||
// @UX_RECOVERY Application timeout: 600s max — transitions to failed if stuck
|
||||
// @RATIONALE Migrated from Svelte 4 writable+derived to Svelte 5 $state+$derived runes.
|
||||
// Polling fallback removed — WS-only with explicit reconnect + timeout.
|
||||
|
||||
import { getTranslateRunWsUrl } from '$lib/api';
|
||||
|
||||
@@ -96,6 +104,12 @@ export const translationRunStore = {
|
||||
|
||||
let _onCompleteCallback: ((data: Record<string, unknown>) => void) | null = null;
|
||||
let _ws: WebSocket | null = null;
|
||||
let _reconnectCount = 0;
|
||||
let _reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let _timeoutTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const MAX_RECONNECT = 3;
|
||||
const RECONNECT_DELAY_MS = 10_000;
|
||||
const RUN_TIMEOUT_MS = 600_000; // 10 minutes
|
||||
|
||||
export function clearOnCompleteCallback(): void {
|
||||
_onCompleteCallback = null;
|
||||
@@ -116,8 +130,68 @@ export function startTranslationRun(runId: string, options: StartRunOptions = {}
|
||||
_notify();
|
||||
|
||||
_onCompleteCallback = options.onComplete || null;
|
||||
_reconnectCount = 0;
|
||||
|
||||
_connectWebSocket(runId);
|
||||
_startTimeout();
|
||||
}
|
||||
|
||||
function _startTimeout(): void {
|
||||
_clearTimeout();
|
||||
_timeoutTimer = setTimeout(() => {
|
||||
console.warn('[translate:ws] run timed out after 10 min — marking as failed', { runId: _state.runId });
|
||||
_cleanup();
|
||||
_state = { ..._state, uxState: 'failed' };
|
||||
_notify();
|
||||
if (_onCompleteCallback) _onCompleteCallback({ status: 'TIMEOUT', error_message: 'Translation run timed out' });
|
||||
}, RUN_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
function _clearTimeout(): void {
|
||||
if (_timeoutTimer) {
|
||||
clearTimeout(_timeoutTimer);
|
||||
_timeoutTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function _clearReconnect(): void {
|
||||
if (_reconnectTimer) {
|
||||
clearTimeout(_reconnectTimer);
|
||||
_reconnectTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function _cleanup(): void {
|
||||
_clearTimeout();
|
||||
_clearReconnect();
|
||||
if (_ws) {
|
||||
_ws.onclose = null;
|
||||
_ws.onerror = null;
|
||||
_ws.close();
|
||||
_ws = null;
|
||||
}
|
||||
}
|
||||
|
||||
function _handleWsFailure(reason: string): void {
|
||||
console.warn('[translate:ws] connection lost', { runId: _state.runId, reason, attempt: _reconnectCount + 1 });
|
||||
|
||||
if (_state.uxState === 'failed' || _state.uxState === 'completed' || _state.uxState === 'cancelled') {
|
||||
return; // Already terminal — don't reconnect
|
||||
}
|
||||
|
||||
_reconnectCount++;
|
||||
if (_reconnectCount <= MAX_RECONNECT) {
|
||||
console.debug('[translate:ws] reconnecting', { runId: _state.runId, attempt: _reconnectCount });
|
||||
_reconnectTimer = setTimeout(() => {
|
||||
if (_state.runId) _connectWebSocket(_state.runId);
|
||||
}, RECONNECT_DELAY_MS);
|
||||
} else {
|
||||
console.warn('[translate:ws] max reconnects exhausted — marking as failed', { runId: _state.runId });
|
||||
_cleanup();
|
||||
_state = { ..._state, uxState: 'failed' };
|
||||
_notify();
|
||||
if (_onCompleteCallback) _onCompleteCallback({ status: 'WS_FAILED', error_message: 'WebSocket connection lost after max reconnects' });
|
||||
}
|
||||
}
|
||||
|
||||
function _connectWebSocket(runId: string): void {
|
||||
@@ -127,6 +201,7 @@ function _connectWebSocket(runId: string): void {
|
||||
|
||||
_ws.onopen = () => {
|
||||
console.debug('[translate:ws] connected', { runId });
|
||||
_reconnectCount = 0; // Reset on successful connect
|
||||
};
|
||||
|
||||
_ws.onmessage = (event: MessageEvent) => {
|
||||
@@ -162,6 +237,7 @@ function _connectWebSocket(runId: string): void {
|
||||
};
|
||||
_notify();
|
||||
if (newUxState !== 'running' && newUxState !== 'inserting') {
|
||||
_cleanup(); // Terminal — stop reconnect timer + timeout
|
||||
if (_onCompleteCallback) _onCompleteCallback(data);
|
||||
}
|
||||
}
|
||||
@@ -171,30 +247,31 @@ function _connectWebSocket(runId: string): void {
|
||||
};
|
||||
|
||||
_ws.onerror = () => {
|
||||
console.warn('[translate:ws] connection error — falling back to polling', { runId });
|
||||
_ws = null;
|
||||
// Don't call _handleWsFailure here — onclose will fire next
|
||||
};
|
||||
|
||||
_ws.onclose = (event: CloseEvent) => {
|
||||
console.debug('[translate:ws] closed', { runId, code: event.code, reason: event.reason });
|
||||
_ws = null;
|
||||
if (event.code !== 1000 && event.code !== 1005) {
|
||||
_handleWsFailure(`close code ${event.code}`);
|
||||
}
|
||||
};
|
||||
} catch (_e) {
|
||||
console.warn('[translate:ws] failed to create WebSocket', { runId, error: String(_e) });
|
||||
_ws = null;
|
||||
_handleWsFailure('constructor failed');
|
||||
}
|
||||
}
|
||||
|
||||
export function stopTranslationRun(): void {
|
||||
if (_ws) {
|
||||
_ws.onclose = null;
|
||||
_ws.close();
|
||||
_ws = null;
|
||||
}
|
||||
_cleanup();
|
||||
}
|
||||
|
||||
export function resetTranslationRun(): void {
|
||||
stopTranslationRun();
|
||||
_reconnectCount = 0;
|
||||
_state = initialState;
|
||||
_notify();
|
||||
_onCompleteCallback = null;
|
||||
|
||||
Reference in New Issue
Block a user