freeze fix

This commit is contained in:
2026-06-02 16:36:00 +03:00
parent 01e0d1c529
commit 29acfb4e69
13 changed files with 1171 additions and 181 deletions

View File

@@ -829,10 +829,10 @@ async def translate_run_websocket(websocket: WebSocket, run_id: str):
try:
from .core.database import SessionLocal
from .plugins.translate.orchestrator_aggregator import TranslationResultAggregator
from .core.event_log import EventLog
from .plugins.translate.events import TranslationEventLog
db = SessionLocal()
try:
event_log = EventLog(db)
event_log = TranslationEventLog(db)
aggregator = TranslationResultAggregator(db, event_log)
status = aggregator.get_run_status(run_id)
total = status.get("total_records", 0) or 0

View File

@@ -224,12 +224,28 @@ class BatchProcessingService:
if tb["warning"]:
logger.explore("Token budget warning", {"batch_id": bid, "warning": tb["warning"]})
return self._llm_service.call_llm_for_batch(
logger.reason(
f"LLM process batch start", {
"batch_id": bid, "llm_rows": len(rows_for_llm),
"provider_model": provider_model,
"max_output_needed": tb.get("max_output_needed"),
"estimated_input_tokens": tb.get("estimated_input_tokens"),
},
)
result = self._llm_service.call_llm_for_batch(
job=job, run_id=run_id, batch_rows=rows_for_llm,
dict_matches=dict_matches, batch_id=bid,
max_tokens=tb["max_output_needed"],
)
logger.reason(
f"LLM process batch complete", {
"batch_id": bid, **result,
},
)
return result
# -- Batch insert (delegation) --
def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)

View File

@@ -57,6 +57,15 @@ class LLMTranslationService:
) -> dict[str, int]:
"""Call LLM for a batch of rows; parse response; create records."""
with belief_scope("LLMTranslationService.call_llm_for_batch"):
provider_label = f"{job.provider_id}/{getattr(job, '_provider_model', '?')}"
logger.reason(
f"LLM batch start", {
"batch_id": batch_id, "row_count": len(batch_rows),
"provider": provider_label, "max_tokens": max_tokens,
"recursion_depth": _recursion_depth,
},
)
dictionary_section = self._build_dictionary_section(dict_matches, batch_rows)
target_languages = self._resolve_target_languages(job)
prompt = self._build_prompt(job, batch_rows, dictionary_section, target_languages)
@@ -65,10 +74,22 @@ class LLMTranslationService:
job, prompt, batch_id, max_tokens,
)
if llm_response is None:
logger.explore(
f"LLM batch failed after {retries} retries", {
"batch_id": batch_id, "row_count": len(batch_rows),
"last_error": last_error, "retries": retries,
},
)
return self._handle_llm_failure(batch_rows, run_id, batch_id, retries, last_error)
if finish_reason == "length" and len(batch_rows) >= 2 and run_id:
if _recursion_depth < MAX_RETRIES_PER_BATCH:
logger.reason(
f"Splitting truncated batch", {
"batch_id": batch_id, "size": len(batch_rows),
"depth": _recursion_depth,
},
)
return self._split_and_retry(job, run_id, batch_rows, dict_matches,
batch_id, max_tokens, _recursion_depth, retries)
logger.explore("Truncation recursion depth exceeded", {"batch_id": batch_id, "depth": _recursion_depth})
@@ -76,11 +97,24 @@ class LLMTranslationService:
try:
translations = parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages)
except ValueError as e:
logger.explore(
f"LLM parse failure", {
"batch_id": batch_id, "error": str(e),
"response_len": len(llm_response),
"response_preview": llm_response[:500],
},
)
return self._handle_parse_failure(batch_rows, run_id, batch_id, retries, e)
return self._create_records_from_translations(
result = self._create_records_from_translations(
batch_rows, run_id, batch_id, target_languages, translations, dict_matches, retries,
)
logger.reason(
f"LLM batch complete", {
"batch_id": batch_id, **result,
},
)
return result
# #endregion call_llm_for_batch
def _build_dictionary_section(self, dict_matches, batch_rows) -> str:
@@ -120,9 +154,16 @@ class LLMTranslationService:
last_error = None
retries = 0
finish_reason = None
logger.reason(f"LLM retry loop start", {"batch_id": batch_id, "max_retries": MAX_RETRIES_PER_BATCH, "prompt_len": len(prompt)})
for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):
try:
llm_response, finish_reason = self.call_llm(job, prompt, max_tokens=max_tokens)
logger.reason(
f"LLM call succeeded (attempt {attempt})", {
"batch_id": batch_id, "finish_reason": finish_reason,
"response_len": len(llm_response) if llm_response else 0,
},
)
break
except Exception as e:
last_error = str(e)
@@ -130,6 +171,8 @@ class LLMTranslationService:
logger.explore(f"LLM call failed (attempt {attempt})", {"batch_id": batch_id, "error": last_error})
if attempt < MAX_RETRIES_PER_BATCH:
time.sleep(2 ** attempt)
if llm_response is None:
logger.explore(f"All LLM retries exhausted", {"batch_id": batch_id, "retries": retries, "last_error": last_error})
return llm_response, finish_reason, retries, last_error
def _handle_llm_failure(self, batch_rows, run_id, batch_id, retries, last_error):
@@ -292,12 +335,27 @@ class LLMTranslationService:
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
disable_reasoning = getattr(job, 'disable_reasoning', False)
logger.reason(
f"LLM provider resolved", {
"provider_id": job.provider_id, "model": model,
"provider_type": provider_type, "base_url": provider.base_url,
"disable_reasoning": disable_reasoning, "max_tokens": max_tokens,
},
)
if provider_type not in ("openai", "openai_compatible", "openrouter", "kilo", "litellm"):
raise ValueError(f"Unsupported provider type '{provider_type}'")
return call_openai_compatible(
result = call_openai_compatible(
base_url=provider.base_url, api_key=api_key, model=model, prompt=prompt,
provider_type=provider_type, max_tokens=max_tokens, disable_reasoning=disable_reasoning,
)
logger.reason(
f"LLM provider call complete", {
"response_len": len(result[0]) if result and result[0] else 0,
"finish_reason": result[1],
},
)
return result
# #endregion call_llm
# -- Static methods delegated to sub-modules for backward compat --