freeze fix
This commit is contained in:
@@ -829,10 +829,10 @@ async def translate_run_websocket(websocket: WebSocket, run_id: str):
|
||||
try:
|
||||
from .core.database import SessionLocal
|
||||
from .plugins.translate.orchestrator_aggregator import TranslationResultAggregator
|
||||
from .core.event_log import EventLog
|
||||
from .plugins.translate.events import TranslationEventLog
|
||||
db = SessionLocal()
|
||||
try:
|
||||
event_log = EventLog(db)
|
||||
event_log = TranslationEventLog(db)
|
||||
aggregator = TranslationResultAggregator(db, event_log)
|
||||
status = aggregator.get_run_status(run_id)
|
||||
total = status.get("total_records", 0) or 0
|
||||
|
||||
@@ -224,12 +224,28 @@ class BatchProcessingService:
|
||||
if tb["warning"]:
|
||||
logger.explore("Token budget warning", {"batch_id": bid, "warning": tb["warning"]})
|
||||
|
||||
return self._llm_service.call_llm_for_batch(
|
||||
logger.reason(
|
||||
f"LLM process batch start", {
|
||||
"batch_id": bid, "llm_rows": len(rows_for_llm),
|
||||
"provider_model": provider_model,
|
||||
"max_output_needed": tb.get("max_output_needed"),
|
||||
"estimated_input_tokens": tb.get("estimated_input_tokens"),
|
||||
},
|
||||
)
|
||||
|
||||
result = self._llm_service.call_llm_for_batch(
|
||||
job=job, run_id=run_id, batch_rows=rows_for_llm,
|
||||
dict_matches=dict_matches, batch_id=bid,
|
||||
max_tokens=tb["max_output_needed"],
|
||||
)
|
||||
|
||||
logger.reason(
|
||||
f"LLM process batch complete", {
|
||||
"batch_id": bid, **result,
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
# -- Batch insert (delegation) --
|
||||
def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
|
||||
insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
|
||||
|
||||
@@ -57,6 +57,15 @@ class LLMTranslationService:
|
||||
) -> dict[str, int]:
|
||||
"""Call LLM for a batch of rows; parse response; create records."""
|
||||
with belief_scope("LLMTranslationService.call_llm_for_batch"):
|
||||
provider_label = f"{job.provider_id}/{getattr(job, '_provider_model', '?')}"
|
||||
logger.reason(
|
||||
f"LLM batch start", {
|
||||
"batch_id": batch_id, "row_count": len(batch_rows),
|
||||
"provider": provider_label, "max_tokens": max_tokens,
|
||||
"recursion_depth": _recursion_depth,
|
||||
},
|
||||
)
|
||||
|
||||
dictionary_section = self._build_dictionary_section(dict_matches, batch_rows)
|
||||
target_languages = self._resolve_target_languages(job)
|
||||
prompt = self._build_prompt(job, batch_rows, dictionary_section, target_languages)
|
||||
@@ -65,10 +74,22 @@ class LLMTranslationService:
|
||||
job, prompt, batch_id, max_tokens,
|
||||
)
|
||||
if llm_response is None:
|
||||
logger.explore(
|
||||
f"LLM batch failed after {retries} retries", {
|
||||
"batch_id": batch_id, "row_count": len(batch_rows),
|
||||
"last_error": last_error, "retries": retries,
|
||||
},
|
||||
)
|
||||
return self._handle_llm_failure(batch_rows, run_id, batch_id, retries, last_error)
|
||||
|
||||
if finish_reason == "length" and len(batch_rows) >= 2 and run_id:
|
||||
if _recursion_depth < MAX_RETRIES_PER_BATCH:
|
||||
logger.reason(
|
||||
f"Splitting truncated batch", {
|
||||
"batch_id": batch_id, "size": len(batch_rows),
|
||||
"depth": _recursion_depth,
|
||||
},
|
||||
)
|
||||
return self._split_and_retry(job, run_id, batch_rows, dict_matches,
|
||||
batch_id, max_tokens, _recursion_depth, retries)
|
||||
logger.explore("Truncation recursion depth exceeded", {"batch_id": batch_id, "depth": _recursion_depth})
|
||||
@@ -76,11 +97,24 @@ class LLMTranslationService:
|
||||
try:
|
||||
translations = parse_llm_response(llm_response, len(batch_rows), target_languages=target_languages)
|
||||
except ValueError as e:
|
||||
logger.explore(
|
||||
f"LLM parse failure", {
|
||||
"batch_id": batch_id, "error": str(e),
|
||||
"response_len": len(llm_response),
|
||||
"response_preview": llm_response[:500],
|
||||
},
|
||||
)
|
||||
return self._handle_parse_failure(batch_rows, run_id, batch_id, retries, e)
|
||||
|
||||
return self._create_records_from_translations(
|
||||
result = self._create_records_from_translations(
|
||||
batch_rows, run_id, batch_id, target_languages, translations, dict_matches, retries,
|
||||
)
|
||||
logger.reason(
|
||||
f"LLM batch complete", {
|
||||
"batch_id": batch_id, **result,
|
||||
},
|
||||
)
|
||||
return result
|
||||
# #endregion call_llm_for_batch
|
||||
|
||||
def _build_dictionary_section(self, dict_matches, batch_rows) -> str:
|
||||
@@ -120,9 +154,16 @@ class LLMTranslationService:
|
||||
last_error = None
|
||||
retries = 0
|
||||
finish_reason = None
|
||||
logger.reason(f"LLM retry loop start", {"batch_id": batch_id, "max_retries": MAX_RETRIES_PER_BATCH, "prompt_len": len(prompt)})
|
||||
for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):
|
||||
try:
|
||||
llm_response, finish_reason = self.call_llm(job, prompt, max_tokens=max_tokens)
|
||||
logger.reason(
|
||||
f"LLM call succeeded (attempt {attempt})", {
|
||||
"batch_id": batch_id, "finish_reason": finish_reason,
|
||||
"response_len": len(llm_response) if llm_response else 0,
|
||||
},
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
@@ -130,6 +171,8 @@ class LLMTranslationService:
|
||||
logger.explore(f"LLM call failed (attempt {attempt})", {"batch_id": batch_id, "error": last_error})
|
||||
if attempt < MAX_RETRIES_PER_BATCH:
|
||||
time.sleep(2 ** attempt)
|
||||
if llm_response is None:
|
||||
logger.explore(f"All LLM retries exhausted", {"batch_id": batch_id, "retries": retries, "last_error": last_error})
|
||||
return llm_response, finish_reason, retries, last_error
|
||||
|
||||
def _handle_llm_failure(self, batch_rows, run_id, batch_id, retries, last_error):
|
||||
@@ -292,12 +335,27 @@ class LLMTranslationService:
|
||||
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
|
||||
disable_reasoning = getattr(job, 'disable_reasoning', False)
|
||||
|
||||
logger.reason(
|
||||
f"LLM provider resolved", {
|
||||
"provider_id": job.provider_id, "model": model,
|
||||
"provider_type": provider_type, "base_url": provider.base_url,
|
||||
"disable_reasoning": disable_reasoning, "max_tokens": max_tokens,
|
||||
},
|
||||
)
|
||||
|
||||
if provider_type not in ("openai", "openai_compatible", "openrouter", "kilo", "litellm"):
|
||||
raise ValueError(f"Unsupported provider type '{provider_type}'")
|
||||
return call_openai_compatible(
|
||||
result = call_openai_compatible(
|
||||
base_url=provider.base_url, api_key=api_key, model=model, prompt=prompt,
|
||||
provider_type=provider_type, max_tokens=max_tokens, disable_reasoning=disable_reasoning,
|
||||
)
|
||||
logger.reason(
|
||||
f"LLM provider call complete", {
|
||||
"response_len": len(result[0]) if result and result[0] else 0,
|
||||
"finish_reason": result[1],
|
||||
},
|
||||
)
|
||||
return result
|
||||
# #endregion call_llm
|
||||
|
||||
# -- Static methods delegated to sub-modules for backward compat --
|
||||
|
||||
Reference in New Issue
Block a user