busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(translate): partial JSON recovery for truncated LLM responses when DeepSeek runs out of tokens

Browse Source 
DeepSeek v4 Flash returns finish_reason=length when reasoning uses
all output tokens. Partial row extraction via regex recovers complete
rows from truncated JSON instead of failing with 400 error.

Also: extra_body reasoning_effort for Kilo/OpenRouter proxied providers
This commit is contained in:
busya
2026-05-15 10:31:51 +03:00
parent ff4804164a
commit 70aec51e0b
2 changed files with 61 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
backend/src/plugins/translate/executor.py
View File

@@ -985,7 +985,8 @@ class TranslationExecutor:
# Works universally via system prompt + API parameter for models that support it
if disable_reasoning:
payload["reasoning_effort"] = "none"
# Universal instruction — all models understand "respond directly without reasoning"
payload["extra_body"] = {"reasoning_effort": "none"}
payload.pop("response_format", None)
payload["messages"][0] = {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."}
logger.reason(
@@ -1027,7 +1028,7 @@ class TranslationExecutor:
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language codes.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None, finish_reason: str | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationExecutor._parse_llm_response"):
try:
data = json.loads(response_text)
@@ -1038,11 +1039,33 @@ class TranslationExecutor:
if match:
try:
data = json.loads(match.group(1))
rows = data.get("rows", [])
if isinstance(rows, list) and rows:
logger.reason("Parsed JSON from markdown code block", {"rows": len(rows)})
except json.JSONDecodeError:
logger.explore("LLM response was not valid JSON (after markdown extraction)", extra={"src": "executor", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON")
pass # fall through to partial recovery
else:
logger.explore("LLM response was not valid JSON (no markdown block)", extra={"src": "executor", "response_preview": response_text[:1000]})
pass # fall through to partial recovery
# If finish_reason=length, try to recover complete rows from truncated JSON
logger.explore("LLM truncated, trying partial row recovery", extra={"src": "executor", "finish_reason": finish_reason, "response_length": len(response_text)})
rows_match = re.findall(r'\{\s*"row_id"\s*:\s*"\d+".*?\}\s*', response_text, re.DOTALL)
if rows_match:
partial_rows = []
for row_text in rows_match:
try:
row_data = json.loads(row_text)
partial_rows.append(row_data)
except json.JSONDecodeError:
continue
if partial_rows:
logger.explore(f"Recovered {len(partial_rows)}/{expected_count} complete rows from truncated response", extra={"src": "executor"})
data = {"rows": partial_rows}
else:
logger.explore("Could not recover any complete rows", extra={"src": "executor", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON (could not recover any rows)")
else:
logger.explore("No complete rows found in truncated response", extra={"src": "executor", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON")
rows = data.get("rows", [])

38
backend/src/plugins/translate/preview.py
View File

@@ -1019,7 +1019,12 @@ class TranslationPreview:
# Suppress Chain of Thought reasoning to save output tokens
# Works universally via system prompt + API parameter for models that support it
if disable_reasoning:
payload["reasoning_effort"] = "none"
# Try multiple methods to suppress reasoning (varies by provider/deployment)
payload["reasoning_effort"] = "none" # DeepSeek, Qwen
payload["extra_body"] = {"reasoning_effort": "none"} # Kilo/OpenRouter proxy
payload.pop("response_format", None) # JSON mode triggers reasoning on some models
# Max tokens must be large enough for output even with some reasoning
payload["max_tokens"] = 8192
# Universal instruction — all models understand "respond directly without reasoning"
system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."
payload["messages"][0] = {"role": "system", "content": system_content}
@@ -1028,6 +1033,7 @@ class TranslationPreview:
f"LLM request model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
f"reasoning={'no' if disable_reasoning else 'yes'} "
f"prompt_len={len(prompt)}"
)
response = http_requests.post(url, headers=headers, json=payload, timeout=120)
@@ -1061,7 +1067,7 @@ class TranslationPreview:
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
# @POST: Returns dict mapping row_id to dict with 'detected_source_language' and per-language keys.
@staticmethod
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None) -> dict[str, dict[str, str]]:
def _parse_llm_response(response_text: str, expected_count: int, target_languages: list[str] | None = None, finish_reason: str | None = None) -> dict[str, dict[str, str]]:
with belief_scope("TranslationPreview._parse_llm_response"):
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
@@ -1074,11 +1080,33 @@ class TranslationPreview:
if match:
try:
data = json.loads(match.group(1))
rows = data.get("rows", [])
if isinstance(rows, list) and rows:
logger.reason("Parsed JSON from markdown code block", {"rows": len(rows)})
except json.JSONDecodeError:
logger.explore("LLM response was not valid JSON (after markdown extraction)", extra={"src": "preview", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON")
pass # fall through to partial recovery
else:
logger.explore("LLM response was not valid JSON (no markdown block)", extra={"src": "preview", "response_preview": response_text[:1000]})
pass # fall through to partial recovery
# If finish_reason=length, try to recover complete rows from truncated JSON
logger.explore("LLM truncated, trying partial row recovery", extra={"src": "preview", "finish_reason": finish_reason, "response_length": len(response_text)})
rows_match = re.findall(r'\{\s*"row_id"\s*:\s*"\d+".*?\}\s*', response_text, re.DOTALL)
if rows_match:
partial_rows = []
for row_text in rows_match:
try:
row_data = json.loads(row_text)
partial_rows.append(row_data)
except json.JSONDecodeError:
continue
if partial_rows:
logger.explore(f"Recovered {len(partial_rows)}/{expected_count} complete rows from truncated response", extra={"src": "preview"})
data = {"rows": partial_rows}
else:
logger.explore("Could not recover any complete rows", extra={"src": "preview", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON (could not recover any rows)")
else:
logger.explore("No complete rows found in truncated response", extra={"src": "preview", "response_preview": response_text[:1000]})
raise ValueError("LLM response was not valid JSON")
rows = data.get("rows", [])