busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(translate): fix slow translation startup — CJK estimation, output budget, provider token config

Browse Source 
Root cause: batch sizing underestimated CJK token density (1.5→1.0 chars/token)
and ignored output budget as primary constraint, causing cascading finish_reason=length.

Changes:
- _token_budget.py: CJK_RATIO 1.5→1.0, OTHER_RATIO 2.2→1.8, safety factors 0.75/0.70
- _token_budget.py: new _compute_max_rows_by_output() — output budget is PRIMARY constraint
- _batch_sizer.py: resolve_provider_config() with DB-level context_window/max_output_tokens
- _batch_sizer.py: INPUT_SAFETY_FACTOR applied, max_rows_by_output used as row cap
- _llm_http.py: log actual usage.prompt_tokens/.completion_tokens from provider
- _llm_call.py: retry only missing rows after finish_reason=length (save partial result)
- models/llm.py + schema: provider-level context_window / max_output_tokens (nullable)
- services/llm_provider.py: get_provider_token_config() helper
- Alembic migration: add columns to llm_providers
- Svelte ProviderConfig: collapsible Advanced: Token Limits section
- 12 new tests (token budget, batch sizer, provider config)
- All 492 tests pass
This commit is contained in:
busya
2026-06-03 23:25:08 +03:00
parent a819e1ec4d
commit 814f2da139
18 changed files with 1303 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py Normal file
View File

@@ -0,0 +1,48 @@
"""Add context_window and max_output_tokens to llm_providers
Revision ID: a1b2c3d4e5f6
Revises: f1a2b3c4d5e6
Create Date: 2026-06-03
Add token window configuration to LLM provider records:
- context_window: total context window in tokens (nullable)
- max_output_tokens: max output tokens limit (nullable)
Both NULL = use PROVIDER_DEFAULTS fallback from model name.
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "a1b2c3d4e5f6"
down_revision: Union[str, None] = "f1a2b3c4d5e6"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"llm_providers",
sa.Column(
"context_window",
sa.Integer(),
nullable=True,
comment="Total context window in tokens. NULL = auto-detect from model name",
),
)
op.add_column(
"llm_providers",
sa.Column(
"max_output_tokens",
sa.Integer(),
nullable=True,
comment="Max output tokens limit. NULL = auto-detect from model name",
),
)
def downgrade() -> None:
op.drop_column("llm_providers", "max_output_tokens")
op.drop_column("llm_providers", "context_window")

6
backend/src/api/routes/llm.py
View File

@@ -85,6 +85,8 @@ async def get_providers(
is_active=p.is_active,
is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False,
max_images=p.max_images,
context_window=p.context_window,
max_output_tokens=p.max_output_tokens,
)
for p in providers
]
@@ -272,6 +274,8 @@ async def create_provider(
is_active=provider.is_active,
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
max_images=provider.max_images,
context_window=provider.context_window,
max_output_tokens=provider.max_output_tokens,
)
@@ -309,6 +313,8 @@ async def update_provider(
is_active=provider.is_active,
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
max_images=provider.max_images,
context_window=provider.context_window,
max_output_tokens=provider.max_output_tokens,
)

2
backend/src/api/routes/settings.py
View File

@@ -456,6 +456,8 @@ async def get_consolidated_settings(
"default_model": p.default_model,
"is_active": p.is_active,
"is_multimodal": bool(p.is_multimodal) if p.is_multimodal is not None else False,
"context_window": p.context_window,
"max_output_tokens": p.max_output_tokens,
}
for p in providers
]

8
backend/src/models/llm.py
View File

@@ -66,6 +66,14 @@ class LLMProvider(Base):
is_active = Column(Boolean, default=True)
is_multimodal = Column(Boolean, default=False)
max_images = Column(Integer, nullable=True, default=None)
context_window = Column(
Integer, nullable=True, default=None,
comment="Total context window in tokens. NULL = fallback to PROVIDER_DEFAULTS from model name.",
)
max_output_tokens = Column(
Integer, nullable=True, default=None,
comment="Max output tokens limit. NULL = fallback to PROVIDER_DEFAULTS from model name.",
)
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
# #endregion LLMProvider

8
backend/src/plugins/llm_analysis/models.py
View File

@@ -30,6 +30,14 @@ class LLMProviderConfig(BaseModel):
is_active: bool = True
is_multimodal: bool = False
max_images: int | None = None
context_window: int | None = Field(
None, ge=1000, le=256000,
description="Context window in tokens. Leave blank for auto-detection.",
)
max_output_tokens: int | None = Field(
None, ge=256,
description="Max output tokens. Must be less than context_window.",
)
# #endregion LLMProviderConfig
# #region ValidationStatus [TYPE Class]

89
backend/src/plugins/translate/__tests__/test_batch_sizer.py Normal file
View File

@@ -0,0 +1,89 @@
# #region TestAdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS test, batch, sizer, provider, config]
# @BRIEF Verify AdaptiveBatchSizer contracts — provider config resolution, safety factor, row cap.
# @RELATION BINDS_TO -> [AdaptiveBatchSizer]
# @TEST_EDGE resolve_provider_config_no_provider — no provider_id returns all-None
# @TEST_EDGE resolve_provider_config_with_provider — returns model + token limits from DB
# @TEST_EDGE resolve_provider_config_exception — DB error returns all-None gracefully
from unittest.mock import MagicMock, patch
from src.models.translate import TranslationJob
from src.plugins.translate._batch_sizer import AdaptiveBatchSizer
# region TestResolveProviderConfig [TYPE Class]
# @BRIEF Tests for AdaptiveBatchSizer.resolve_provider_config.
class TestResolveProviderConfig:
# region test_no_provider_id [TYPE Function]
# @BRIEF When job has no provider_id, returns all-None dict.
def test_no_provider_id(self):
job = MagicMock(spec=TranslationJob)
job.provider_id = None
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
result = sizer.resolve_provider_config(job)
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
# endregion test_no_provider_id
# region test_with_provider_full_config [TYPE Function]
# @BRIEF Provider with context_window and max_output_tokens returns all values.
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
def test_with_provider_full_config(self, mock_provider_svc):
job = MagicMock(spec=TranslationJob)
job.provider_id = "provider-1"
mock_svc_instance = MagicMock()
mock_svc_instance.get_provider_token_config.return_value = {
"model": "gpt-4o-mini",
"context_window": 128000,
"max_output_tokens": 16384,
}
mock_provider_svc.return_value = mock_svc_instance
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
result = sizer.resolve_provider_config(job)
assert result["model"] == "gpt-4o-mini"
assert result["context_window"] == 128000
assert result["max_output_tokens"] == 16384
# endregion test_with_provider_full_config
# region test_with_provider_null_config [TYPE Function]
# @BRIEF Provider with NULL token limits still returns model + None values.
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
def test_with_provider_null_config(self, mock_provider_svc):
job = MagicMock(spec=TranslationJob)
job.provider_id = "provider-2"
mock_svc_instance = MagicMock()
mock_svc_instance.get_provider_token_config.return_value = {
"model": "deepseek-v4-flash",
"context_window": None,
"max_output_tokens": None,
}
mock_provider_svc.return_value = mock_svc_instance
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
result = sizer.resolve_provider_config(job)
assert result["model"] == "deepseek-v4-flash"
assert result["context_window"] is None # NULL → use PROVIDER_DEFAULTS
assert result["max_output_tokens"] is None
# endregion test_with_provider_null_config
# region test_exception_returns_safe_defaults [TYPE Function]
# @BRIEF DB exception returns all-None dict gracefully.
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
def test_exception_returns_safe_defaults(self, mock_provider_svc):
job = MagicMock(spec=TranslationJob)
job.provider_id = "provider-3"
mock_svc_instance = MagicMock()
mock_svc_instance.get_provider_token_config.side_effect = Exception("DB error")
mock_provider_svc.return_value = mock_svc_instance
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
result = sizer.resolve_provider_config(job)
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
# endregion test_exception_returns_safe_defaults
# endregion TestResolveProviderConfig
# #endregion TestAdaptiveBatchSizer

24
backend/src/plugins/translate/__tests__/test_executor.py
View File

@@ -307,12 +307,12 @@ class TestEstimateRowTokens:
# region test_cjk_text [TYPE Function]
def test_cjk_text(self) -> None:
"""CJK characters are token-denser (~1.5 chars/token)."""
"""CJK characters are token-denser (~1.0 chars/token with conservative estimate)."""
job = MagicMock(spec=TranslationJob)
job.context_columns = []
# 12 CJK chars → 12/1.5 = 8 tokens, plus 1 for empty context = 9
# 12 CJK chars → 12/1.0 = 12 tokens (conservative), plus 1 for context = 13
tokens = estimate_row_tokens("你好世界这是一个测试消息", None, job)
assert tokens == 9, f"Expected 9 tokens (8 CJK + 1 empty ctx) for CJK, got {tokens}"
assert tokens == 13, f"Expected 13 tokens (12 CJK + 1 ctx) for CJK, got {tokens}"
# endregion test_cjk_text
@@ -589,24 +589,26 @@ class TestAutoSizeBatches:
executor: TranslationExecutor,
job: MagicMock,
) -> None:
"""When provider_info is None, _resolve_provider_model is called."""
"""When provider_info is None, AdaptiveBatchSizer resolves provider config internally."""
mock_estimate.return_value = {
"batch_size_adjusted": 10,
"estimated_input_tokens": 5000,
"estimated_output_tokens": 2000,
"max_output_needed": 4096,
"warning": None,
"max_rows_by_output": 20,
"available_input_budget": 47616,
"max_output_tokens": 16384,
}
source_rows = [
{"row_index": "0", "source_text": "hello", "source_data": None},
]
# provider_info=None → should call _resolve_provider_model
with patch.object(executor, '_resolve_provider_model', return_value="gpt-4o-mini") as mock_resolve:
batches = executor._auto_size_batches(
job, source_rows, ["en"], provider_info=None,
)
mock_resolve.assert_called_once_with(job)
assert len(batches) == 1
# provider_info=None → AdaptiveBatchSizer resolves config from job.provider_id
# No longer calls executor._resolve_provider_model
batches = executor._auto_size_batches(
job, source_rows, ["en"], provider_info=None,
)
assert len(batches) == 1
# endregion test_provider_info_resolution

93
backend/src/plugins/translate/__tests__/test_token_budget.py
View File

@@ -14,7 +14,15 @@
# @TEST_INVARIANT max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
# @TEST_INVARIANT warning is None when batch fits, str when reduced
from src.plugins.translate._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, estimate_token_budget
from src.plugins.translate._token_budget import (
CJK_RATIO,
DEFAULT_CONTEXT_WINDOW,
DEFAULT_MAX_OUTPUT_TOKENS,
OTHER_RATIO,
estimate_token_budget,
_compute_max_rows_by_output,
_estimate_tokens_for_text,
)
# region _make_row [TYPE Function]
@@ -26,10 +34,91 @@ def _make_row(text: str, **context) -> dict:
# endregion _make_row
# region _make_cjk_row [TYPE Function]
# @BRIEF Create a test source row with CJK text.
def _make_cjk_row() -> dict:
return {"source_text": "你好世界这是一个测试消息", "row_index": "0"}
# endregion _make_cjk_row
# region TestTokenBudget [TYPE Class]
# @BRIEF Test suite for estimate_token_budget.
# @BRIEF Test suite for estimate_token_budget and related token estimation functions.
class TestTokenBudget:
# region test_cjk_ratio_estimate [TYPE Function]
# @BRIEF Verify CJK token estimation uses the conservative CJK_RATIO.
def test_cjk_ratio_estimate(self):
"""12 CJK chars / 1.0 = 12 tokens (conservative), plus 1 for empty context."""
tokens = _estimate_tokens_for_text("你好世界这是一个测试消息")
expected = int(12 / CJK_RATIO)
assert tokens == expected, f"CJK estimate {tokens} != expected {expected}"
# endregion test_cjk_ratio_estimate
# region test_mixed_text_estimate [TYPE Function]
# @BRIEF Verify mixed CJK+Ltn text estimation.
def test_mixed_text_estimate(self):
"""Mixed text: CJK at CJK_RATIO, Latin at OTHER_RATIO."""
text = "你好世界! Hello world, this is a test!"
tokens = _estimate_tokens_for_text(text)
# 4 CJK chars / 1.0 = 4, 30 non-CJK / 1.8 = 16, total = 20
assert tokens == 20, f"Expected 20, got {tokens}"
# endregion test_mixed_text_estimate
# region test_empty_text_estimate [TYPE Function]
# @BRIEF Empty text returns 1 token minimum.
def test_empty_text_estimate(self):
assert _estimate_tokens_for_text("") == 1
assert _estimate_tokens_for_text(None) == 1
# endregion test_empty_text_estimate
# region test_compute_max_rows_by_output_single_lang [TYPE Function]
# @BRIEF Single target language: compute max rows from output budget.
def test_compute_max_rows_by_output_single_lang(self):
"""With max_output_tokens=8192 and 1 language, should return limited rows."""
max_rows = _compute_max_rows_by_output(max_output_tokens=8192, num_languages=1)
assert max_rows >= 1
assert max_rows < 100 # Should be reasonable
# endregion test_compute_max_rows_by_output_single_lang
# region test_compute_max_rows_by_output_multi_lang [TYPE Function]
# @BRIEF Multiple target languages reduce max rows from output budget.
def test_compute_max_rows_by_output_multi_lang(self):
"""More languages = fewer rows that fit in the same output budget."""
single = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=1)
multi = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=4)
assert multi <= single
assert multi >= 1
# endregion test_compute_max_rows_by_output_multi_lang
# region test_compute_max_rows_by_output_small_budget [TYPE Function]
# @BRIEF Very small output budget returns at least 1 row.
def test_compute_max_rows_by_output_small_budget(self):
"""Even with minimal budget, at least 1 row fits."""
max_rows = _compute_max_rows_by_output(max_output_tokens=4096, num_languages=2)
assert max_rows >= 1
# endregion test_compute_max_rows_by_output_small_budget
# region test_max_rows_by_output_in_return_dict [TYPE Function]
# @BRIEF estimate_token_budget returns max_rows_by_output field.
def test_max_rows_by_output_in_return_dict(self):
"""The return dict includes max_rows_by_output as a positive int."""
rows = [_make_row("Short text.")] * 10
result = estimate_token_budget(rows, ["ru"], batch_size=10)
assert "max_rows_by_output" in result
assert isinstance(result["max_rows_by_output"], int)
assert result["max_rows_by_output"] >= 1
# endregion test_max_rows_by_output_in_return_dict
# region test_max_rows_by_output_changes_with_languages [TYPE Function]
# @BRIEF max_rows_by_output decreases with more target languages.
def test_max_rows_by_output_changes_with_languages(self):
"""More target languages = smaller max_rows_by_output."""
rows = [_make_row("Hello world")] * 10
single = estimate_token_budget(rows, ["ru"], batch_size=10)
multi = estimate_token_budget(rows, ["ru", "en", "fr", "de"], batch_size=10)
assert multi["max_rows_by_output"] <= single["max_rows_by_output"]
# endregion test_max_rows_by_output_changes_with_languages
# region test_small_rows_fit_at_requested_size [TYPE Function]
# @BRIEF Short text rows fill the requested batch_size without reduction.
def test_small_rows_fit_at_requested_size(self):

20
backend/src/plugins/translate/_batch_proc.py
View File

@@ -206,30 +206,36 @@ class BatchProcessingService:
return count
def _process_llm(self, job, run_id, rows_for_llm, dict_matches, bid, tls):
provider_model = None
# Resolve provider token config (DB values take priority over PROVIDER_DEFAULTS)
token_config = {"model": None, "context_window": None, "max_output_tokens": None}
if job.provider_id:
try:
p = LLMProviderService(self.db).get_provider(job.provider_id)
if p:
provider_model = p.default_model or "gpt-4o-mini"
token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id)
except Exception:
provider_model = None
pass
tb = estimate_token_budget(
source_rows=rows_for_llm, target_languages=tls,
source_column="source_text", context_columns=None,
dictionary_entries=dict_matches, batch_size=len(rows_for_llm),
provider_info=provider_model,
provider_info=token_config["model"],
context_window=token_config["context_window"],
max_output_tokens=token_config["max_output_tokens"],
)
if tb["warning"]:
logger.explore("Token budget warning", {"batch_id": bid, "warning": tb["warning"]})
max_rows_by_out = tb.get("max_rows_by_output", 0)
logger.reason(
f"LLM process batch start", {
"batch_id": bid, "llm_rows": len(rows_for_llm),
"provider_model": provider_model,
"provider_model": token_config["model"],
"max_output_needed": tb.get("max_output_needed"),
"estimated_input_tokens": tb.get("estimated_input_tokens"),
"max_rows_by_output": max_rows_by_out,
"context_window": token_config["context_window"],
"max_output_tokens": token_config["max_output_tokens"],
},
)

110
backend/src/plugins/translate/_batch_sizer.py
View File

@@ -1,13 +1,15 @@
# #region AdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS translate, batch, sizing, token-budget]
# @BRIEF Adaptive batch sizing for LLM translation — splits source rows into variable-sized
# batches based on actual content length and token budget estimates.
# Output budget is the PRIMARY constraint; input budget is secondary.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [estimate_token_budget]
# @RELATION DEPENDS_ON -> [LLMProviderService]
# @RELATION DEPENDS_ON -> [TranslationJob]
# @RATIONALE Extracted from TranslationExecutor to comply with INV_7 (module < 400 lines).
# Fixed batch_size of 50 wastes LLM context for short rows and overflows for
# long rows. Variable sizing maximizes throughput while preventing truncation.
# Safety factor applied to input budget to account for tokenizer variance.
# Output-aware row cap computed from estimate_token_budget result.
# Provider-level context_window/max_output_tokens take priority over PROVIDER_DEFAULTS.
# @REJECTED Fixed batch_size of 50 — causes truncation on long-content rows.
# Single monolithic batch — would lose all progress on any failure.
@@ -19,11 +21,8 @@ from ...core.logger import belief_scope, logger
from ...models.translate import TranslationJob
from ...services.llm_provider import LLMProviderService
from ._token_budget import (
JSON_OVERHEAD_PER_ROW,
MAX_OUTPUT_HEADROOM,
OUTPUT_PER_ROW_PER_LANG,
INPUT_SAFETY_FACTOR,
PROMPT_BASE_TOKENS,
REASONING_OVERHEAD,
estimate_token_budget,
)
from ._utils import estimate_row_tokens
@@ -32,41 +31,33 @@ from ._utils import estimate_row_tokens
# #region AdaptiveBatchSizer [C:3] [TYPE Class]
# @BRIEF Split source rows into auto-sized batches based on token budget estimates.
class AdaptiveBatchSizer:
"""Split source rows into auto-sized batches based on token budget estimates.
Each batch is sized so that its total estimated tokens fit within the
available context window (input budget), accounting for prompt overhead,
dictionary entries, and output tokens.
"""
def __init__(self, db: Session, config_manager: ConfigManager) -> None:
self.db = db
self.config_manager = config_manager
# #region resolve_provider_model [C:2] [TYPE Function] [SEMANTICS llm, provider, model]
# @BRIEF Resolve the LLM provider model name for token budget estimation.
# @POST Returns model name string or None if resolution fails.
# #region resolve_provider_config [C:2] [TYPE Function] [SEMANTICS llm, provider, model, token-config]
# @BRIEF Resolve provider token config (model name + token limits) for budget estimation.
# DB values (context_window, max_output_tokens) take priority over PROVIDER_DEFAULTS.
# @POST Returns dict with model, context_window, max_output_tokens. None values = use fallback.
# @SIDE_EFFECT DB query to LLM provider table.
def resolve_provider_model(self, job: TranslationJob) -> str | None:
"""Resolve the provider model name for token budget estimation."""
def resolve_provider_config(self, job: TranslationJob) -> dict:
"""Resolve provider token config for token budget estimation."""
if not job.provider_id:
return None
return {"model": None, "context_window": None, "max_output_tokens": None}
try:
p_svc = LLMProviderService(self.db)
p = p_svc.get_provider(job.provider_id)
if p:
return p.default_model or "gpt-4o-mini"
token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id)
return token_config
except Exception:
pass
return None
# #endregion resolve_provider_model
return {"model": None, "context_window": None, "max_output_tokens": None}
# #endregion resolve_provider_config
# #region auto_size_batches [C:3] [TYPE Function] [SEMANTICS translate, batch, sizing]
# @BRIEF Split source rows into variable-sized batches based on content length.
# Output budget (max_rows_by_output) is the PRIMARY row-count constraint.
# @PRE source_rows is non-empty. job has valid config.
# @POST Returns list of batches, each batch is a list of row dicts.
# Each batch fits within the estimated token budget for its rows.
# @SIDE_EFFECT DB query to resolve provider model. Logs batch statistics.
# @POST Returns list of batches. Each batch fits within estimated input token budget
# with INPUT_SAFETY_FACTOR headroom AND within output row cap.
# @SIDE_EFFECT DB query to resolve provider config. Logs batch statistics.
def auto_size_batches(
self,
job: TranslationJob,
@@ -74,18 +65,14 @@ class AdaptiveBatchSizer:
target_languages: list[str],
provider_info: str | None = None,
) -> list[list[dict]]:
"""Split source rows into auto-sized batches based on content length.
Each batch is sized so that its total estimated tokens fit within the
available context window (input budget), accounting for prompt overhead,
dictionary entries, and output tokens.
"""
with belief_scope("AdaptiveBatchSizer.auto_size_batches"):
if not source_rows:
return []
if provider_info is None:
provider_info = self.resolve_provider_model(job)
# Resolve provider config (DB values > PROVIDER_DEFAULTS)
token_config = self.resolve_provider_config(job)
if provider_info is not None and token_config["model"] is None:
token_config["model"] = provider_info
# 1. Estimate per-row token counts
row_tokens: list[int] = []
@@ -95,14 +82,16 @@ class AdaptiveBatchSizer:
tokens = estimate_row_tokens(source_text, source_data, job)
row_tokens.append(tokens)
# 2. Get budget recommendation
# 2. Get budget recommendation (uses provider config from DB if available)
budget = estimate_token_budget(
source_rows=source_rows,
target_languages=target_languages,
source_column=job.translation_column or "source_text",
context_columns=job.context_columns,
batch_size=len(source_rows),
provider_info=provider_info,
provider_info=token_config["model"],
context_window=token_config["context_window"],
max_output_tokens=token_config["max_output_tokens"],
)
recommended = budget.get("batch_size_adjusted", 0)
@@ -119,18 +108,15 @@ class AdaptiveBatchSizer:
for i in range(0, len(source_rows), fallback_size)
]
# 3. Compute per-batch row-content budget
# Use the ACTUAL available input capacity (context_window - max_output_tokens),
# NOT the sum of the first N rows. This prevents long rows from being
# incorrectly placed in 1-row batches when the context window has plenty of room.
# 3. Compute per-batch row-content budget with safety factor
available_input = budget.get("available_input_budget")
if available_input is not None:
# New-style: use actual available input capacity
per_batch_budget = available_input - PROMPT_BASE_TOKENS
raw_budget = available_input - PROMPT_BASE_TOKENS
per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR)
else:
# Fallback for tests: use estimated_input (sum of first N rows)
estimated_input = budget.get("estimated_input_tokens", 50000)
per_batch_budget = estimated_input - PROMPT_BASE_TOKENS
raw_budget = estimated_input - PROMPT_BASE_TOKENS
per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR)
if per_batch_budget <= 0:
fallback_size = job.batch_size or 50
@@ -144,27 +130,18 @@ class AdaptiveBatchSizer:
for i in range(0, len(source_rows), fallback_size)
]
# 4. Greedy batch splitting
# Compute max rows per batch from OUTPUT constraint.
# This prevents output truncation (finish_reason=length) when batching
# many short rows within the large input budget.
num_languages = len(target_languages)
max_output_tokens_val = budget.get("max_output_tokens")
if max_output_tokens_val is not None:
output_per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
available_output = max_output_tokens_val - REASONING_OVERHEAD - MAX_OUTPUT_HEADROOM
max_rows_by_output = max(available_output // output_per_row, 1) if output_per_row > 0 else 20
max_rows_hard_cap = max_rows_by_output
else:
# Fallback for tests: old formula
max_rows_hard_cap = max(recommended * 2, 20)
# 4. Compute max rows per batch — OUTPUT BUDGET IS PRIMARY CONSTRAINT
max_rows_by_output = budget.get("max_rows_by_output", 20)
# 5. Respect job.batch_size as the absolute maximum rows per batch.
# User-configured batch_size overrides model-based estimates to
# prevent LLM quality degradation on large batches.
# Absolute hard cap as safety net (not the primary constraint)
absolute_hard_cap = 50
max_rows_hard_cap = min(max_rows_by_output, absolute_hard_cap)
# Respect job.batch_size as user override
if job.batch_size:
max_rows_hard_cap = min(max_rows_hard_cap, job.batch_size)
# 5. Greedy batch splitting with output constraint
batches: list[list[dict]] = []
current_batch: list[dict] = []
current_tokens = 0
@@ -202,7 +179,7 @@ class AdaptiveBatchSizer:
if current_batch:
batches.append(current_batch)
# 5. Log adaptive batch statistics
# 6. Log stats
if batches:
avg_size = sum(len(b) for b in batches) / max(1, len(batches))
max_size = max(len(b) for b in batches)
@@ -213,6 +190,9 @@ class AdaptiveBatchSizer:
"max_batch_size": max_size,
"recommended_batch_size": recommended,
"per_batch_budget": per_batch_budget,
"safety_factor": INPUT_SAFETY_FACTOR,
"max_rows_by_output": max_rows_by_output,
"max_rows_hard_cap": max_rows_hard_cap,
})
return batches

176
backend/src/plugins/translate/_llm_call.py
View File

@@ -40,7 +40,6 @@ MAX_RETRIES_PER_BATCH = 3
# #region LLMTranslationService [C:4] [TYPE Class]
# @BRIEF Call LLM, handle retry/truncation, parse response, persist records.
class LLMTranslationService:
"""LLM interaction for batch translation with retry, truncation handling, and parsing."""
def __init__(self, db: Session) -> None:
self.db = db
@@ -55,7 +54,6 @@ class LLMTranslationService:
batch_rows: list[dict[str, Any]], dict_matches: list[dict[str, Any]],
batch_id: str, max_tokens: int = 8192, _recursion_depth: int = 0,
) -> dict[str, int]:
"""Call LLM for a batch of rows; parse response; create records."""
with belief_scope("LLMTranslationService.call_llm_for_batch"):
provider_label = f"{job.provider_id}/{getattr(job, '_provider_model', '?')}"
logger.reason(
@@ -83,6 +81,34 @@ class LLMTranslationService:
return self._handle_llm_failure(batch_rows, run_id, batch_id, retries, last_error)
if finish_reason == "length" and len(batch_rows) >= 2 and run_id:
# Try recovery first: parse partial response, save recovered rows,
# retry only missing rows. This avoids binary-splitting already-translated rows.
recovered_ids = self._try_recover_partial(
llm_response, batch_rows, run_id, batch_id, target_languages,
)
if recovered_ids:
remaining = [
r for r in batch_rows
if str(r.get("row_index", "")) not in recovered_ids
]
if remaining and len(remaining) < len(batch_rows):
logger.reason(
f"Retrying only {len(remaining)}/{len(batch_rows)} missing rows",
{"batch_id": batch_id, "recovered": len(recovered_ids), "remaining": len(remaining)},
)
return self._retry_missing_rows(
job, run_id, remaining, dict_matches,
batch_id, max_tokens, _recursion_depth,
)
# All rows recovered — nothing to retry
if not remaining:
logger.reason(
"All rows recovered from truncated response",
{"batch_id": batch_id, "recovered": len(recovered_ids)},
)
return {"successful": len(recovered_ids), "failed": 0, "skipped": 0, "retries": 0}
# Fall back to binary split if recovery didn't help
if _recursion_depth < MAX_RETRIES_PER_BATCH:
logger.reason(
f"Splitting truncated batch", {
@@ -117,6 +143,8 @@ class LLMTranslationService:
return result
# #endregion call_llm_for_batch
# #region _build_dictionary_section [C:2] [TYPE Function] [SEMANTICS translate, llm, dictionary]
# @BRIEF Build dictionary section string for LLM prompt from matched entries.
def _build_dictionary_section(self, dict_matches, batch_rows) -> str:
if not dict_matches:
return ""
@@ -124,12 +152,18 @@ class LLMTranslationService:
annotated = ContextAwarePromptBuilder.build_context_entries(dict_matches, row_context)
return "Terminology dictionary (use these translations when applicable):\n" + \
"\n".join(f"- {a}" for a in annotated) + "\n\n"
# #endregion _build_dictionary_section
# #region _resolve_target_languages [C:2] [TYPE Function] [SEMANTICS translate, llm, languages]
# @BRIEF Resolve target language list from job config.
@staticmethod
def _resolve_target_languages(job):
langs = job.target_languages or [job.target_dialect or "en"]
return [str(langs)] if not isinstance(langs, list) else langs
# #endregion _resolve_target_languages
# #region _build_prompt [C:2] [TYPE Function] [SEMANTICS translate, llm, prompt]
# @BRIEF Build the full LLM prompt from batch rows, dictionary, and target languages.
@staticmethod
def _build_prompt(job, batch_rows, dictionary_section, target_languages):
target_languages_str = ", ".join(target_languages)
@@ -148,7 +182,11 @@ class LLMTranslationService:
"rows_json": rows_json,
"row_count": str(len(batch_rows)),
})
# #endregion _build_prompt
# #region _call_llm_with_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, retry]
# @BRIEF Call LLM with retry loop (max 3 attempts, exponential backoff).
# @SIDE_EFFECT HTTP calls to LLM provider on each attempt.
def _call_llm_with_retry(self, job, prompt, batch_id, max_tokens):
llm_response = None
last_error = None
@@ -174,7 +212,11 @@ class LLMTranslationService:
if llm_response is None:
logger.explore(f"All LLM retries exhausted", {"batch_id": batch_id, "retries": retries, "last_error": last_error})
return llm_response, finish_reason, retries, last_error
# #endregion _call_llm_with_retry
# #region _handle_llm_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, failure-handling]
# @BRIEF Handle complete LLM failure — mark all batch rows as FAILED.
# @SIDE_EFFECT DB writes for each row in the batch.
def _handle_llm_failure(self, batch_rows, run_id, batch_id, retries, last_error):
for row in batch_rows:
self.db.add(TranslationRecord(
@@ -186,7 +228,11 @@ class LLMTranslationService:
error_message=f"LLM call failed after {retries} retries: {last_error}",
))
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
# #endregion _handle_llm_failure
# #region _split_and_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, split, retry]
# @BRIEF Binary-split a batch and retry each half recursively.
# @SIDE_EFFECT Creates two child TranslationBatch rows; recursive LLM calls.
def _split_and_retry(self, job, run_id, batch_rows, dict_matches, batch_id, max_tokens, depth, retries):
mid = len(batch_rows) // 2
logger.explore("LLM output truncated — splitting batch",
@@ -232,6 +278,112 @@ class LLMTranslationService:
"skipped": left["skipped"] + right["skipped"],
"retries": retries + left.get("retries", 0) + right.get("retries", 0)}
# #region _try_recover_partial [C:3] [TYPE Function] [SEMANTICS translate, llm, recovery]
# @BRIEF Try to recover translated rows from a truncated LLM response.
# Saves recovered rows as SUCCESS records. Returns set of recovered row_index values.
# @PRE llm_response is valid text (possibly truncated JSON). batch_rows is non-empty.
# @POST Returns set of recovered row indices, or None if no rows could be recovered from the partial response.
# @SIDE_EFFECT Creates TranslationRecord + TranslationLanguage rows for recovered rows.
# @RATIONALE Instead of binary-splitting (which loses all progress), saves whatever
# the model produced before hitting max_tokens. Only unrecovered rows are retried.
def _try_recover_partial(self, llm_response, batch_rows, run_id, batch_id, target_languages):
try:
translations = parse_llm_response(
llm_response, len(batch_rows), target_languages=target_languages,
finish_reason="length",
)
except ValueError:
return None
if not translations:
return None
recovered_ids: set[str] = set()
for row in batch_rows:
row_id = str(row.get("row_index", ""))
if row_id in translations:
recovered_ids.add(row_id)
if not recovered_ids:
return None
# Persist recovered rows as SUCCESS records
recovered_rows = [
r for r in batch_rows
if str(r.get("row_index", "")) in recovered_ids
]
for row in recovered_rows:
row_id = str(row.get("row_index", ""))
td = translations[row_id]
source_text = row.get("source_text", "")
detected_lang = row.get("_detected_lang", "und") or "und"
if detected_lang == "und" and td:
detected_lang = td.get("detected_source_language", "und") or "und"
plv = self._extract_per_lang_values(td, target_languages)
primary = next(iter(plv.values()), "")
record = TranslationRecord(
id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id,
source_sql=source_text, target_sql=primary,
source_object_type="table_row", source_object_id=row.get("row_index"),
source_object_name=row.get("source_object_name", ""),
source_data=row.get("source_data"), source_hash=row.get("_source_hash"),
status="SUCCESS",
)
self.db.add(record)
for lang_code in target_languages:
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
continue
val = plv.get(lang_code, "")
self.db.add(TranslationLanguage(
id=str(uuid.uuid4()), record_id=record.id, language_code=lang_code,
source_language_detected=detected_lang, translated_value=val or "",
final_value=val or "", status="translated", needs_review=(detected_lang == "und"),
))
logger.reason(
f"Recovered {len(recovered_ids)}/{len(batch_rows)} rows from truncated response",
{"batch_id": batch_id, "recovered": len(recovered_ids), "total": len(batch_rows)},
)
return recovered_ids
# #endregion _try_recover_partial
# #region _retry_missing_rows [C:3] [TYPE Function] [SEMANTICS translate, llm, retry]
# @BRIEF Retry only the rows that were not recovered from a truncated response.
# Creates a new sub-batch for the missing rows and calls call_llm_for_batch recursively.
# @PRE missing_rows is a subset of the original batch rows, or empty.
# @POST Returns dict with successful/failed/skipped counts from the sub-batch.
# @SIDE_EFFECT Creates TranslationBatch for the retry sub-batch; may recurse.
def _retry_missing_rows(self, job, run_id, missing_rows, dict_matches, _batch_id, max_tokens, depth):
if not missing_rows:
return {"successful": 0, "failed": 0, "skipped": 0, "retries": 0}
sub_batch = TranslationBatch(
id=str(uuid.uuid4()), run_id=run_id, batch_index=-1,
status="RUNNING", total_records=len(missing_rows),
started_at=datetime.now(UTC),
)
self.db.add(sub_batch)
self.db.flush()
result = self.call_llm_for_batch(
job, run_id, missing_rows, dict_matches,
sub_batch.id, max_tokens, depth,
)
sub_batch.completed_at = datetime.now(UTC)
sub_batch.successful_records = result["successful"]
sub_batch.failed_records = result["failed"]
sub_batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS"
self.db.flush()
return result
# #endregion _retry_missing_rows
# #region _handle_parse_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, parse, failure]
# @BRIEF Handle LLM parse failure — mark rows as SKIPPED.
# @SIDE_EFFECT DB writes for each row in the batch.
def _handle_parse_failure(self, batch_rows, run_id, batch_id, retries, error):
for row in batch_rows:
self.db.add(TranslationRecord(
@@ -243,7 +395,11 @@ class LLMTranslationService:
error_message=f"LLM parse failure: {error}",
))
return {"successful": 0, "failed": 0, "skipped": len(batch_rows), "retries": retries}
# #endregion _handle_parse_failure
# #region _create_records_from_translations [C:3] [TYPE Function] [SEMANTICS translate, llm, persist]
# @BRIEF Create TranslationRecord + TranslationLanguage rows from parsed LLM translations.
# @SIDE_EFFECT DB writes for each successfully translated row.
def _create_records_from_translations(self, batch_rows, run_id, batch_id, target_languages, translations, dict_matches, retries):
successful = failed = skipped = 0
for row in batch_rows:
@@ -290,7 +446,10 @@ class LLMTranslationService:
final_value=val or "", status="translated", needs_review=needs_review,
))
return {"successful": successful, "failed": failed, "skipped": skipped, "retries": retries}
# #endregion _create_records_from_translations
# #region _extract_per_lang_values [C:2] [TYPE Function] [SEMANTICS translate, llm, parse]
# @BRIEF Extract per-language translation values from a parsed LLM response row.
@staticmethod
def _extract_per_lang_values(td, target_languages):
plv = {}
@@ -306,7 +465,11 @@ class LLMTranslationService:
plv[target_languages[0]] = t
has_any = True
return plv if has_any else {}
# #endregion _extract_per_lang_values
# #region _add_skipped [C:3] [TYPE Function] [SEMANTICS translate, llm, record]
# @BRIEF Add a SKIPPED TranslationRecord for a row with no translation data.
# @SIDE_EFFECT DB write to translation_records.
def _add_skipped(self, row, run_id, batch_id, source_text, reason):
self.db.add(TranslationRecord(
id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id,
@@ -316,11 +479,11 @@ class LLMTranslationService:
source_data=row.get("source_data"), status="SKIPPED",
error_message=reason,
))
# #endregion _add_skipped
# #region call_llm [C:3] [TYPE Function] [SEMANTICS translate, llm, call]
# @BRIEF Route to provider-specific LLM call implementation.
def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> tuple[str, str | None]:
"""Call the configured LLM provider with the batch prompt."""
with belief_scope("LLMTranslationService.call_llm"):
if not job.provider_id:
raise ValueError("Job has no LLM provider configured")
@@ -358,13 +521,18 @@ class LLMTranslationService:
return result
# #endregion call_llm
# -- Static methods delegated to sub-modules for backward compat --
# #region call_openai_compatible [C:1] [TYPE Function] [SEMANTICS translate, llm, compat]
# @BRIEF Backward-compat delegation to _llm_http.call_openai_compatible.
@staticmethod
def call_openai_compatible(*a, **kw):
return call_openai_compatible(*a, **kw)
# #endregion call_openai_compatible
# #region _parse_llm_response [C:1] [TYPE Function] [SEMANTICS translate, llm, compat]
# @BRIEF Backward-compat delegation to _llm_parse.parse_llm_response.
@staticmethod
def _parse_llm_response(*a, **kw):
return parse_llm_response(*a, **kw)
# #endregion _parse_llm_response
# #endregion LLMTranslationService
# #endregion LLMTranslationService

15
backend/src/plugins/translate/_llm_http.py
View File

@@ -125,6 +125,21 @@ def call_openai_compatible(
})
raise ValueError(f"LLM response processing failed: {e}")
# Log provider token usage for batch sizing calibration
usage = data.get("usage") or {}
if usage:
logger.reason(
"LLM provider usage",
{
"prompt_tokens": usage.get("prompt_tokens"),
"completion_tokens": usage.get("completion_tokens"),
"total_tokens": usage.get("total_tokens"),
"finish_reason": finish_reason,
"max_tokens_sent": max_tokens,
"chars_sent": len(prompt),
},
)
refusal = msg.get("refusal") if isinstance(msg, dict) else None
if refusal:
logger.explore("LLM refused to respond", extra={

86
backend/src/plugins/translate/_token_budget.py
View File

@@ -1,13 +1,17 @@
# #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm]
# @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits.
# Output budget is now the PRIMARY constraint — finish_reason=length is usually output exceeding max_tokens, not input overflowing context_window.
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RELATION DEPENDS_ON -> [TranslationExecutor]
# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API.
# @RATIONALE CJK ratio lowered from 1.5→1.0 and other from 2.2→1.8 — modern models (Qwen, DeepSeek)
# tokenize more densely. SAFETY_FACTOR of 0.75 applied to input budget to account for
# variance across tokenizers. Output budget now computed first and serves as the hard cap.
# PROVIDER_DEFAULTS is a fallback — provider-level context_window/max_output_tokens in DB take priority.
# DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens.
# @REJECTED External tokenizer library — would introduce heavy dependency for estimation only.
# Fixed batch_size of 50 — causes truncation on long-content rows.
# CJK ratio 1.5 — too optimistic for Qwen/DeepSeek tokenizers (actual ~1.0-1.2).
# Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length).
# #region DEFAULT_CONTEXT_WINDOW [TYPE Constant]
@@ -64,9 +68,35 @@ DICT_TOKENS_PER_ENTRY = 20
DICT_TOKENS_MAX = 5000
# #endregion DICT_TOKENS_MAX
# #region CHARS_PER_TOKEN_MIXED [TYPE Constant]
# @BRIEF Deprecated — use CJK_RATIO + OTHER_RATIO instead. Kept for backward compat in estimate_row_tokens.
CHARS_PER_TOKEN_MIXED = 2.2
# #endregion CHARS_PER_TOKEN_MIXED
# #region CJK_RATIO [TYPE Constant]
# @BRIEF Conservative CJK chars/token ratio. Modern models (Qwen, DeepSeek) tokenize at ~1.0-1.3.
# @RATIONALE Lowered from 1.5 to 1.0 to produce more conservative (higher) token estimates.
# Actual token density varies by model tokenizer; this is intentionally pessimistic.
CJK_RATIO = 1.0
# #endregion CJK_RATIO
# #region OTHER_RATIO [TYPE Constant]
# @BRIEF Conservative ratio for non-CJK text. cl100k_base averages ~1.8-2.5.
OTHER_RATIO = 1.8
# #endregion OTHER_RATIO
# #region INPUT_SAFETY_FACTOR [TYPE Constant]
# @BRIEF Multiplier applied to per-batch input budget in batch sizer.
# Accounts for CJK estimation variance across different tokenizers.
# @RATIONALE Single centralised factor instead of scattered margins in multiple functions.
# 0.75 means batch uses at most 75% of estimated budget → 33% headroom.
INPUT_SAFETY_FACTOR = 0.75
# #endregion INPUT_SAFETY_FACTOR
# #region OUTPUT_SAFETY_FACTOR [TYPE Constant]
# @BRIEF Multiplier applied to output budget when computing max rows per batch.
OUTPUT_SAFETY_FACTOR = 0.70
# #endregion OUTPUT_SAFETY_FACTOR
# #region MIN_MAX_TOKENS [TYPE Constant]
MIN_MAX_TOKENS = 4096
@@ -80,18 +110,16 @@ MAX_OUTPUT_HEADROOM = 3000
# region _estimate_tokens_for_text [TYPE Function]
# @BRIEF Estimate token count for a text string with CJK-aware heuristics.
# CJK characters (~1.5 chars/token) vs other text (~2.2 chars/token).
# Conservative ratios: CJK ~1.0 chars/token, other ~1.8 chars/token.
# @PRE text is a string.
# @POST Returns estimated token count >= 1.
# @RATIONALE CJK characters are more token-dense than Latin/Cyrillic text.
# Using a single ratio undercounts CJK input and causes truncation.
# @POST Returns estimated token count >= 1. Intentionally pessimistic (over-estimates)
# to prevent LLM truncation from content that tokenizes more densely than expected.
# @RATIONALE Lowered ratios for modern models: Qwen/DeepSeek tokenize CJK at ~1.0-1.3.
# cl100k_base averages ~1.8-2.5 for non-CJK. Using conservative values prevents
# the primary cause of finish_reason=length: underestimating actual token count.
# @REJECTED Using tiktoken — would introduce a heavy dependency for estimation only.
# CJK ratio 1.5 — too optimistic, caused 3x underestimation in production logs.
def _estimate_tokens_for_text(text: str) -> int:
"""Estimate token count with CJK-aware heuristics.
CJK characters (CJK Unified Ideographs) use ~1.5 chars/token.
All other characters use ~2.2 chars/token.
"""
if not text:
return 1
@@ -103,8 +131,8 @@ def _estimate_tokens_for_text(text: str) -> int:
else:
other_count += 1
cjk_tokens = cjk_count / 1.5 if cjk_count else 0
other_tokens = other_count / 2.2 if other_count else 0
cjk_tokens = cjk_count / CJK_RATIO if cjk_count else 0
other_tokens = other_count / OTHER_RATIO if other_count else 0
return max(1, int(cjk_tokens + other_tokens))
# endregion _estimate_tokens_for_text
@@ -157,6 +185,28 @@ def _calculate_output_tokens(
)
def _compute_max_rows_by_output(max_output_tokens: int, num_languages: int) -> int:
"""Compute the maximum number of rows that fit within the output budget.
This is the PRIMARY constraint — finish_reason=length most often occurs when
the model's generated JSON output exceeds max_tokens, not when input overflows
context_window.
@RATIONALE Output budget is the dominant cause of truncation. Input budget
is secondary because modern models have large context windows (64K-256K)
but limited output tokens (8K-16K). Computing output-first prevents
packing too many rows that the model cannot fit in its response.
"""
overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM
per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
if per_row <= 0:
return 20 # sensible fallback
available = int((max_output_tokens - overhead) * OUTPUT_SAFETY_FACTOR)
if available <= 0:
return 1
return max(available // per_row, 1)
def _apply_output_aware_batch_sizing(
safe_size: int,
num_languages: int,
@@ -330,6 +380,8 @@ def estimate_token_budget(
dict_warning,
)
max_rows_by_output = _compute_max_rows_by_output(max_output_tokens, num_languages)
return {
"batch_size_adjusted": safe_size,
"estimated_input_tokens": estimated_input,
@@ -338,8 +390,6 @@ def estimate_token_budget(
"warning": warning,
"available_input_budget": available_input_budget,
"max_output_tokens": max_output_tokens,
"max_rows_by_output": max_rows_by_output,
}
# endregion estimate_token_budget
# #endregion estimate_token_budget
# endregion estimate_token_budget
# #endregion estimate_token_budget

16
backend/src/plugins/translate/executor.py
View File

@@ -221,11 +221,23 @@ class TranslationExecutor:
pass
return None
def _resolve_provider_config(self, job) -> dict:
if not job.provider_id:
return {"model": None, "context_window": None, "max_output_tokens": None}
try:
return LLMProviderService(self.db).get_provider_token_config(job.provider_id)
except Exception:
return {"model": None, "context_window": None, "max_output_tokens": None}
# Backward-compat alias for preview.py (calls via resolve_provider_model, not _resolve_provider_model)
def resolve_provider_model(self, job) -> str | None:
return self._resolve_provider_model(job)
def _auto_size_batches(self, job, source_rows, target_languages, provider_info=None) -> list:
"""Split source rows into auto-sized batches based on content length."""
from ._batch_sizer import AdaptiveBatchSizer
if provider_info is None:
provider_info = self._resolve_provider_model(job)
# AdaptiveBatchSizer.auto_size_batches resolves provider config internally
# via job.provider_id. provider_info is optional override for tests.
return AdaptiveBatchSizer(self.db, self.config_manager).auto_size_batches(
job, source_rows, target_languages, provider_info)

118
backend/src/services/__tests__/test_llm_provider.py
View File

@@ -380,3 +380,121 @@ def test_llm_provider_config_multimodal_explicit():
is_multimodal=True,
)
assert config.is_multimodal is True
# endregion test_llm_provider_config_multimodal_explicit
# region test_llm_provider_config_context_window_default [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig.context_window defaults to None.
def test_llm_provider_config_context_window_default():
"""Verify default context_window is None in schema."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Default Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4",
)
assert config.context_window is None
# endregion test_llm_provider_config_context_window_default
# region test_llm_provider_config_context_window_explicit [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderConfig]
# @PURPOSE: Verify LLMProviderConfig accepts explicit context_window value.
def test_llm_provider_config_context_window_explicit():
"""Verify setting context_window explicitly works."""
config = LLMProviderConfig(
provider_type=LLMProviderType.OPENAI,
name="Test",
base_url="https://api.openai.com/v1",
api_key="sk-test",
default_model="gpt-4",
context_window=128000,
max_output_tokens=16384,
)
assert config.context_window == 128000
assert config.max_output_tokens == 16384
# endregion test_llm_provider_config_context_window_explicit
# region test_get_provider_token_config_no_provider [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns all-None when provider not found.
def test_get_provider_token_config_no_provider():
"""When provider_id doesn't exist, returns all values as None."""
db = MagicMock(spec=Session)
db.query.return_value.filter.return_value.first.return_value = None
service = LLMProviderService(db)
result = service.get_provider_token_config("nonexistent-id")
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
# endregion test_get_provider_token_config_no_provider
# region test_get_provider_token_config_with_values [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns provider token limits from DB.
def test_get_provider_token_config_with_values():
"""Provider with context_window and max_output_tokens returns them."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = "gpt-4o-mini"
mock_provider.context_window = 128000
mock_provider.max_output_tokens = 16384
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-1")
assert result["model"] == "gpt-4o-mini"
assert result["context_window"] == 128000
assert result["max_output_tokens"] == 16384
# endregion test_get_provider_token_config_with_values
# region test_get_provider_token_config_null_limits [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config returns None for null DB fields.
def test_get_provider_token_config_null_limits():
"""Provider with NULL token limits returns None values (signal to use defaults)."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = "qwen-flash"
mock_provider.context_window = None
mock_provider.max_output_tokens = None
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-2")
assert result["model"] == "qwen-flash"
assert result["context_window"] is None
assert result["max_output_tokens"] is None
# endregion test_get_provider_token_config_null_limits
# region test_provider_token_config_default_model_fallback [TYPE Function]
# @RELATION BINDS_TO -> [LLMProviderService]
# @PURPOSE: Verify get_provider_token_config falls back to "gpt-4o-mini" when default_model is None.
def test_provider_token_config_default_model_fallback():
"""Provider without explicit default_model uses 'gpt-4o-mini' fallback."""
db = MagicMock(spec=Session)
mock_provider = MagicMock(spec=LLMProvider)
mock_provider.default_model = None
mock_provider.context_window = None
mock_provider.max_output_tokens = None
db.query.return_value.filter.return_value.first.return_value = mock_provider
service = LLMProviderService(db)
result = service.get_provider_token_config("provider-3")
assert result["model"] == "gpt-4o-mini"
assert result["context_window"] is None
assert result["max_output_tokens"] is None
# endregion test_provider_token_config_default_model_fallback

22
backend/src/services/llm_provider.py
View File

@@ -115,6 +115,8 @@ class LLMProviderService:
is_active=config.is_active,
is_multimodal=config.is_multimodal,
max_images=config.max_images,
context_window=config.context_window,
max_output_tokens=config.max_output_tokens,
)
self.db.add(db_provider)
self.db.commit()
@@ -148,6 +150,8 @@ class LLMProviderService:
db_provider.is_active = config.is_active
db_provider.is_multimodal = config.is_multimodal
db_provider.max_images = config.max_images
db_provider.context_window = config.context_window
db_provider.max_output_tokens = config.max_output_tokens
self.db.commit()
self.db.refresh(db_provider)
@@ -235,6 +239,24 @@ class LLMProviderService:
# endregion get_decrypted_api_key
# region get_provider_token_config [TYPE Function]
# @PURPOSE: Returns provider token limits for batch sizing.
# @PRE provider_id must be valid.
# @POST Returns dict with model name, context_window, max_output_tokens.
# Values from DB take priority; None means "use PROVIDER_DEFAULTS fallback".
# @RATIONALE Centralised helper — both _batch_proc.py and _batch_sizer.py need
# the same resolution logic. Avoids duplicating DB queries and defaults.
def get_provider_token_config(self, provider_id: str) -> dict:
provider = self.get_provider(provider_id)
if not provider:
return {"model": None, "context_window": None, "max_output_tokens": None}
return {
"model": provider.default_model or "gpt-4o-mini",
"context_window": provider.context_window,
"max_output_tokens": provider.max_output_tokens,
}
# endregion get_provider_token_config
# #endregion LLMProviderService