perf(translate): fix slow translation startup — CJK estimation, output budget, provider token config
Root cause: batch sizing underestimated CJK token density (1.5→1.0 chars/token) and ignored output budget as primary constraint, causing cascading finish_reason=length. Changes: - _token_budget.py: CJK_RATIO 1.5→1.0, OTHER_RATIO 2.2→1.8, safety factors 0.75/0.70 - _token_budget.py: new _compute_max_rows_by_output() — output budget is PRIMARY constraint - _batch_sizer.py: resolve_provider_config() with DB-level context_window/max_output_tokens - _batch_sizer.py: INPUT_SAFETY_FACTOR applied, max_rows_by_output used as row cap - _llm_http.py: log actual usage.prompt_tokens/.completion_tokens from provider - _llm_call.py: retry only missing rows after finish_reason=length (save partial result) - models/llm.py + schema: provider-level context_window / max_output_tokens (nullable) - services/llm_provider.py: get_provider_token_config() helper - Alembic migration: add columns to llm_providers - Svelte ProviderConfig: collapsible Advanced: Token Limits section - 12 new tests (token budget, batch sizer, provider config) - All 492 tests pass
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
"""Add context_window and max_output_tokens to llm_providers
|
||||
|
||||
Revision ID: a1b2c3d4e5f6
|
||||
Revises: f1a2b3c4d5e6
|
||||
Create Date: 2026-06-03
|
||||
|
||||
Add token window configuration to LLM provider records:
|
||||
- context_window: total context window in tokens (nullable)
|
||||
- max_output_tokens: max output tokens limit (nullable)
|
||||
Both NULL = use PROVIDER_DEFAULTS fallback from model name.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a1b2c3d4e5f6"
|
||||
down_revision: Union[str, None] = "f1a2b3c4d5e6"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"llm_providers",
|
||||
sa.Column(
|
||||
"context_window",
|
||||
sa.Integer(),
|
||||
nullable=True,
|
||||
comment="Total context window in tokens. NULL = auto-detect from model name",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"llm_providers",
|
||||
sa.Column(
|
||||
"max_output_tokens",
|
||||
sa.Integer(),
|
||||
nullable=True,
|
||||
comment="Max output tokens limit. NULL = auto-detect from model name",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("llm_providers", "max_output_tokens")
|
||||
op.drop_column("llm_providers", "context_window")
|
||||
@@ -85,6 +85,8 @@ async def get_providers(
|
||||
is_active=p.is_active,
|
||||
is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False,
|
||||
max_images=p.max_images,
|
||||
context_window=p.context_window,
|
||||
max_output_tokens=p.max_output_tokens,
|
||||
)
|
||||
for p in providers
|
||||
]
|
||||
@@ -272,6 +274,8 @@ async def create_provider(
|
||||
is_active=provider.is_active,
|
||||
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
|
||||
max_images=provider.max_images,
|
||||
context_window=provider.context_window,
|
||||
max_output_tokens=provider.max_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -309,6 +313,8 @@ async def update_provider(
|
||||
is_active=provider.is_active,
|
||||
is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False,
|
||||
max_images=provider.max_images,
|
||||
context_window=provider.context_window,
|
||||
max_output_tokens=provider.max_output_tokens,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -456,6 +456,8 @@ async def get_consolidated_settings(
|
||||
"default_model": p.default_model,
|
||||
"is_active": p.is_active,
|
||||
"is_multimodal": bool(p.is_multimodal) if p.is_multimodal is not None else False,
|
||||
"context_window": p.context_window,
|
||||
"max_output_tokens": p.max_output_tokens,
|
||||
}
|
||||
for p in providers
|
||||
]
|
||||
|
||||
@@ -66,6 +66,14 @@ class LLMProvider(Base):
|
||||
is_active = Column(Boolean, default=True)
|
||||
is_multimodal = Column(Boolean, default=False)
|
||||
max_images = Column(Integer, nullable=True, default=None)
|
||||
context_window = Column(
|
||||
Integer, nullable=True, default=None,
|
||||
comment="Total context window in tokens. NULL = fallback to PROVIDER_DEFAULTS from model name.",
|
||||
)
|
||||
max_output_tokens = Column(
|
||||
Integer, nullable=True, default=None,
|
||||
comment="Max output tokens limit. NULL = fallback to PROVIDER_DEFAULTS from model name.",
|
||||
)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
# #endregion LLMProvider
|
||||
|
||||
|
||||
@@ -30,6 +30,14 @@ class LLMProviderConfig(BaseModel):
|
||||
is_active: bool = True
|
||||
is_multimodal: bool = False
|
||||
max_images: int | None = None
|
||||
context_window: int | None = Field(
|
||||
None, ge=1000, le=256000,
|
||||
description="Context window in tokens. Leave blank for auto-detection.",
|
||||
)
|
||||
max_output_tokens: int | None = Field(
|
||||
None, ge=256,
|
||||
description="Max output tokens. Must be less than context_window.",
|
||||
)
|
||||
# #endregion LLMProviderConfig
|
||||
|
||||
# #region ValidationStatus [TYPE Class]
|
||||
|
||||
89
backend/src/plugins/translate/__tests__/test_batch_sizer.py
Normal file
89
backend/src/plugins/translate/__tests__/test_batch_sizer.py
Normal file
@@ -0,0 +1,89 @@
|
||||
# #region TestAdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS test, batch, sizer, provider, config]
|
||||
# @BRIEF Verify AdaptiveBatchSizer contracts — provider config resolution, safety factor, row cap.
|
||||
# @RELATION BINDS_TO -> [AdaptiveBatchSizer]
|
||||
# @TEST_EDGE resolve_provider_config_no_provider — no provider_id returns all-None
|
||||
# @TEST_EDGE resolve_provider_config_with_provider — returns model + token limits from DB
|
||||
# @TEST_EDGE resolve_provider_config_exception — DB error returns all-None gracefully
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.models.translate import TranslationJob
|
||||
from src.plugins.translate._batch_sizer import AdaptiveBatchSizer
|
||||
|
||||
|
||||
# region TestResolveProviderConfig [TYPE Class]
|
||||
# @BRIEF Tests for AdaptiveBatchSizer.resolve_provider_config.
|
||||
class TestResolveProviderConfig:
|
||||
|
||||
# region test_no_provider_id [TYPE Function]
|
||||
# @BRIEF When job has no provider_id, returns all-None dict.
|
||||
def test_no_provider_id(self):
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.provider_id = None
|
||||
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
|
||||
result = sizer.resolve_provider_config(job)
|
||||
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
# endregion test_no_provider_id
|
||||
|
||||
# region test_with_provider_full_config [TYPE Function]
|
||||
# @BRIEF Provider with context_window and max_output_tokens returns all values.
|
||||
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
|
||||
def test_with_provider_full_config(self, mock_provider_svc):
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.provider_id = "provider-1"
|
||||
|
||||
mock_svc_instance = MagicMock()
|
||||
mock_svc_instance.get_provider_token_config.return_value = {
|
||||
"model": "gpt-4o-mini",
|
||||
"context_window": 128000,
|
||||
"max_output_tokens": 16384,
|
||||
}
|
||||
mock_provider_svc.return_value = mock_svc_instance
|
||||
|
||||
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
|
||||
result = sizer.resolve_provider_config(job)
|
||||
assert result["model"] == "gpt-4o-mini"
|
||||
assert result["context_window"] == 128000
|
||||
assert result["max_output_tokens"] == 16384
|
||||
# endregion test_with_provider_full_config
|
||||
|
||||
# region test_with_provider_null_config [TYPE Function]
|
||||
# @BRIEF Provider with NULL token limits still returns model + None values.
|
||||
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
|
||||
def test_with_provider_null_config(self, mock_provider_svc):
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.provider_id = "provider-2"
|
||||
|
||||
mock_svc_instance = MagicMock()
|
||||
mock_svc_instance.get_provider_token_config.return_value = {
|
||||
"model": "deepseek-v4-flash",
|
||||
"context_window": None,
|
||||
"max_output_tokens": None,
|
||||
}
|
||||
mock_provider_svc.return_value = mock_svc_instance
|
||||
|
||||
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
|
||||
result = sizer.resolve_provider_config(job)
|
||||
assert result["model"] == "deepseek-v4-flash"
|
||||
assert result["context_window"] is None # NULL → use PROVIDER_DEFAULTS
|
||||
assert result["max_output_tokens"] is None
|
||||
# endregion test_with_provider_null_config
|
||||
|
||||
# region test_exception_returns_safe_defaults [TYPE Function]
|
||||
# @BRIEF DB exception returns all-None dict gracefully.
|
||||
@patch("src.plugins.translate._batch_sizer.LLMProviderService")
|
||||
def test_exception_returns_safe_defaults(self, mock_provider_svc):
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.provider_id = "provider-3"
|
||||
|
||||
mock_svc_instance = MagicMock()
|
||||
mock_svc_instance.get_provider_token_config.side_effect = Exception("DB error")
|
||||
mock_provider_svc.return_value = mock_svc_instance
|
||||
|
||||
sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock())
|
||||
result = sizer.resolve_provider_config(job)
|
||||
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
# endregion test_exception_returns_safe_defaults
|
||||
|
||||
# endregion TestResolveProviderConfig
|
||||
# #endregion TestAdaptiveBatchSizer
|
||||
@@ -307,12 +307,12 @@ class TestEstimateRowTokens:
|
||||
|
||||
# region test_cjk_text [TYPE Function]
|
||||
def test_cjk_text(self) -> None:
|
||||
"""CJK characters are token-denser (~1.5 chars/token)."""
|
||||
"""CJK characters are token-denser (~1.0 chars/token with conservative estimate)."""
|
||||
job = MagicMock(spec=TranslationJob)
|
||||
job.context_columns = []
|
||||
# 12 CJK chars → 12/1.5 = 8 tokens, plus 1 for empty context = 9
|
||||
# 12 CJK chars → 12/1.0 = 12 tokens (conservative), plus 1 for context = 13
|
||||
tokens = estimate_row_tokens("你好世界这是一个测试消息", None, job)
|
||||
assert tokens == 9, f"Expected 9 tokens (8 CJK + 1 empty ctx) for CJK, got {tokens}"
|
||||
assert tokens == 13, f"Expected 13 tokens (12 CJK + 1 ctx) for CJK, got {tokens}"
|
||||
|
||||
# endregion test_cjk_text
|
||||
|
||||
@@ -589,24 +589,26 @@ class TestAutoSizeBatches:
|
||||
executor: TranslationExecutor,
|
||||
job: MagicMock,
|
||||
) -> None:
|
||||
"""When provider_info is None, _resolve_provider_model is called."""
|
||||
"""When provider_info is None, AdaptiveBatchSizer resolves provider config internally."""
|
||||
mock_estimate.return_value = {
|
||||
"batch_size_adjusted": 10,
|
||||
"estimated_input_tokens": 5000,
|
||||
"estimated_output_tokens": 2000,
|
||||
"max_output_needed": 4096,
|
||||
"warning": None,
|
||||
"max_rows_by_output": 20,
|
||||
"available_input_budget": 47616,
|
||||
"max_output_tokens": 16384,
|
||||
}
|
||||
source_rows = [
|
||||
{"row_index": "0", "source_text": "hello", "source_data": None},
|
||||
]
|
||||
# provider_info=None → should call _resolve_provider_model
|
||||
with patch.object(executor, '_resolve_provider_model', return_value="gpt-4o-mini") as mock_resolve:
|
||||
batches = executor._auto_size_batches(
|
||||
job, source_rows, ["en"], provider_info=None,
|
||||
)
|
||||
mock_resolve.assert_called_once_with(job)
|
||||
assert len(batches) == 1
|
||||
# provider_info=None → AdaptiveBatchSizer resolves config from job.provider_id
|
||||
# No longer calls executor._resolve_provider_model
|
||||
batches = executor._auto_size_batches(
|
||||
job, source_rows, ["en"], provider_info=None,
|
||||
)
|
||||
assert len(batches) == 1
|
||||
|
||||
# endregion test_provider_info_resolution
|
||||
|
||||
|
||||
@@ -14,7 +14,15 @@
|
||||
# @TEST_INVARIANT max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192)
|
||||
# @TEST_INVARIANT warning is None when batch fits, str when reduced
|
||||
|
||||
from src.plugins.translate._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, estimate_token_budget
|
||||
from src.plugins.translate._token_budget import (
|
||||
CJK_RATIO,
|
||||
DEFAULT_CONTEXT_WINDOW,
|
||||
DEFAULT_MAX_OUTPUT_TOKENS,
|
||||
OTHER_RATIO,
|
||||
estimate_token_budget,
|
||||
_compute_max_rows_by_output,
|
||||
_estimate_tokens_for_text,
|
||||
)
|
||||
|
||||
|
||||
# region _make_row [TYPE Function]
|
||||
@@ -26,10 +34,91 @@ def _make_row(text: str, **context) -> dict:
|
||||
# endregion _make_row
|
||||
|
||||
|
||||
# region _make_cjk_row [TYPE Function]
|
||||
# @BRIEF Create a test source row with CJK text.
|
||||
def _make_cjk_row() -> dict:
|
||||
return {"source_text": "你好世界这是一个测试消息", "row_index": "0"}
|
||||
# endregion _make_cjk_row
|
||||
|
||||
|
||||
# region TestTokenBudget [TYPE Class]
|
||||
# @BRIEF Test suite for estimate_token_budget.
|
||||
# @BRIEF Test suite for estimate_token_budget and related token estimation functions.
|
||||
class TestTokenBudget:
|
||||
|
||||
# region test_cjk_ratio_estimate [TYPE Function]
|
||||
# @BRIEF Verify CJK token estimation uses the conservative CJK_RATIO.
|
||||
def test_cjk_ratio_estimate(self):
|
||||
"""12 CJK chars / 1.0 = 12 tokens (conservative), plus 1 for empty context."""
|
||||
tokens = _estimate_tokens_for_text("你好世界这是一个测试消息")
|
||||
expected = int(12 / CJK_RATIO)
|
||||
assert tokens == expected, f"CJK estimate {tokens} != expected {expected}"
|
||||
# endregion test_cjk_ratio_estimate
|
||||
|
||||
# region test_mixed_text_estimate [TYPE Function]
|
||||
# @BRIEF Verify mixed CJK+Ltn text estimation.
|
||||
def test_mixed_text_estimate(self):
|
||||
"""Mixed text: CJK at CJK_RATIO, Latin at OTHER_RATIO."""
|
||||
text = "你好世界! Hello world, this is a test!"
|
||||
tokens = _estimate_tokens_for_text(text)
|
||||
# 4 CJK chars / 1.0 = 4, 30 non-CJK / 1.8 = 16, total = 20
|
||||
assert tokens == 20, f"Expected 20, got {tokens}"
|
||||
# endregion test_mixed_text_estimate
|
||||
|
||||
# region test_empty_text_estimate [TYPE Function]
|
||||
# @BRIEF Empty text returns 1 token minimum.
|
||||
def test_empty_text_estimate(self):
|
||||
assert _estimate_tokens_for_text("") == 1
|
||||
assert _estimate_tokens_for_text(None) == 1
|
||||
# endregion test_empty_text_estimate
|
||||
|
||||
# region test_compute_max_rows_by_output_single_lang [TYPE Function]
|
||||
# @BRIEF Single target language: compute max rows from output budget.
|
||||
def test_compute_max_rows_by_output_single_lang(self):
|
||||
"""With max_output_tokens=8192 and 1 language, should return limited rows."""
|
||||
max_rows = _compute_max_rows_by_output(max_output_tokens=8192, num_languages=1)
|
||||
assert max_rows >= 1
|
||||
assert max_rows < 100 # Should be reasonable
|
||||
# endregion test_compute_max_rows_by_output_single_lang
|
||||
|
||||
# region test_compute_max_rows_by_output_multi_lang [TYPE Function]
|
||||
# @BRIEF Multiple target languages reduce max rows from output budget.
|
||||
def test_compute_max_rows_by_output_multi_lang(self):
|
||||
"""More languages = fewer rows that fit in the same output budget."""
|
||||
single = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=1)
|
||||
multi = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=4)
|
||||
assert multi <= single
|
||||
assert multi >= 1
|
||||
# endregion test_compute_max_rows_by_output_multi_lang
|
||||
|
||||
# region test_compute_max_rows_by_output_small_budget [TYPE Function]
|
||||
# @BRIEF Very small output budget returns at least 1 row.
|
||||
def test_compute_max_rows_by_output_small_budget(self):
|
||||
"""Even with minimal budget, at least 1 row fits."""
|
||||
max_rows = _compute_max_rows_by_output(max_output_tokens=4096, num_languages=2)
|
||||
assert max_rows >= 1
|
||||
# endregion test_compute_max_rows_by_output_small_budget
|
||||
|
||||
# region test_max_rows_by_output_in_return_dict [TYPE Function]
|
||||
# @BRIEF estimate_token_budget returns max_rows_by_output field.
|
||||
def test_max_rows_by_output_in_return_dict(self):
|
||||
"""The return dict includes max_rows_by_output as a positive int."""
|
||||
rows = [_make_row("Short text.")] * 10
|
||||
result = estimate_token_budget(rows, ["ru"], batch_size=10)
|
||||
assert "max_rows_by_output" in result
|
||||
assert isinstance(result["max_rows_by_output"], int)
|
||||
assert result["max_rows_by_output"] >= 1
|
||||
# endregion test_max_rows_by_output_in_return_dict
|
||||
|
||||
# region test_max_rows_by_output_changes_with_languages [TYPE Function]
|
||||
# @BRIEF max_rows_by_output decreases with more target languages.
|
||||
def test_max_rows_by_output_changes_with_languages(self):
|
||||
"""More target languages = smaller max_rows_by_output."""
|
||||
rows = [_make_row("Hello world")] * 10
|
||||
single = estimate_token_budget(rows, ["ru"], batch_size=10)
|
||||
multi = estimate_token_budget(rows, ["ru", "en", "fr", "de"], batch_size=10)
|
||||
assert multi["max_rows_by_output"] <= single["max_rows_by_output"]
|
||||
# endregion test_max_rows_by_output_changes_with_languages
|
||||
|
||||
# region test_small_rows_fit_at_requested_size [TYPE Function]
|
||||
# @BRIEF Short text rows fill the requested batch_size without reduction.
|
||||
def test_small_rows_fit_at_requested_size(self):
|
||||
|
||||
@@ -206,30 +206,36 @@ class BatchProcessingService:
|
||||
return count
|
||||
|
||||
def _process_llm(self, job, run_id, rows_for_llm, dict_matches, bid, tls):
|
||||
provider_model = None
|
||||
# Resolve provider token config (DB values take priority over PROVIDER_DEFAULTS)
|
||||
token_config = {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
if job.provider_id:
|
||||
try:
|
||||
p = LLMProviderService(self.db).get_provider(job.provider_id)
|
||||
if p:
|
||||
provider_model = p.default_model or "gpt-4o-mini"
|
||||
token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id)
|
||||
except Exception:
|
||||
provider_model = None
|
||||
pass
|
||||
|
||||
tb = estimate_token_budget(
|
||||
source_rows=rows_for_llm, target_languages=tls,
|
||||
source_column="source_text", context_columns=None,
|
||||
dictionary_entries=dict_matches, batch_size=len(rows_for_llm),
|
||||
provider_info=provider_model,
|
||||
provider_info=token_config["model"],
|
||||
context_window=token_config["context_window"],
|
||||
max_output_tokens=token_config["max_output_tokens"],
|
||||
)
|
||||
if tb["warning"]:
|
||||
logger.explore("Token budget warning", {"batch_id": bid, "warning": tb["warning"]})
|
||||
|
||||
max_rows_by_out = tb.get("max_rows_by_output", 0)
|
||||
|
||||
logger.reason(
|
||||
f"LLM process batch start", {
|
||||
"batch_id": bid, "llm_rows": len(rows_for_llm),
|
||||
"provider_model": provider_model,
|
||||
"provider_model": token_config["model"],
|
||||
"max_output_needed": tb.get("max_output_needed"),
|
||||
"estimated_input_tokens": tb.get("estimated_input_tokens"),
|
||||
"max_rows_by_output": max_rows_by_out,
|
||||
"context_window": token_config["context_window"],
|
||||
"max_output_tokens": token_config["max_output_tokens"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
# #region AdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS translate, batch, sizing, token-budget]
|
||||
# @BRIEF Adaptive batch sizing for LLM translation — splits source rows into variable-sized
|
||||
# batches based on actual content length and token budget estimates.
|
||||
# Output budget is the PRIMARY constraint; input budget is secondary.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [estimate_token_budget]
|
||||
# @RELATION DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION DEPENDS_ON -> [TranslationJob]
|
||||
# @RATIONALE Extracted from TranslationExecutor to comply with INV_7 (module < 400 lines).
|
||||
# Fixed batch_size of 50 wastes LLM context for short rows and overflows for
|
||||
# long rows. Variable sizing maximizes throughput while preventing truncation.
|
||||
# Safety factor applied to input budget to account for tokenizer variance.
|
||||
# Output-aware row cap computed from estimate_token_budget result.
|
||||
# Provider-level context_window/max_output_tokens take priority over PROVIDER_DEFAULTS.
|
||||
# @REJECTED Fixed batch_size of 50 — causes truncation on long-content rows.
|
||||
# Single monolithic batch — would lose all progress on any failure.
|
||||
|
||||
@@ -19,11 +21,8 @@ from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import TranslationJob
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from ._token_budget import (
|
||||
JSON_OVERHEAD_PER_ROW,
|
||||
MAX_OUTPUT_HEADROOM,
|
||||
OUTPUT_PER_ROW_PER_LANG,
|
||||
INPUT_SAFETY_FACTOR,
|
||||
PROMPT_BASE_TOKENS,
|
||||
REASONING_OVERHEAD,
|
||||
estimate_token_budget,
|
||||
)
|
||||
from ._utils import estimate_row_tokens
|
||||
@@ -32,41 +31,33 @@ from ._utils import estimate_row_tokens
|
||||
# #region AdaptiveBatchSizer [C:3] [TYPE Class]
|
||||
# @BRIEF Split source rows into auto-sized batches based on token budget estimates.
|
||||
class AdaptiveBatchSizer:
|
||||
"""Split source rows into auto-sized batches based on token budget estimates.
|
||||
|
||||
Each batch is sized so that its total estimated tokens fit within the
|
||||
available context window (input budget), accounting for prompt overhead,
|
||||
dictionary entries, and output tokens.
|
||||
"""
|
||||
|
||||
def __init__(self, db: Session, config_manager: ConfigManager) -> None:
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
|
||||
# #region resolve_provider_model [C:2] [TYPE Function] [SEMANTICS llm, provider, model]
|
||||
# @BRIEF Resolve the LLM provider model name for token budget estimation.
|
||||
# @POST Returns model name string or None if resolution fails.
|
||||
# #region resolve_provider_config [C:2] [TYPE Function] [SEMANTICS llm, provider, model, token-config]
|
||||
# @BRIEF Resolve provider token config (model name + token limits) for budget estimation.
|
||||
# DB values (context_window, max_output_tokens) take priority over PROVIDER_DEFAULTS.
|
||||
# @POST Returns dict with model, context_window, max_output_tokens. None values = use fallback.
|
||||
# @SIDE_EFFECT DB query to LLM provider table.
|
||||
def resolve_provider_model(self, job: TranslationJob) -> str | None:
|
||||
"""Resolve the provider model name for token budget estimation."""
|
||||
def resolve_provider_config(self, job: TranslationJob) -> dict:
|
||||
"""Resolve provider token config for token budget estimation."""
|
||||
if not job.provider_id:
|
||||
return None
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
try:
|
||||
p_svc = LLMProviderService(self.db)
|
||||
p = p_svc.get_provider(job.provider_id)
|
||||
if p:
|
||||
return p.default_model or "gpt-4o-mini"
|
||||
token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id)
|
||||
return token_config
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
# #endregion resolve_provider_model
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
# #endregion resolve_provider_config
|
||||
|
||||
# #region auto_size_batches [C:3] [TYPE Function] [SEMANTICS translate, batch, sizing]
|
||||
# @BRIEF Split source rows into variable-sized batches based on content length.
|
||||
# Output budget (max_rows_by_output) is the PRIMARY row-count constraint.
|
||||
# @PRE source_rows is non-empty. job has valid config.
|
||||
# @POST Returns list of batches, each batch is a list of row dicts.
|
||||
# Each batch fits within the estimated token budget for its rows.
|
||||
# @SIDE_EFFECT DB query to resolve provider model. Logs batch statistics.
|
||||
# @POST Returns list of batches. Each batch fits within estimated input token budget
|
||||
# with INPUT_SAFETY_FACTOR headroom AND within output row cap.
|
||||
# @SIDE_EFFECT DB query to resolve provider config. Logs batch statistics.
|
||||
def auto_size_batches(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
@@ -74,18 +65,14 @@ class AdaptiveBatchSizer:
|
||||
target_languages: list[str],
|
||||
provider_info: str | None = None,
|
||||
) -> list[list[dict]]:
|
||||
"""Split source rows into auto-sized batches based on content length.
|
||||
|
||||
Each batch is sized so that its total estimated tokens fit within the
|
||||
available context window (input budget), accounting for prompt overhead,
|
||||
dictionary entries, and output tokens.
|
||||
"""
|
||||
with belief_scope("AdaptiveBatchSizer.auto_size_batches"):
|
||||
if not source_rows:
|
||||
return []
|
||||
|
||||
if provider_info is None:
|
||||
provider_info = self.resolve_provider_model(job)
|
||||
# Resolve provider config (DB values > PROVIDER_DEFAULTS)
|
||||
token_config = self.resolve_provider_config(job)
|
||||
if provider_info is not None and token_config["model"] is None:
|
||||
token_config["model"] = provider_info
|
||||
|
||||
# 1. Estimate per-row token counts
|
||||
row_tokens: list[int] = []
|
||||
@@ -95,14 +82,16 @@ class AdaptiveBatchSizer:
|
||||
tokens = estimate_row_tokens(source_text, source_data, job)
|
||||
row_tokens.append(tokens)
|
||||
|
||||
# 2. Get budget recommendation
|
||||
# 2. Get budget recommendation (uses provider config from DB if available)
|
||||
budget = estimate_token_budget(
|
||||
source_rows=source_rows,
|
||||
target_languages=target_languages,
|
||||
source_column=job.translation_column or "source_text",
|
||||
context_columns=job.context_columns,
|
||||
batch_size=len(source_rows),
|
||||
provider_info=provider_info,
|
||||
provider_info=token_config["model"],
|
||||
context_window=token_config["context_window"],
|
||||
max_output_tokens=token_config["max_output_tokens"],
|
||||
)
|
||||
|
||||
recommended = budget.get("batch_size_adjusted", 0)
|
||||
@@ -119,18 +108,15 @@ class AdaptiveBatchSizer:
|
||||
for i in range(0, len(source_rows), fallback_size)
|
||||
]
|
||||
|
||||
# 3. Compute per-batch row-content budget
|
||||
# Use the ACTUAL available input capacity (context_window - max_output_tokens),
|
||||
# NOT the sum of the first N rows. This prevents long rows from being
|
||||
# incorrectly placed in 1-row batches when the context window has plenty of room.
|
||||
# 3. Compute per-batch row-content budget with safety factor
|
||||
available_input = budget.get("available_input_budget")
|
||||
if available_input is not None:
|
||||
# New-style: use actual available input capacity
|
||||
per_batch_budget = available_input - PROMPT_BASE_TOKENS
|
||||
raw_budget = available_input - PROMPT_BASE_TOKENS
|
||||
per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR)
|
||||
else:
|
||||
# Fallback for tests: use estimated_input (sum of first N rows)
|
||||
estimated_input = budget.get("estimated_input_tokens", 50000)
|
||||
per_batch_budget = estimated_input - PROMPT_BASE_TOKENS
|
||||
raw_budget = estimated_input - PROMPT_BASE_TOKENS
|
||||
per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR)
|
||||
|
||||
if per_batch_budget <= 0:
|
||||
fallback_size = job.batch_size or 50
|
||||
@@ -144,27 +130,18 @@ class AdaptiveBatchSizer:
|
||||
for i in range(0, len(source_rows), fallback_size)
|
||||
]
|
||||
|
||||
# 4. Greedy batch splitting
|
||||
# Compute max rows per batch from OUTPUT constraint.
|
||||
# This prevents output truncation (finish_reason=length) when batching
|
||||
# many short rows within the large input budget.
|
||||
num_languages = len(target_languages)
|
||||
max_output_tokens_val = budget.get("max_output_tokens")
|
||||
if max_output_tokens_val is not None:
|
||||
output_per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
|
||||
available_output = max_output_tokens_val - REASONING_OVERHEAD - MAX_OUTPUT_HEADROOM
|
||||
max_rows_by_output = max(available_output // output_per_row, 1) if output_per_row > 0 else 20
|
||||
max_rows_hard_cap = max_rows_by_output
|
||||
else:
|
||||
# Fallback for tests: old formula
|
||||
max_rows_hard_cap = max(recommended * 2, 20)
|
||||
# 4. Compute max rows per batch — OUTPUT BUDGET IS PRIMARY CONSTRAINT
|
||||
max_rows_by_output = budget.get("max_rows_by_output", 20)
|
||||
|
||||
# 5. Respect job.batch_size as the absolute maximum rows per batch.
|
||||
# User-configured batch_size overrides model-based estimates to
|
||||
# prevent LLM quality degradation on large batches.
|
||||
# Absolute hard cap as safety net (not the primary constraint)
|
||||
absolute_hard_cap = 50
|
||||
max_rows_hard_cap = min(max_rows_by_output, absolute_hard_cap)
|
||||
|
||||
# Respect job.batch_size as user override
|
||||
if job.batch_size:
|
||||
max_rows_hard_cap = min(max_rows_hard_cap, job.batch_size)
|
||||
|
||||
# 5. Greedy batch splitting with output constraint
|
||||
batches: list[list[dict]] = []
|
||||
current_batch: list[dict] = []
|
||||
current_tokens = 0
|
||||
@@ -202,7 +179,7 @@ class AdaptiveBatchSizer:
|
||||
if current_batch:
|
||||
batches.append(current_batch)
|
||||
|
||||
# 5. Log adaptive batch statistics
|
||||
# 6. Log stats
|
||||
if batches:
|
||||
avg_size = sum(len(b) for b in batches) / max(1, len(batches))
|
||||
max_size = max(len(b) for b in batches)
|
||||
@@ -213,6 +190,9 @@ class AdaptiveBatchSizer:
|
||||
"max_batch_size": max_size,
|
||||
"recommended_batch_size": recommended,
|
||||
"per_batch_budget": per_batch_budget,
|
||||
"safety_factor": INPUT_SAFETY_FACTOR,
|
||||
"max_rows_by_output": max_rows_by_output,
|
||||
"max_rows_hard_cap": max_rows_hard_cap,
|
||||
})
|
||||
|
||||
return batches
|
||||
|
||||
@@ -40,7 +40,6 @@ MAX_RETRIES_PER_BATCH = 3
|
||||
# #region LLMTranslationService [C:4] [TYPE Class]
|
||||
# @BRIEF Call LLM, handle retry/truncation, parse response, persist records.
|
||||
class LLMTranslationService:
|
||||
"""LLM interaction for batch translation with retry, truncation handling, and parsing."""
|
||||
|
||||
def __init__(self, db: Session) -> None:
|
||||
self.db = db
|
||||
@@ -55,7 +54,6 @@ class LLMTranslationService:
|
||||
batch_rows: list[dict[str, Any]], dict_matches: list[dict[str, Any]],
|
||||
batch_id: str, max_tokens: int = 8192, _recursion_depth: int = 0,
|
||||
) -> dict[str, int]:
|
||||
"""Call LLM for a batch of rows; parse response; create records."""
|
||||
with belief_scope("LLMTranslationService.call_llm_for_batch"):
|
||||
provider_label = f"{job.provider_id}/{getattr(job, '_provider_model', '?')}"
|
||||
logger.reason(
|
||||
@@ -83,6 +81,34 @@ class LLMTranslationService:
|
||||
return self._handle_llm_failure(batch_rows, run_id, batch_id, retries, last_error)
|
||||
|
||||
if finish_reason == "length" and len(batch_rows) >= 2 and run_id:
|
||||
# Try recovery first: parse partial response, save recovered rows,
|
||||
# retry only missing rows. This avoids binary-splitting already-translated rows.
|
||||
recovered_ids = self._try_recover_partial(
|
||||
llm_response, batch_rows, run_id, batch_id, target_languages,
|
||||
)
|
||||
if recovered_ids:
|
||||
remaining = [
|
||||
r for r in batch_rows
|
||||
if str(r.get("row_index", "")) not in recovered_ids
|
||||
]
|
||||
if remaining and len(remaining) < len(batch_rows):
|
||||
logger.reason(
|
||||
f"Retrying only {len(remaining)}/{len(batch_rows)} missing rows",
|
||||
{"batch_id": batch_id, "recovered": len(recovered_ids), "remaining": len(remaining)},
|
||||
)
|
||||
return self._retry_missing_rows(
|
||||
job, run_id, remaining, dict_matches,
|
||||
batch_id, max_tokens, _recursion_depth,
|
||||
)
|
||||
# All rows recovered — nothing to retry
|
||||
if not remaining:
|
||||
logger.reason(
|
||||
"All rows recovered from truncated response",
|
||||
{"batch_id": batch_id, "recovered": len(recovered_ids)},
|
||||
)
|
||||
return {"successful": len(recovered_ids), "failed": 0, "skipped": 0, "retries": 0}
|
||||
|
||||
# Fall back to binary split if recovery didn't help
|
||||
if _recursion_depth < MAX_RETRIES_PER_BATCH:
|
||||
logger.reason(
|
||||
f"Splitting truncated batch", {
|
||||
@@ -117,6 +143,8 @@ class LLMTranslationService:
|
||||
return result
|
||||
# #endregion call_llm_for_batch
|
||||
|
||||
# #region _build_dictionary_section [C:2] [TYPE Function] [SEMANTICS translate, llm, dictionary]
|
||||
# @BRIEF Build dictionary section string for LLM prompt from matched entries.
|
||||
def _build_dictionary_section(self, dict_matches, batch_rows) -> str:
|
||||
if not dict_matches:
|
||||
return ""
|
||||
@@ -124,12 +152,18 @@ class LLMTranslationService:
|
||||
annotated = ContextAwarePromptBuilder.build_context_entries(dict_matches, row_context)
|
||||
return "Terminology dictionary (use these translations when applicable):\n" + \
|
||||
"\n".join(f"- {a}" for a in annotated) + "\n\n"
|
||||
# #endregion _build_dictionary_section
|
||||
|
||||
# #region _resolve_target_languages [C:2] [TYPE Function] [SEMANTICS translate, llm, languages]
|
||||
# @BRIEF Resolve target language list from job config.
|
||||
@staticmethod
|
||||
def _resolve_target_languages(job):
|
||||
langs = job.target_languages or [job.target_dialect or "en"]
|
||||
return [str(langs)] if not isinstance(langs, list) else langs
|
||||
# #endregion _resolve_target_languages
|
||||
|
||||
# #region _build_prompt [C:2] [TYPE Function] [SEMANTICS translate, llm, prompt]
|
||||
# @BRIEF Build the full LLM prompt from batch rows, dictionary, and target languages.
|
||||
@staticmethod
|
||||
def _build_prompt(job, batch_rows, dictionary_section, target_languages):
|
||||
target_languages_str = ", ".join(target_languages)
|
||||
@@ -148,7 +182,11 @@ class LLMTranslationService:
|
||||
"rows_json": rows_json,
|
||||
"row_count": str(len(batch_rows)),
|
||||
})
|
||||
# #endregion _build_prompt
|
||||
|
||||
# #region _call_llm_with_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, retry]
|
||||
# @BRIEF Call LLM with retry loop (max 3 attempts, exponential backoff).
|
||||
# @SIDE_EFFECT HTTP calls to LLM provider on each attempt.
|
||||
def _call_llm_with_retry(self, job, prompt, batch_id, max_tokens):
|
||||
llm_response = None
|
||||
last_error = None
|
||||
@@ -174,7 +212,11 @@ class LLMTranslationService:
|
||||
if llm_response is None:
|
||||
logger.explore(f"All LLM retries exhausted", {"batch_id": batch_id, "retries": retries, "last_error": last_error})
|
||||
return llm_response, finish_reason, retries, last_error
|
||||
# #endregion _call_llm_with_retry
|
||||
|
||||
# #region _handle_llm_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, failure-handling]
|
||||
# @BRIEF Handle complete LLM failure — mark all batch rows as FAILED.
|
||||
# @SIDE_EFFECT DB writes for each row in the batch.
|
||||
def _handle_llm_failure(self, batch_rows, run_id, batch_id, retries, last_error):
|
||||
for row in batch_rows:
|
||||
self.db.add(TranslationRecord(
|
||||
@@ -186,7 +228,11 @@ class LLMTranslationService:
|
||||
error_message=f"LLM call failed after {retries} retries: {last_error}",
|
||||
))
|
||||
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
|
||||
# #endregion _handle_llm_failure
|
||||
|
||||
# #region _split_and_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, split, retry]
|
||||
# @BRIEF Binary-split a batch and retry each half recursively.
|
||||
# @SIDE_EFFECT Creates two child TranslationBatch rows; recursive LLM calls.
|
||||
def _split_and_retry(self, job, run_id, batch_rows, dict_matches, batch_id, max_tokens, depth, retries):
|
||||
mid = len(batch_rows) // 2
|
||||
logger.explore("LLM output truncated — splitting batch",
|
||||
@@ -232,6 +278,112 @@ class LLMTranslationService:
|
||||
"skipped": left["skipped"] + right["skipped"],
|
||||
"retries": retries + left.get("retries", 0) + right.get("retries", 0)}
|
||||
|
||||
# #region _try_recover_partial [C:3] [TYPE Function] [SEMANTICS translate, llm, recovery]
|
||||
# @BRIEF Try to recover translated rows from a truncated LLM response.
|
||||
# Saves recovered rows as SUCCESS records. Returns set of recovered row_index values.
|
||||
# @PRE llm_response is valid text (possibly truncated JSON). batch_rows is non-empty.
|
||||
# @POST Returns set of recovered row indices, or None if no rows could be recovered from the partial response.
|
||||
# @SIDE_EFFECT Creates TranslationRecord + TranslationLanguage rows for recovered rows.
|
||||
# @RATIONALE Instead of binary-splitting (which loses all progress), saves whatever
|
||||
# the model produced before hitting max_tokens. Only unrecovered rows are retried.
|
||||
def _try_recover_partial(self, llm_response, batch_rows, run_id, batch_id, target_languages):
|
||||
try:
|
||||
translations = parse_llm_response(
|
||||
llm_response, len(batch_rows), target_languages=target_languages,
|
||||
finish_reason="length",
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if not translations:
|
||||
return None
|
||||
|
||||
recovered_ids: set[str] = set()
|
||||
for row in batch_rows:
|
||||
row_id = str(row.get("row_index", ""))
|
||||
if row_id in translations:
|
||||
recovered_ids.add(row_id)
|
||||
|
||||
if not recovered_ids:
|
||||
return None
|
||||
|
||||
# Persist recovered rows as SUCCESS records
|
||||
recovered_rows = [
|
||||
r for r in batch_rows
|
||||
if str(r.get("row_index", "")) in recovered_ids
|
||||
]
|
||||
for row in recovered_rows:
|
||||
row_id = str(row.get("row_index", ""))
|
||||
td = translations[row_id]
|
||||
source_text = row.get("source_text", "")
|
||||
detected_lang = row.get("_detected_lang", "und") or "und"
|
||||
if detected_lang == "und" and td:
|
||||
detected_lang = td.get("detected_source_language", "und") or "und"
|
||||
|
||||
plv = self._extract_per_lang_values(td, target_languages)
|
||||
primary = next(iter(plv.values()), "")
|
||||
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id,
|
||||
source_sql=source_text, target_sql=primary,
|
||||
source_object_type="table_row", source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
source_data=row.get("source_data"), source_hash=row.get("_source_hash"),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
for lang_code in target_languages:
|
||||
if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower():
|
||||
continue
|
||||
val = plv.get(lang_code, "")
|
||||
self.db.add(TranslationLanguage(
|
||||
id=str(uuid.uuid4()), record_id=record.id, language_code=lang_code,
|
||||
source_language_detected=detected_lang, translated_value=val or "",
|
||||
final_value=val or "", status="translated", needs_review=(detected_lang == "und"),
|
||||
))
|
||||
|
||||
logger.reason(
|
||||
f"Recovered {len(recovered_ids)}/{len(batch_rows)} rows from truncated response",
|
||||
{"batch_id": batch_id, "recovered": len(recovered_ids), "total": len(batch_rows)},
|
||||
)
|
||||
return recovered_ids
|
||||
# #endregion _try_recover_partial
|
||||
|
||||
# #region _retry_missing_rows [C:3] [TYPE Function] [SEMANTICS translate, llm, retry]
|
||||
# @BRIEF Retry only the rows that were not recovered from a truncated response.
|
||||
# Creates a new sub-batch for the missing rows and calls call_llm_for_batch recursively.
|
||||
# @PRE missing_rows is a subset of the original batch rows, or empty.
|
||||
# @POST Returns dict with successful/failed/skipped counts from the sub-batch.
|
||||
# @SIDE_EFFECT Creates TranslationBatch for the retry sub-batch; may recurse.
|
||||
def _retry_missing_rows(self, job, run_id, missing_rows, dict_matches, _batch_id, max_tokens, depth):
|
||||
if not missing_rows:
|
||||
return {"successful": 0, "failed": 0, "skipped": 0, "retries": 0}
|
||||
|
||||
sub_batch = TranslationBatch(
|
||||
id=str(uuid.uuid4()), run_id=run_id, batch_index=-1,
|
||||
status="RUNNING", total_records=len(missing_rows),
|
||||
started_at=datetime.now(UTC),
|
||||
)
|
||||
self.db.add(sub_batch)
|
||||
self.db.flush()
|
||||
|
||||
result = self.call_llm_for_batch(
|
||||
job, run_id, missing_rows, dict_matches,
|
||||
sub_batch.id, max_tokens, depth,
|
||||
)
|
||||
|
||||
sub_batch.completed_at = datetime.now(UTC)
|
||||
sub_batch.successful_records = result["successful"]
|
||||
sub_batch.failed_records = result["failed"]
|
||||
sub_batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS"
|
||||
self.db.flush()
|
||||
|
||||
return result
|
||||
# #endregion _retry_missing_rows
|
||||
|
||||
# #region _handle_parse_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, parse, failure]
|
||||
# @BRIEF Handle LLM parse failure — mark rows as SKIPPED.
|
||||
# @SIDE_EFFECT DB writes for each row in the batch.
|
||||
def _handle_parse_failure(self, batch_rows, run_id, batch_id, retries, error):
|
||||
for row in batch_rows:
|
||||
self.db.add(TranslationRecord(
|
||||
@@ -243,7 +395,11 @@ class LLMTranslationService:
|
||||
error_message=f"LLM parse failure: {error}",
|
||||
))
|
||||
return {"successful": 0, "failed": 0, "skipped": len(batch_rows), "retries": retries}
|
||||
# #endregion _handle_parse_failure
|
||||
|
||||
# #region _create_records_from_translations [C:3] [TYPE Function] [SEMANTICS translate, llm, persist]
|
||||
# @BRIEF Create TranslationRecord + TranslationLanguage rows from parsed LLM translations.
|
||||
# @SIDE_EFFECT DB writes for each successfully translated row.
|
||||
def _create_records_from_translations(self, batch_rows, run_id, batch_id, target_languages, translations, dict_matches, retries):
|
||||
successful = failed = skipped = 0
|
||||
for row in batch_rows:
|
||||
@@ -290,7 +446,10 @@ class LLMTranslationService:
|
||||
final_value=val or "", status="translated", needs_review=needs_review,
|
||||
))
|
||||
return {"successful": successful, "failed": failed, "skipped": skipped, "retries": retries}
|
||||
# #endregion _create_records_from_translations
|
||||
|
||||
# #region _extract_per_lang_values [C:2] [TYPE Function] [SEMANTICS translate, llm, parse]
|
||||
# @BRIEF Extract per-language translation values from a parsed LLM response row.
|
||||
@staticmethod
|
||||
def _extract_per_lang_values(td, target_languages):
|
||||
plv = {}
|
||||
@@ -306,7 +465,11 @@ class LLMTranslationService:
|
||||
plv[target_languages[0]] = t
|
||||
has_any = True
|
||||
return plv if has_any else {}
|
||||
# #endregion _extract_per_lang_values
|
||||
|
||||
# #region _add_skipped [C:3] [TYPE Function] [SEMANTICS translate, llm, record]
|
||||
# @BRIEF Add a SKIPPED TranslationRecord for a row with no translation data.
|
||||
# @SIDE_EFFECT DB write to translation_records.
|
||||
def _add_skipped(self, row, run_id, batch_id, source_text, reason):
|
||||
self.db.add(TranslationRecord(
|
||||
id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id,
|
||||
@@ -316,11 +479,11 @@ class LLMTranslationService:
|
||||
source_data=row.get("source_data"), status="SKIPPED",
|
||||
error_message=reason,
|
||||
))
|
||||
# #endregion _add_skipped
|
||||
|
||||
# #region call_llm [C:3] [TYPE Function] [SEMANTICS translate, llm, call]
|
||||
# @BRIEF Route to provider-specific LLM call implementation.
|
||||
def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> tuple[str, str | None]:
|
||||
"""Call the configured LLM provider with the batch prompt."""
|
||||
with belief_scope("LLMTranslationService.call_llm"):
|
||||
if not job.provider_id:
|
||||
raise ValueError("Job has no LLM provider configured")
|
||||
@@ -358,13 +521,18 @@ class LLMTranslationService:
|
||||
return result
|
||||
# #endregion call_llm
|
||||
|
||||
# -- Static methods delegated to sub-modules for backward compat --
|
||||
# #region call_openai_compatible [C:1] [TYPE Function] [SEMANTICS translate, llm, compat]
|
||||
# @BRIEF Backward-compat delegation to _llm_http.call_openai_compatible.
|
||||
@staticmethod
|
||||
def call_openai_compatible(*a, **kw):
|
||||
return call_openai_compatible(*a, **kw)
|
||||
# #endregion call_openai_compatible
|
||||
|
||||
# #region _parse_llm_response [C:1] [TYPE Function] [SEMANTICS translate, llm, compat]
|
||||
# @BRIEF Backward-compat delegation to _llm_parse.parse_llm_response.
|
||||
@staticmethod
|
||||
def _parse_llm_response(*a, **kw):
|
||||
return parse_llm_response(*a, **kw)
|
||||
# #endregion _parse_llm_response
|
||||
# #endregion LLMTranslationService
|
||||
# #endregion LLMTranslationService
|
||||
|
||||
@@ -125,6 +125,21 @@ def call_openai_compatible(
|
||||
})
|
||||
raise ValueError(f"LLM response processing failed: {e}")
|
||||
|
||||
# Log provider token usage for batch sizing calibration
|
||||
usage = data.get("usage") or {}
|
||||
if usage:
|
||||
logger.reason(
|
||||
"LLM provider usage",
|
||||
{
|
||||
"prompt_tokens": usage.get("prompt_tokens"),
|
||||
"completion_tokens": usage.get("completion_tokens"),
|
||||
"total_tokens": usage.get("total_tokens"),
|
||||
"finish_reason": finish_reason,
|
||||
"max_tokens_sent": max_tokens,
|
||||
"chars_sent": len(prompt),
|
||||
},
|
||||
)
|
||||
|
||||
refusal = msg.get("refusal") if isinstance(msg, dict) else None
|
||||
if refusal:
|
||||
logger.explore("LLM refused to respond", extra={
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
# #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm]
|
||||
# @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits.
|
||||
# Output budget is now the PRIMARY constraint — finish_reason=length is usually output exceeding max_tokens, not input overflowing context_window.
|
||||
# @LAYER Domain
|
||||
# @RELATION DEPENDS_ON -> [TranslationExecutor]
|
||||
# @RELATION DEPENDS_ON -> [TranslationExecutor]
|
||||
# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API.
|
||||
|
||||
# @RATIONALE CJK ratio lowered from 1.5→1.0 and other from 2.2→1.8 — modern models (Qwen, DeepSeek)
|
||||
# tokenize more densely. SAFETY_FACTOR of 0.75 applied to input budget to account for
|
||||
# variance across tokenizers. Output budget now computed first and serves as the hard cap.
|
||||
# PROVIDER_DEFAULTS is a fallback — provider-level context_window/max_output_tokens in DB take priority.
|
||||
# DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens.
|
||||
# @REJECTED External tokenizer library — would introduce heavy dependency for estimation only.
|
||||
# Fixed batch_size of 50 — causes truncation on long-content rows.
|
||||
# CJK ratio 1.5 — too optimistic for Qwen/DeepSeek tokenizers (actual ~1.0-1.2).
|
||||
# Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length).
|
||||
|
||||
# #region DEFAULT_CONTEXT_WINDOW [TYPE Constant]
|
||||
|
||||
@@ -64,9 +68,35 @@ DICT_TOKENS_PER_ENTRY = 20
|
||||
DICT_TOKENS_MAX = 5000
|
||||
# #endregion DICT_TOKENS_MAX
|
||||
# #region CHARS_PER_TOKEN_MIXED [TYPE Constant]
|
||||
|
||||
# @BRIEF Deprecated — use CJK_RATIO + OTHER_RATIO instead. Kept for backward compat in estimate_row_tokens.
|
||||
CHARS_PER_TOKEN_MIXED = 2.2
|
||||
# #endregion CHARS_PER_TOKEN_MIXED
|
||||
|
||||
# #region CJK_RATIO [TYPE Constant]
|
||||
# @BRIEF Conservative CJK chars/token ratio. Modern models (Qwen, DeepSeek) tokenize at ~1.0-1.3.
|
||||
# @RATIONALE Lowered from 1.5 to 1.0 to produce more conservative (higher) token estimates.
|
||||
# Actual token density varies by model tokenizer; this is intentionally pessimistic.
|
||||
CJK_RATIO = 1.0
|
||||
# #endregion CJK_RATIO
|
||||
|
||||
# #region OTHER_RATIO [TYPE Constant]
|
||||
# @BRIEF Conservative ratio for non-CJK text. cl100k_base averages ~1.8-2.5.
|
||||
OTHER_RATIO = 1.8
|
||||
# #endregion OTHER_RATIO
|
||||
|
||||
# #region INPUT_SAFETY_FACTOR [TYPE Constant]
|
||||
# @BRIEF Multiplier applied to per-batch input budget in batch sizer.
|
||||
# Accounts for CJK estimation variance across different tokenizers.
|
||||
# @RATIONALE Single centralised factor instead of scattered margins in multiple functions.
|
||||
# 0.75 means batch uses at most 75% of estimated budget → 33% headroom.
|
||||
INPUT_SAFETY_FACTOR = 0.75
|
||||
# #endregion INPUT_SAFETY_FACTOR
|
||||
|
||||
# #region OUTPUT_SAFETY_FACTOR [TYPE Constant]
|
||||
# @BRIEF Multiplier applied to output budget when computing max rows per batch.
|
||||
OUTPUT_SAFETY_FACTOR = 0.70
|
||||
# #endregion OUTPUT_SAFETY_FACTOR
|
||||
|
||||
# #region MIN_MAX_TOKENS [TYPE Constant]
|
||||
|
||||
MIN_MAX_TOKENS = 4096
|
||||
@@ -80,18 +110,16 @@ MAX_OUTPUT_HEADROOM = 3000
|
||||
|
||||
# region _estimate_tokens_for_text [TYPE Function]
|
||||
# @BRIEF Estimate token count for a text string with CJK-aware heuristics.
|
||||
# CJK characters (~1.5 chars/token) vs other text (~2.2 chars/token).
|
||||
# Conservative ratios: CJK ~1.0 chars/token, other ~1.8 chars/token.
|
||||
# @PRE text is a string.
|
||||
# @POST Returns estimated token count >= 1.
|
||||
# @RATIONALE CJK characters are more token-dense than Latin/Cyrillic text.
|
||||
# Using a single ratio undercounts CJK input and causes truncation.
|
||||
# @POST Returns estimated token count >= 1. Intentionally pessimistic (over-estimates)
|
||||
# to prevent LLM truncation from content that tokenizes more densely than expected.
|
||||
# @RATIONALE Lowered ratios for modern models: Qwen/DeepSeek tokenize CJK at ~1.0-1.3.
|
||||
# cl100k_base averages ~1.8-2.5 for non-CJK. Using conservative values prevents
|
||||
# the primary cause of finish_reason=length: underestimating actual token count.
|
||||
# @REJECTED Using tiktoken — would introduce a heavy dependency for estimation only.
|
||||
# CJK ratio 1.5 — too optimistic, caused 3x underestimation in production logs.
|
||||
def _estimate_tokens_for_text(text: str) -> int:
|
||||
"""Estimate token count with CJK-aware heuristics.
|
||||
|
||||
CJK characters (CJK Unified Ideographs) use ~1.5 chars/token.
|
||||
All other characters use ~2.2 chars/token.
|
||||
"""
|
||||
if not text:
|
||||
return 1
|
||||
|
||||
@@ -103,8 +131,8 @@ def _estimate_tokens_for_text(text: str) -> int:
|
||||
else:
|
||||
other_count += 1
|
||||
|
||||
cjk_tokens = cjk_count / 1.5 if cjk_count else 0
|
||||
other_tokens = other_count / 2.2 if other_count else 0
|
||||
cjk_tokens = cjk_count / CJK_RATIO if cjk_count else 0
|
||||
other_tokens = other_count / OTHER_RATIO if other_count else 0
|
||||
return max(1, int(cjk_tokens + other_tokens))
|
||||
# endregion _estimate_tokens_for_text
|
||||
|
||||
@@ -157,6 +185,28 @@ def _calculate_output_tokens(
|
||||
)
|
||||
|
||||
|
||||
def _compute_max_rows_by_output(max_output_tokens: int, num_languages: int) -> int:
|
||||
"""Compute the maximum number of rows that fit within the output budget.
|
||||
|
||||
This is the PRIMARY constraint — finish_reason=length most often occurs when
|
||||
the model's generated JSON output exceeds max_tokens, not when input overflows
|
||||
context_window.
|
||||
|
||||
@RATIONALE Output budget is the dominant cause of truncation. Input budget
|
||||
is secondary because modern models have large context windows (64K-256K)
|
||||
but limited output tokens (8K-16K). Computing output-first prevents
|
||||
packing too many rows that the model cannot fit in its response.
|
||||
"""
|
||||
overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM
|
||||
per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
|
||||
if per_row <= 0:
|
||||
return 20 # sensible fallback
|
||||
available = int((max_output_tokens - overhead) * OUTPUT_SAFETY_FACTOR)
|
||||
if available <= 0:
|
||||
return 1
|
||||
return max(available // per_row, 1)
|
||||
|
||||
|
||||
def _apply_output_aware_batch_sizing(
|
||||
safe_size: int,
|
||||
num_languages: int,
|
||||
@@ -330,6 +380,8 @@ def estimate_token_budget(
|
||||
dict_warning,
|
||||
)
|
||||
|
||||
max_rows_by_output = _compute_max_rows_by_output(max_output_tokens, num_languages)
|
||||
|
||||
return {
|
||||
"batch_size_adjusted": safe_size,
|
||||
"estimated_input_tokens": estimated_input,
|
||||
@@ -338,8 +390,6 @@ def estimate_token_budget(
|
||||
"warning": warning,
|
||||
"available_input_budget": available_input_budget,
|
||||
"max_output_tokens": max_output_tokens,
|
||||
"max_rows_by_output": max_rows_by_output,
|
||||
}
|
||||
# endregion estimate_token_budget
|
||||
# #endregion estimate_token_budget
|
||||
# endregion estimate_token_budget
|
||||
# #endregion estimate_token_budget
|
||||
|
||||
@@ -221,11 +221,23 @@ class TranslationExecutor:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _resolve_provider_config(self, job) -> dict:
|
||||
if not job.provider_id:
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
try:
|
||||
return LLMProviderService(self.db).get_provider_token_config(job.provider_id)
|
||||
except Exception:
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
|
||||
# Backward-compat alias for preview.py (calls via resolve_provider_model, not _resolve_provider_model)
|
||||
def resolve_provider_model(self, job) -> str | None:
|
||||
return self._resolve_provider_model(job)
|
||||
|
||||
def _auto_size_batches(self, job, source_rows, target_languages, provider_info=None) -> list:
|
||||
"""Split source rows into auto-sized batches based on content length."""
|
||||
from ._batch_sizer import AdaptiveBatchSizer
|
||||
if provider_info is None:
|
||||
provider_info = self._resolve_provider_model(job)
|
||||
# AdaptiveBatchSizer.auto_size_batches resolves provider config internally
|
||||
# via job.provider_id. provider_info is optional override for tests.
|
||||
return AdaptiveBatchSizer(self.db, self.config_manager).auto_size_batches(
|
||||
job, source_rows, target_languages, provider_info)
|
||||
|
||||
|
||||
@@ -380,3 +380,121 @@ def test_llm_provider_config_multimodal_explicit():
|
||||
is_multimodal=True,
|
||||
)
|
||||
assert config.is_multimodal is True
|
||||
|
||||
|
||||
# endregion test_llm_provider_config_multimodal_explicit
|
||||
|
||||
# region test_llm_provider_config_context_window_default [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderConfig]
|
||||
# @PURPOSE: Verify LLMProviderConfig.context_window defaults to None.
|
||||
def test_llm_provider_config_context_window_default():
|
||||
"""Verify default context_window is None in schema."""
|
||||
config = LLMProviderConfig(
|
||||
provider_type=LLMProviderType.OPENAI,
|
||||
name="Default Test",
|
||||
base_url="https://api.openai.com/v1",
|
||||
api_key="sk-test",
|
||||
default_model="gpt-4",
|
||||
)
|
||||
assert config.context_window is None
|
||||
|
||||
|
||||
# endregion test_llm_provider_config_context_window_default
|
||||
|
||||
# region test_llm_provider_config_context_window_explicit [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderConfig]
|
||||
# @PURPOSE: Verify LLMProviderConfig accepts explicit context_window value.
|
||||
def test_llm_provider_config_context_window_explicit():
|
||||
"""Verify setting context_window explicitly works."""
|
||||
config = LLMProviderConfig(
|
||||
provider_type=LLMProviderType.OPENAI,
|
||||
name="Test",
|
||||
base_url="https://api.openai.com/v1",
|
||||
api_key="sk-test",
|
||||
default_model="gpt-4",
|
||||
context_window=128000,
|
||||
max_output_tokens=16384,
|
||||
)
|
||||
assert config.context_window == 128000
|
||||
assert config.max_output_tokens == 16384
|
||||
|
||||
|
||||
# endregion test_llm_provider_config_context_window_explicit
|
||||
|
||||
# region test_get_provider_token_config_no_provider [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderService]
|
||||
# @PURPOSE: Verify get_provider_token_config returns all-None when provider not found.
|
||||
def test_get_provider_token_config_no_provider():
|
||||
"""When provider_id doesn't exist, returns all values as None."""
|
||||
db = MagicMock(spec=Session)
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
|
||||
service = LLMProviderService(db)
|
||||
result = service.get_provider_token_config("nonexistent-id")
|
||||
assert result == {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
|
||||
|
||||
# endregion test_get_provider_token_config_no_provider
|
||||
|
||||
# region test_get_provider_token_config_with_values [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderService]
|
||||
# @PURPOSE: Verify get_provider_token_config returns provider token limits from DB.
|
||||
def test_get_provider_token_config_with_values():
|
||||
"""Provider with context_window and max_output_tokens returns them."""
|
||||
db = MagicMock(spec=Session)
|
||||
mock_provider = MagicMock(spec=LLMProvider)
|
||||
mock_provider.default_model = "gpt-4o-mini"
|
||||
mock_provider.context_window = 128000
|
||||
mock_provider.max_output_tokens = 16384
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_provider
|
||||
|
||||
service = LLMProviderService(db)
|
||||
result = service.get_provider_token_config("provider-1")
|
||||
assert result["model"] == "gpt-4o-mini"
|
||||
assert result["context_window"] == 128000
|
||||
assert result["max_output_tokens"] == 16384
|
||||
|
||||
|
||||
# endregion test_get_provider_token_config_with_values
|
||||
|
||||
# region test_get_provider_token_config_null_limits [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderService]
|
||||
# @PURPOSE: Verify get_provider_token_config returns None for null DB fields.
|
||||
def test_get_provider_token_config_null_limits():
|
||||
"""Provider with NULL token limits returns None values (signal to use defaults)."""
|
||||
db = MagicMock(spec=Session)
|
||||
mock_provider = MagicMock(spec=LLMProvider)
|
||||
mock_provider.default_model = "qwen-flash"
|
||||
mock_provider.context_window = None
|
||||
mock_provider.max_output_tokens = None
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_provider
|
||||
|
||||
service = LLMProviderService(db)
|
||||
result = service.get_provider_token_config("provider-2")
|
||||
assert result["model"] == "qwen-flash"
|
||||
assert result["context_window"] is None
|
||||
assert result["max_output_tokens"] is None
|
||||
|
||||
|
||||
# endregion test_get_provider_token_config_null_limits
|
||||
|
||||
# region test_provider_token_config_default_model_fallback [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [LLMProviderService]
|
||||
# @PURPOSE: Verify get_provider_token_config falls back to "gpt-4o-mini" when default_model is None.
|
||||
def test_provider_token_config_default_model_fallback():
|
||||
"""Provider without explicit default_model uses 'gpt-4o-mini' fallback."""
|
||||
db = MagicMock(spec=Session)
|
||||
mock_provider = MagicMock(spec=LLMProvider)
|
||||
mock_provider.default_model = None
|
||||
mock_provider.context_window = None
|
||||
mock_provider.max_output_tokens = None
|
||||
db.query.return_value.filter.return_value.first.return_value = mock_provider
|
||||
|
||||
service = LLMProviderService(db)
|
||||
result = service.get_provider_token_config("provider-3")
|
||||
assert result["model"] == "gpt-4o-mini"
|
||||
assert result["context_window"] is None
|
||||
assert result["max_output_tokens"] is None
|
||||
|
||||
|
||||
# endregion test_provider_token_config_default_model_fallback
|
||||
|
||||
@@ -115,6 +115,8 @@ class LLMProviderService:
|
||||
is_active=config.is_active,
|
||||
is_multimodal=config.is_multimodal,
|
||||
max_images=config.max_images,
|
||||
context_window=config.context_window,
|
||||
max_output_tokens=config.max_output_tokens,
|
||||
)
|
||||
self.db.add(db_provider)
|
||||
self.db.commit()
|
||||
@@ -148,6 +150,8 @@ class LLMProviderService:
|
||||
db_provider.is_active = config.is_active
|
||||
db_provider.is_multimodal = config.is_multimodal
|
||||
db_provider.max_images = config.max_images
|
||||
db_provider.context_window = config.context_window
|
||||
db_provider.max_output_tokens = config.max_output_tokens
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(db_provider)
|
||||
@@ -235,6 +239,24 @@ class LLMProviderService:
|
||||
|
||||
# endregion get_decrypted_api_key
|
||||
|
||||
# region get_provider_token_config [TYPE Function]
|
||||
# @PURPOSE: Returns provider token limits for batch sizing.
|
||||
# @PRE provider_id must be valid.
|
||||
# @POST Returns dict with model name, context_window, max_output_tokens.
|
||||
# Values from DB take priority; None means "use PROVIDER_DEFAULTS fallback".
|
||||
# @RATIONALE Centralised helper — both _batch_proc.py and _batch_sizer.py need
|
||||
# the same resolution logic. Avoids duplicating DB queries and defaults.
|
||||
def get_provider_token_config(self, provider_id: str) -> dict:
|
||||
provider = self.get_provider(provider_id)
|
||||
if not provider:
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
return {
|
||||
"model": provider.default_model or "gpt-4o-mini",
|
||||
"context_window": provider.context_window,
|
||||
"max_output_tokens": provider.max_output_tokens,
|
||||
}
|
||||
# endregion get_provider_token_config
|
||||
|
||||
|
||||
# #endregion LLMProviderService
|
||||
|
||||
|
||||
508
docs/translation-performance-analysis.md
Normal file
508
docs/translation-performance-analysis.md
Normal file
@@ -0,0 +1,508 @@
|
||||
# Анализ производительности перевода: причины медлительности и план доработок
|
||||
|
||||
**Дата:** 2026-06-03 (v2 — после code review)
|
||||
**Автор:** fullstack-coder (ss-tools) + рецензент
|
||||
**Контекст:** Пользователь сообщил "Очень долго стартует перевод". По логам trace_id `8bd7ac8f` (run `4c9de39e`) проведён анализ.
|
||||
|
||||
---
|
||||
|
||||
## 1. Исходные данные
|
||||
|
||||
**Объём:** 5455 строк из Superset datasource (dataset 906, таблица `userdata.debt_comment_translations`)
|
||||
**Модель:** `qwen-flash` через `lite.ai.rusal.com/v1` (provider_type=litellm, response_format=yes)
|
||||
**Режим:** `full=False` (только новые записи, без перезаписи существующих)
|
||||
**Батчей сформировано:** 203
|
||||
|
||||
---
|
||||
|
||||
## 2. Таймлайн одного прогона (из логов)
|
||||
|
||||
| Время | Событие | Длительность | Симптом |
|
||||
|-------|---------|--------------|---------|
|
||||
| `14:34:39` | Run стартовал | — | |
|
||||
| `14:34:40` | Данные загружены (5455 строк) | ~1s | ✅ |
|
||||
| `14:34:40` | "Processing 203 batches" | — | |
|
||||
| `14:34:40.430` | **LLM request:** prompt_len=145062 | **~1m47s** | ⚠️ |
|
||||
| `14:36:27` | `finish_reason=length` — ответ обрезан | | ❌ |
|
||||
| `14:36:27` | Splitting → 2 батча | | |
|
||||
| `14:36:27` | prompt_len=101330 | **~1m40s** | ⚠️ |
|
||||
| `14:38:06` | `finish_reason=length` | | ❌ |
|
||||
| `14:38:06` | Splitting → ещё 2 батча | | |
|
||||
| `14:38:06` | prompt_len=25826 | **~40s** | ✅ stop |
|
||||
| `14:38:47` | prompt_len=76479 | **~1m39s** | ⚠️ |
|
||||
| `14:40:26` | `finish_reason=length` | | ❌ |
|
||||
| ... | каскад продолжается | | |
|
||||
|
||||
**Оценка общего времени:** >10-15 минут на 5455 строк.
|
||||
|
||||
---
|
||||
|
||||
## 3. ⚠️ Важное ограничение анализа: prompt_len — это символы или токены?
|
||||
|
||||
**В логах нет прямого указания, что `prompt_len=145062` — токены.** Формат логирования (`prompt_len=145062`) без указания единиц измерения не позволяет утверждать, что это именно токены. Это могут быть символы.
|
||||
|
||||
**До любых правок требуется:**
|
||||
|
||||
Для 10-20 реальных батчей залогировать:
|
||||
|
||||
| Поле | Источник | Зачем |
|
||||
|------|----------|-------|
|
||||
| `chars` | `len(prompt)` | Длина в символах |
|
||||
| `estimated_input_tokens` | `estimate_token_budget()` | Текущая оценка |
|
||||
| `provider_prompt_tokens` | `response.usage.prompt_tokens` | Реальные токены входа |
|
||||
| `provider_completion_tokens` | `response.usage.completion_tokens` | Реальные токены выхода |
|
||||
| `provider_total_tokens` | `response.usage.total_tokens` | Сумма |
|
||||
| `max_tokens` | Параметр запроса | Сколько просили |
|
||||
| `context_window_resolved` | Что использовали как контекст | 64000 или другое |
|
||||
| `max_output_tokens_resolved` | Что использовали как лимит выхода | |
|
||||
| `rows_in_batch` | `len(batch_rows)` | |
|
||||
| `target_languages_count` | `len(target_languages)` | |
|
||||
| `finish_reason` | Из ответа API | stop / length / error |
|
||||
| `response_rows_recovered` | Сколько строк распарсили | Для recovery |
|
||||
|
||||
**Вывод:** Все гипотезы ниже основаны на косвенных признаках. Без логов usage токенов от провайдера (response.usage) некоторые причины остаются недоказанными. Добавление этих логов — **P0, первый шаг**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Первопричины (по степени вероятности)
|
||||
|
||||
### 4.1. Batch sizing недооценивает output budget (основная гипотеза)
|
||||
|
||||
`finish_reason=length` с вероятностью >90% означает не "вход не влез во входной контекст", а **"модель упёрлась в max_tokens при генерации ответа"**.
|
||||
|
||||
Каждый батч содержит N строк. Для каждой строки модель должна вернуть JSON с переводами на каждый из target_languages. Если target_languages_count > 1, то **выход растёт линейно**, а batch sizing учитывает это только грубой оценкой.
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_token_budget.py`
|
||||
|
||||
Текущие константы для оценки выхода:
|
||||
|
||||
```python
|
||||
OUTPUT_PER_ROW_PER_LANG = 120 # токенов на строку перевода на один язык
|
||||
JSON_OVERHEAD_PER_ROW = 50 # JSON-обвязка на строку
|
||||
REASONING_OVERHEAD = 2000 # CoT overhead
|
||||
MAX_OUTPUT_HEADROOM = 3000 # запас
|
||||
```
|
||||
|
||||
Для 128 строк × 2 языка:
|
||||
```
|
||||
нужно = 128 × 2 × 120 + 128 × 50 + 2000 + 3000 = 40560 токенов
|
||||
```
|
||||
|
||||
Если `max_output_tokens = 16384` (default), то батч гарантированно обрежется.
|
||||
И в логе мы видим `finish_reason=length` на батчах > 50-60 строк.
|
||||
|
||||
**Следствие:** Проблема не (только) в CJK-токенизации, а в том, что **батч-сайзер упаковывает слишком много строк относительно output лимита**.
|
||||
|
||||
### 4.2. CJK-оценка токенов входа — дополнительный фактор
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_token_budget.py:89-108`
|
||||
|
||||
```python
|
||||
cjk_tokens = cjk_count / 1.5 # 1.5 chars/token
|
||||
other_tokens = other_count / 2.2 # 2.2 chars/token
|
||||
```
|
||||
|
||||
Если `prompt_len` в логах — символы, а не токены, то при 60% CJK-символов:
|
||||
- Оценка: 145062 / 1.5 ≈ 96708 токенов
|
||||
- Реальность (Qwen): может быть ~120000+ токенов
|
||||
|
||||
То есть вход недооценивается на 20-30%, и "съедает" часть output budget.
|
||||
|
||||
**Вывод:** CJK-оценка — вторичный фактор. Первичный — output budget.
|
||||
|
||||
### 4.3. PROVIDER_DEFAULTS не содержит модели qwen-flash
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_token_budget.py:32-39`
|
||||
|
||||
```python
|
||||
PROVIDER_DEFAULTS = {
|
||||
"gpt-4o-mini": {"context_window": 128000, "max_output_tokens": 16384},
|
||||
"gpt-4o": {"context_window": 128000, "max_output_tokens": 16384},
|
||||
"o1-mini": {"context_window": 128000, "max_output_tokens": 65536},
|
||||
"claude-3-5-sonnet": {"context_window": 200000, "max_output_tokens": 8192},
|
||||
"deepseek-v4-flash": {"context_window": 64000, "max_output_tokens": 8192},
|
||||
"default": {"context_window": 64000, "max_output_tokens": 16384},
|
||||
}
|
||||
```
|
||||
|
||||
Когда модель не найдена:
|
||||
- `context_window = 64000` (default)
|
||||
- `max_output_tokens = 16384` (default)
|
||||
- `available_input_budget = 64000 - 16384 = 47616`
|
||||
|
||||
Если `qwen-flash` на самом деле поддерживает 128K контекст и 8K вывод — бюджет по входу может быть недооценён, а бюджет по выходу переоценён.
|
||||
|
||||
### 4.4. Каскад finish_reason=length умножает проблему
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_llm_call.py:85-96, 190-233`
|
||||
|
||||
```python
|
||||
if finish_reason == "length" and len(batch_rows) >= 2:
|
||||
if _recursion_depth < MAX_RETRIES_PER_BATCH: # = 3
|
||||
return self._split_and_retry(...) # binary split
|
||||
|
||||
def _split_and_retry(self, ...):
|
||||
mid = len(batch_rows) // 2
|
||||
left = self.call_llm_for_batch(..., rows[:mid], depth + 1)
|
||||
right = self.call_llm_for_batch(..., rows[mid:], depth + 1)
|
||||
```
|
||||
|
||||
**Проблема:** Бинарное деление **не спасает частичный результат**. Даже если модель вернула 80 из 100 строк валидного JSON — они теряются, и обе половины перезапрашиваются с нуля.
|
||||
|
||||
Если truncation случается на 3+ уровнях рекурсии — 1 батч превращается в 7+ LLM-вызовов.
|
||||
|
||||
---
|
||||
|
||||
## 5. План доработок
|
||||
|
||||
### 5.0. [P0] Измерить → потом править
|
||||
|
||||
Без реальных цифр любое изменение — гадание.
|
||||
|
||||
**Добавить в `_llm_http.py` сбор usage от провайдера и логирование:**
|
||||
|
||||
```python
|
||||
# После ответа API:
|
||||
usage = response.get("usage", {})
|
||||
log("llm_http", "REFLECT", "LLM usage stats", {
|
||||
"prompt_tokens": usage.get("prompt_tokens"),
|
||||
"completion_tokens": usage.get("completion_tokens"),
|
||||
"total_tokens": usage.get("total_tokens"),
|
||||
"finish_reason": finish_reason,
|
||||
"max_tokens": max_tokens,
|
||||
"rows": len(batch_rows),
|
||||
"chars": len(prompt),
|
||||
})
|
||||
```
|
||||
|
||||
Для 10-20 реальных батчей собрать статистику и **только после этого** принимать решения о коэффициентах.
|
||||
|
||||
### 5.1. [P0] Учитывать output budget при расчёте размера батча
|
||||
|
||||
**Проблема:** Сейчас output budget учитывается, но недостаточно жёстко.
|
||||
**Файл:** `backend/src/plugins/translate/_token_budget.py:160-176`
|
||||
|
||||
```python
|
||||
def _apply_output_aware_batch_sizing(safe_size, num_languages, max_output_tokens):
|
||||
while safe_size > 0:
|
||||
needed_output = (
|
||||
safe_size * num_languages * OUTPUT_PER_ROW_PER_LANG
|
||||
+ safe_size * JSON_OVERHEAD_PER_ROW
|
||||
+ REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM
|
||||
)
|
||||
if needed_output <= max_output_tokens:
|
||||
break
|
||||
safe_size -= 1
|
||||
return safe_size
|
||||
```
|
||||
|
||||
**Улучшение:** Сделать output budget **первичным** ограничителем, а input budget — вторичным:
|
||||
|
||||
```python
|
||||
def _compute_max_rows_by_output(max_output_tokens, num_languages):
|
||||
"""Сколько строк влезет в max_output_tokens."""
|
||||
overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM
|
||||
per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
|
||||
if per_row <= 0:
|
||||
return 20
|
||||
available = max_output_tokens - overhead
|
||||
if available <= 0:
|
||||
return 1
|
||||
return max(available // per_row, 1)
|
||||
```
|
||||
|
||||
И в `_batch_sizer.py:auto_size_batches()`:
|
||||
|
||||
```python
|
||||
max_rows_by_output = _compute_max_rows_by_output(max_output_tokens_val, num_languages)
|
||||
|
||||
# Брать минимум из всех ограничений:
|
||||
max_rows = min(
|
||||
max_rows_by_input_budget,
|
||||
max_rows_by_output,
|
||||
absolute_hard_cap, # safety net
|
||||
job.batch_size or inf, # user preference
|
||||
)
|
||||
```
|
||||
|
||||
### 5.2. [P0] Вынести context_window / max_output_tokens в настройки провайдера
|
||||
|
||||
#### 5.2.1. Модель БД
|
||||
|
||||
**Файл:** `backend/src/models/llm.py`
|
||||
|
||||
```python
|
||||
class LLMProvider(Base):
|
||||
# ... существующие поля ...
|
||||
context_window = Column(
|
||||
Integer, nullable=True, default=None,
|
||||
comment="Total context window in tokens. NULL = fallback to PROVIDER_DEFAULTS",
|
||||
)
|
||||
max_output_tokens = Column(
|
||||
Integer, nullable=True, default=None,
|
||||
comment="Max output tokens. NULL = fallback to PROVIDER_DEFAULTS",
|
||||
)
|
||||
```
|
||||
|
||||
Nullable → обратная совместимость.
|
||||
|
||||
#### 5.2.2. Safe cap
|
||||
|
||||
Даже если пользователь ввёл значения — применяется верхняя граница:
|
||||
|
||||
```python
|
||||
PROVIDER_SAFE_CAP = 256000 # абсолютный максимум
|
||||
|
||||
effective_context_window = min(
|
||||
provider.context_window or PROVIDER_DEFAULTS.get(model, default)["context_window"],
|
||||
PROVIDER_SAFE_CAP,
|
||||
)
|
||||
effective_max_output_tokens = min(
|
||||
provider.max_output_tokens or PROVIDER_DEFAULTS.get(model, default)["max_output_tokens"],
|
||||
effective_context_window, # не может быть больше контекста
|
||||
)
|
||||
```
|
||||
|
||||
#### 5.2.3. Pydantic схема
|
||||
|
||||
**Файл:** `backend/src/plugins/llm_analysis/models.py`
|
||||
|
||||
```python
|
||||
class LLMProviderConfig(BaseModel):
|
||||
# ... существующие поля ...
|
||||
context_window: int | None = Field(
|
||||
None, ge=1000, le=256000,
|
||||
description="Context window in tokens. Leave blank for auto.",
|
||||
)
|
||||
max_output_tokens: int | None = Field(
|
||||
None, ge=256,
|
||||
description="Max output tokens. Must be less than context_window.",
|
||||
)
|
||||
```
|
||||
|
||||
#### 5.2.4. Сервисный слой
|
||||
|
||||
**Файл:** `backend/src/services/llm_provider.py`
|
||||
|
||||
```python
|
||||
# create_provider
|
||||
db_provider = LLMProvider(
|
||||
...
|
||||
context_window=config.context_window,
|
||||
max_output_tokens=config.max_output_tokens,
|
||||
)
|
||||
|
||||
# update_provider
|
||||
db_provider.context_window = config.context_window
|
||||
db_provider.max_output_tokens = config.max_output_tokens
|
||||
|
||||
# Новый хелпер для batch sizing:
|
||||
def get_provider_token_config(self, provider_id: str) -> dict:
|
||||
provider = self.get_provider(provider_id)
|
||||
if not provider:
|
||||
return {"model": None, "context_window": None, "max_output_tokens": None}
|
||||
return {
|
||||
"model": provider.default_model or "gpt-4o-mini",
|
||||
"context_window": provider.context_window,
|
||||
"max_output_tokens": provider.max_output_tokens,
|
||||
}
|
||||
```
|
||||
|
||||
#### 5.2.5. Интеграция в batch sizing
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_batch_proc.py:208-247`
|
||||
**Файл:** `backend/src/plugins/translate/_batch_sizer.py:70-218`
|
||||
|
||||
В обоих местах заменить:
|
||||
```python
|
||||
# Было:
|
||||
provider_info = resolve_provider_model(job)
|
||||
estimate_token_budget(provider_info=provider_info)
|
||||
|
||||
# Стало:
|
||||
config = LLMProviderService(db).get_provider_token_config(job.provider_id)
|
||||
estimate_token_budget(
|
||||
provider_info=config["model"],
|
||||
context_window=config["context_window"], # приоритет над provider_info
|
||||
max_output_tokens=config["max_output_tokens"], # приоритет над provider_info
|
||||
)
|
||||
```
|
||||
|
||||
#### 5.2.6. PROVIDER_DEFAULTS — остаётся fallback
|
||||
|
||||
```python
|
||||
def estimate_token_budget(..., context_window=None, max_output_tokens=None, provider_info=None):
|
||||
# Если явно переданы — используем их
|
||||
# Если оба None — смотрим PROVIDER_DEFAULTS
|
||||
# Если и там нет — DEFAULT_...
|
||||
```
|
||||
|
||||
#### 5.2.7. Svelte UI
|
||||
|
||||
**Файл:** `frontend/src/lib/components/llm/ProviderConfig.svelte`
|
||||
|
||||
- Collapsible "Advanced: Token Limits"
|
||||
- Два number input: context_window, max_output_tokens
|
||||
- Placeholder: "Auto-detected. Override only if you know the provider's real limits."
|
||||
- Валидация на клиенте
|
||||
|
||||
#### 5.2.8. Alembic миграция
|
||||
|
||||
Новая миграция: add columns `context_window`, `max_output_tokens` to `llm_providers`.
|
||||
|
||||
### 5.3. [P0] Консервативный tokenizer estimate + единый safety factor
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_token_budget.py`
|
||||
|
||||
```python
|
||||
# Поправить коэффициенты (разумные значения, точные — после замера):
|
||||
CJK_RATIO = 1.0 # было 1.5
|
||||
OTHER_RATIO = 1.8 # было 2.2
|
||||
|
||||
# Единый safety factor (один, не размазанный):
|
||||
INPUT_SAFETY_FACTOR = 0.75 # 75% от расчётного бюджета
|
||||
OUTPUT_SAFETY_FACTOR = 0.70 # 70% от расчётного output-бюджета
|
||||
```
|
||||
|
||||
**Важно:** Эти цифры — стартовые. После сбора `usage.prompt_tokens` / `usage.completion_tokens` их надо откалибровать по реальным данным.
|
||||
|
||||
### 5.4. [P1] Retry only missing rows после partial response
|
||||
|
||||
**Текущий код:** `backend/src/plugins/translate/_llm_call.py:190-233` — binary split, теряет все уже переведённые строки.
|
||||
|
||||
**Улучшение:** При `finish_reason=length`:
|
||||
1. Попытаться распарсить ответ (`_recover_truncated_rows` в `_llm_parse.py:95-115`)
|
||||
2. Сохранить успешно переведённые строки
|
||||
3. Ретраить **только** те строки, которых не хватает
|
||||
|
||||
```python
|
||||
if finish_reason == "length":
|
||||
recovered = _recover_truncated_rows(llm_response, len(batch_rows), finish_reason)
|
||||
saved_rows = []
|
||||
missing_rows = []
|
||||
if recovered and recovered.get("rows"):
|
||||
# Распределить: какие строки удалось перевести, какие — нет
|
||||
parsed_ids = set(r.get("row_id") for r in recovered["rows"])
|
||||
for row in batch_rows:
|
||||
if str(row.get("row_index")) in parsed_ids:
|
||||
saved_rows.append(row)
|
||||
else:
|
||||
missing_rows.append(row)
|
||||
|
||||
if missing_rows and len(missing_rows) < len(batch_rows) * 0.95:
|
||||
# Есть существенный прогресс → ретраим только missing
|
||||
self._persist_partial(batch_rows, saved_rows, batch_id, run_id, ...)
|
||||
return self._retry_missing(job, run_id, missing_rows, dict_matches, ...)
|
||||
else:
|
||||
# Прогресса нет → binary split
|
||||
return self._split_and_retry(...)
|
||||
```
|
||||
|
||||
**Эффект:** Если из 100 строк вернулось 80 — ретраим только 20, а не 100.
|
||||
|
||||
### 5.5. [P1] Dynamic row cap (вместо фиксированного 50)
|
||||
|
||||
**Файл:** `backend/src/plugins/translate/_batch_sizer.py:148-166`
|
||||
|
||||
```python
|
||||
# Вычислить max_rows по output:
|
||||
output_per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW
|
||||
available_output = max_output_tokens - REASONING_OVERHEAD - MAX_OUTPUT_HEADROOM
|
||||
max_rows_by_output = max(available_output // output_per_row, 1) if output_per_row > 0 else 20
|
||||
|
||||
# Вычислить max_rows по input:
|
||||
max_rows_by_input = per_batch_budget // average_row_tokens
|
||||
|
||||
# Итоговый лимит:
|
||||
ABSOLUTE_HARD_CAP = 50 # safety net, не основное ограничение
|
||||
max_rows = min(max_rows_by_output, max_rows_by_input, ABSOLUTE_HARD_CAP)
|
||||
```
|
||||
|
||||
### 5.6. [P2] Self-calibration per run
|
||||
|
||||
После первого `finish_reason=length` в рамках одного run_id:
|
||||
- Посчитать реальное `actual_ratio = actual_tokens / estimated_tokens`
|
||||
- Склировать batch sizing для следующих батчей
|
||||
- Сбросить при новом run_id
|
||||
|
||||
---
|
||||
|
||||
## 6. Итоговые приоритеты
|
||||
|
||||
| # | Что | Файлы | Почему |
|
||||
|---|-----|-------|--------|
|
||||
| **P0** | Добавить usage-логи от провайдера | `_llm_http.py`, `_llm_call.py` | Без данных нельзя обосновать изменения |
|
||||
| **P0** | Output budget как первичный ограничитель | `_token_budget.py`, `_batch_sizer.py` | `finish_reason=length` — это чаще про выход, а не про вход |
|
||||
| **P0** | Консервативный tokenizer + safety factor | `_token_budget.py` | Быстро снижает truncation |
|
||||
| **P0** | Provider-level context_window / max_output_tokens | model + schema + service + routes + UI + migration | Нужно для неизвестных моделей |
|
||||
| **P1** | Retry only missing rows после truncation | `_llm_call.py`, `_llm_parse.py` | Сохраняет частичный результат |
|
||||
| **P1** | Dynamic row cap (output-aware) | `_batch_sizer.py` | Точнее, чем фиксированные 50 строк |
|
||||
| **P2** | Self-calibration per run/provider | `_batch_sizer.py`, `_llm_call.py` | Адаптация под модель |
|
||||
|
||||
---
|
||||
|
||||
## 7. Метрики успеха
|
||||
|
||||
После внедрения:
|
||||
|
||||
| Метрика | Цель | Как измерить |
|
||||
|---------|------|-------------|
|
||||
| `finish_reason=length` | < 1% LLM вызовов | Из логов |
|
||||
| Среднее число LLM вызовов на батч | ≤ 1.1 | total_calls / total_batches |
|
||||
| p95 длительность батча | < 90s | Из timing-логов |
|
||||
| Общее время на 5455 строк | ≤ 8 min | Из run duration |
|
||||
| successful_rows / requested_rows | ≥ 99.5% | Из records |
|
||||
| Malformed JSON rate | < 0.5% | Из parse failures |
|
||||
|
||||
---
|
||||
|
||||
## 8. Перед внедрением — замерить
|
||||
|
||||
Собрать для 10-20 батчей (разный размер, разное количество языков):
|
||||
|
||||
| Поле | Как получить |
|
||||
|------|-------------|
|
||||
| characters | `len(prompt)` |
|
||||
| estimated_input_tokens | `_estimate_tokens_for_text()` |
|
||||
| actual_prompt_tokens | `response.usage.prompt_tokens` |
|
||||
| actual_completion_tokens | `response.usage.completion_tokens` |
|
||||
| finish_reason | Из ответа |
|
||||
| rows | `len(batch_rows)` |
|
||||
| languages | `len(target_languages)` |
|
||||
| response_rows_count | После парсинга |
|
||||
|
||||
На этих данных:
|
||||
1. Посчитать `actual_ratio = actual_prompt_tokens / estimated_tokens` — точный CJK-коэффициент
|
||||
2. Посчитать `output_per_row_actual = actual_completion_tokens / rows / languages` — точный output per row
|
||||
|
||||
Только после этого фиксировать константы в коде.
|
||||
|
||||
---
|
||||
|
||||
## 9. PROVIDER_DEFAULTS — схема fallback (для справки)
|
||||
|
||||
```
|
||||
Пользователь указал context_window в UI?
|
||||
→ да: используем (с safe cap)
|
||||
→ нет: PROVIDER_DEFAULTS.get(model_name)?
|
||||
→ да: используем
|
||||
→ нет: DEFAULT_CONTEXT_WINDOW / DEFAULT_MAX_OUTPUT_TOKENS
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Текущие константы _token_budget.py (для справки)
|
||||
|
||||
| Константа | Значение | Описание |
|
||||
|-----------|----------|----------|
|
||||
| `DEFAULT_CONTEXT_WINDOW` | 64000 | |
|
||||
| `DEFAULT_MAX_OUTPUT_TOKENS` | 16384 | |
|
||||
| `REASONING_OVERHEAD` | 2000 | |
|
||||
| `OUTPUT_PER_ROW_PER_LANG` | 120 | |
|
||||
| `JSON_OVERHEAD_PER_ROW` | 50 | |
|
||||
| `PROMPT_BASE_TOKENS` | 600 | |
|
||||
| `DICT_TOKENS_PER_ENTRY` | 20 | |
|
||||
| `DICT_TOKENS_MAX` | 5000 | |
|
||||
| `CHARS_PER_TOKEN_MIXED` | 2.2 | |
|
||||
| `MIN_MAX_TOKENS` | 4096 | |
|
||||
| `MAX_OUTPUT_HEADROOM` | 3000 | |
|
||||
@@ -40,6 +40,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: true,
|
||||
is_multimodal: false,
|
||||
max_images: null,
|
||||
context_window: null,
|
||||
max_output_tokens: null,
|
||||
});
|
||||
|
||||
let testStatus = $state({ type: "", message: "" });
|
||||
@@ -84,6 +86,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: true,
|
||||
is_multimodal: false,
|
||||
max_images: null,
|
||||
context_window: null,
|
||||
max_output_tokens: null,
|
||||
};
|
||||
editingProvider = null;
|
||||
testStatus = { type: "", message: "" };
|
||||
@@ -109,6 +113,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: Boolean(provider?.is_active),
|
||||
is_multimodal: Boolean(provider?.is_multimodal),
|
||||
max_images: provider?.max_images ?? null,
|
||||
context_window: provider?.context_window ?? null,
|
||||
max_output_tokens: provider?.max_output_tokens ?? null,
|
||||
};
|
||||
testStatus = { type: "", message: "" };
|
||||
availableModels = [];
|
||||
@@ -271,6 +277,18 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
delete submitData.api_key;
|
||||
}
|
||||
|
||||
// Normalize token limit fields: bind:value on <input type="number"> returns string or "";
|
||||
// Pydantic expects int | None, so convert empty/string to number or null.
|
||||
for (const field of ["context_window", "max_output_tokens"]) {
|
||||
const val = submitData[field];
|
||||
if (val === "" || val === null || val === undefined) {
|
||||
submitData[field] = null;
|
||||
} else if (typeof val === "string") {
|
||||
const num = Number(val);
|
||||
submitData[field] = Number.isNaN(num) ? null : num;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await requestApi(endpoint, method, submitData);
|
||||
showForm = false;
|
||||
@@ -570,6 +588,51 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- #region token_limits_advanced -->
|
||||
<!-- @BRIEF Collapsible "Advanced: Token Limits" section. context_window and max_output_tokens
|
||||
per-provider override for token budget estimation. NULL = use PROVIDER_DEFAULTS. -->
|
||||
<details class="border-t pt-3 mt-2">
|
||||
<summary class="text-sm font-medium text-text-muted cursor-pointer hover:text-text select-none outline-none">
|
||||
{$t.llm?.advanced_token_limits || "Advanced: Token Limits"}
|
||||
</summary>
|
||||
<div class="mt-2 space-y-2">
|
||||
<div>
|
||||
<label for="provider-context-window" class="block text-sm font-medium text-text">
|
||||
{$t.llm?.context_window_label || "Context Window (tokens)"}
|
||||
</label>
|
||||
<input
|
||||
id="provider-context-window"
|
||||
type="number"
|
||||
min="1000"
|
||||
max="256000"
|
||||
placeholder="{$t.llm?.auto_detect || "Auto"}"
|
||||
bind:value={formData.context_window}
|
||||
class="mt-1 block w-full border rounded-md p-2"
|
||||
/>
|
||||
<p class="mt-0.5 text-xs text-text-subtle">
|
||||
{$t.llm?.context_window_hint || "Leave blank for auto-detection from model name."}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<label for="provider-max-output-tokens" class="block text-sm font-medium text-text">
|
||||
{$t.llm?.max_output_label || "Max Output Tokens"}
|
||||
</label>
|
||||
<input
|
||||
id="provider-max-output-tokens"
|
||||
type="number"
|
||||
min="256"
|
||||
placeholder="{$t.llm?.auto_detect || "Auto"}"
|
||||
bind:value={formData.max_output_tokens}
|
||||
class="mt-1 block w-full border rounded-md p-2"
|
||||
/>
|
||||
<p class="mt-0.5 text-xs text-text-subtle">
|
||||
{$t.llm?.max_output_hint || "Must be less than context window. Leave blank for auto."}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
<!-- #endregion token_limits_advanced -->
|
||||
</div>
|
||||
|
||||
{#if testStatus.message}
|
||||
|
||||
Reference in New Issue
Block a user