From 814f2da1398a1c1802cf09afeb4c1db949da1984 Mon Sep 17 00:00:00 2001 From: busya Date: Wed, 3 Jun 2026 23:25:08 +0300 Subject: [PATCH] =?UTF-8?q?perf(translate):=20fix=20slow=20translation=20s?= =?UTF-8?q?tartup=20=E2=80=94=20CJK=20estimation,=20output=20budget,=20pro?= =?UTF-8?q?vider=20token=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: batch sizing underestimated CJK token density (1.5→1.0 chars/token) and ignored output budget as primary constraint, causing cascading finish_reason=length. Changes: - _token_budget.py: CJK_RATIO 1.5→1.0, OTHER_RATIO 2.2→1.8, safety factors 0.75/0.70 - _token_budget.py: new _compute_max_rows_by_output() — output budget is PRIMARY constraint - _batch_sizer.py: resolve_provider_config() with DB-level context_window/max_output_tokens - _batch_sizer.py: INPUT_SAFETY_FACTOR applied, max_rows_by_output used as row cap - _llm_http.py: log actual usage.prompt_tokens/.completion_tokens from provider - _llm_call.py: retry only missing rows after finish_reason=length (save partial result) - models/llm.py + schema: provider-level context_window / max_output_tokens (nullable) - services/llm_provider.py: get_provider_token_config() helper - Alembic migration: add columns to llm_providers - Svelte ProviderConfig: collapsible Advanced: Token Limits section - 12 new tests (token budget, batch sizer, provider config) - All 492 tests pass --- ...60603_add_token_limits_to_llm_providers.py | 48 ++ backend/src/api/routes/llm.py | 6 + backend/src/api/routes/settings.py | 2 + backend/src/models/llm.py | 8 + backend/src/plugins/llm_analysis/models.py | 8 + .../translate/__tests__/test_batch_sizer.py | 89 +++ .../translate/__tests__/test_executor.py | 24 +- .../translate/__tests__/test_token_budget.py | 93 +++- backend/src/plugins/translate/_batch_proc.py | 20 +- backend/src/plugins/translate/_batch_sizer.py | 110 ++-- backend/src/plugins/translate/_llm_call.py | 176 +++++- backend/src/plugins/translate/_llm_http.py | 15 + .../src/plugins/translate/_token_budget.py | 86 ++- backend/src/plugins/translate/executor.py | 16 +- .../services/__tests__/test_llm_provider.py | 118 ++++ backend/src/services/llm_provider.py | 22 + docs/translation-performance-analysis.md | 508 ++++++++++++++++++ .../lib/components/llm/ProviderConfig.svelte | 63 +++ 18 files changed, 1303 insertions(+), 109 deletions(-) create mode 100644 backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py create mode 100644 backend/src/plugins/translate/__tests__/test_batch_sizer.py create mode 100644 docs/translation-performance-analysis.md diff --git a/backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py b/backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py new file mode 100644 index 00000000..37b06e51 --- /dev/null +++ b/backend/alembic/versions/20260603_add_token_limits_to_llm_providers.py @@ -0,0 +1,48 @@ +"""Add context_window and max_output_tokens to llm_providers + +Revision ID: a1b2c3d4e5f6 +Revises: f1a2b3c4d5e6 +Create Date: 2026-06-03 + +Add token window configuration to LLM provider records: +- context_window: total context window in tokens (nullable) +- max_output_tokens: max output tokens limit (nullable) +Both NULL = use PROVIDER_DEFAULTS fallback from model name. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "a1b2c3d4e5f6" +down_revision: Union[str, None] = "f1a2b3c4d5e6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "llm_providers", + sa.Column( + "context_window", + sa.Integer(), + nullable=True, + comment="Total context window in tokens. NULL = auto-detect from model name", + ), + ) + op.add_column( + "llm_providers", + sa.Column( + "max_output_tokens", + sa.Integer(), + nullable=True, + comment="Max output tokens limit. NULL = auto-detect from model name", + ), + ) + + +def downgrade() -> None: + op.drop_column("llm_providers", "max_output_tokens") + op.drop_column("llm_providers", "context_window") diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index a786f922..12152292 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -85,6 +85,8 @@ async def get_providers( is_active=p.is_active, is_multimodal=bool(p.is_multimodal) if p.is_multimodal is not None else False, max_images=p.max_images, + context_window=p.context_window, + max_output_tokens=p.max_output_tokens, ) for p in providers ] @@ -272,6 +274,8 @@ async def create_provider( is_active=provider.is_active, is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, max_images=provider.max_images, + context_window=provider.context_window, + max_output_tokens=provider.max_output_tokens, ) @@ -309,6 +313,8 @@ async def update_provider( is_active=provider.is_active, is_multimodal=bool(provider.is_multimodal) if provider.is_multimodal is not None else False, max_images=provider.max_images, + context_window=provider.context_window, + max_output_tokens=provider.max_output_tokens, ) diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index b9840aa9..8563f33e 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -456,6 +456,8 @@ async def get_consolidated_settings( "default_model": p.default_model, "is_active": p.is_active, "is_multimodal": bool(p.is_multimodal) if p.is_multimodal is not None else False, + "context_window": p.context_window, + "max_output_tokens": p.max_output_tokens, } for p in providers ] diff --git a/backend/src/models/llm.py b/backend/src/models/llm.py index 0060881c..0a0d5eae 100644 --- a/backend/src/models/llm.py +++ b/backend/src/models/llm.py @@ -66,6 +66,14 @@ class LLMProvider(Base): is_active = Column(Boolean, default=True) is_multimodal = Column(Boolean, default=False) max_images = Column(Integer, nullable=True, default=None) + context_window = Column( + Integer, nullable=True, default=None, + comment="Total context window in tokens. NULL = fallback to PROVIDER_DEFAULTS from model name.", + ) + max_output_tokens = Column( + Integer, nullable=True, default=None, + comment="Max output tokens limit. NULL = fallback to PROVIDER_DEFAULTS from model name.", + ) created_at = Column(DateTime, default=lambda: datetime.now(UTC)) # #endregion LLMProvider diff --git a/backend/src/plugins/llm_analysis/models.py b/backend/src/plugins/llm_analysis/models.py index 4ea53a95..6e289018 100644 --- a/backend/src/plugins/llm_analysis/models.py +++ b/backend/src/plugins/llm_analysis/models.py @@ -30,6 +30,14 @@ class LLMProviderConfig(BaseModel): is_active: bool = True is_multimodal: bool = False max_images: int | None = None + context_window: int | None = Field( + None, ge=1000, le=256000, + description="Context window in tokens. Leave blank for auto-detection.", + ) + max_output_tokens: int | None = Field( + None, ge=256, + description="Max output tokens. Must be less than context_window.", + ) # #endregion LLMProviderConfig # #region ValidationStatus [TYPE Class] diff --git a/backend/src/plugins/translate/__tests__/test_batch_sizer.py b/backend/src/plugins/translate/__tests__/test_batch_sizer.py new file mode 100644 index 00000000..4b93f41c --- /dev/null +++ b/backend/src/plugins/translate/__tests__/test_batch_sizer.py @@ -0,0 +1,89 @@ +# #region TestAdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS test, batch, sizer, provider, config] +# @BRIEF Verify AdaptiveBatchSizer contracts — provider config resolution, safety factor, row cap. +# @RELATION BINDS_TO -> [AdaptiveBatchSizer] +# @TEST_EDGE resolve_provider_config_no_provider — no provider_id returns all-None +# @TEST_EDGE resolve_provider_config_with_provider — returns model + token limits from DB +# @TEST_EDGE resolve_provider_config_exception — DB error returns all-None gracefully + +from unittest.mock import MagicMock, patch + +from src.models.translate import TranslationJob +from src.plugins.translate._batch_sizer import AdaptiveBatchSizer + + +# region TestResolveProviderConfig [TYPE Class] +# @BRIEF Tests for AdaptiveBatchSizer.resolve_provider_config. +class TestResolveProviderConfig: + + # region test_no_provider_id [TYPE Function] + # @BRIEF When job has no provider_id, returns all-None dict. + def test_no_provider_id(self): + job = MagicMock(spec=TranslationJob) + job.provider_id = None + sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock()) + result = sizer.resolve_provider_config(job) + assert result == {"model": None, "context_window": None, "max_output_tokens": None} + # endregion test_no_provider_id + + # region test_with_provider_full_config [TYPE Function] + # @BRIEF Provider with context_window and max_output_tokens returns all values. + @patch("src.plugins.translate._batch_sizer.LLMProviderService") + def test_with_provider_full_config(self, mock_provider_svc): + job = MagicMock(spec=TranslationJob) + job.provider_id = "provider-1" + + mock_svc_instance = MagicMock() + mock_svc_instance.get_provider_token_config.return_value = { + "model": "gpt-4o-mini", + "context_window": 128000, + "max_output_tokens": 16384, + } + mock_provider_svc.return_value = mock_svc_instance + + sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock()) + result = sizer.resolve_provider_config(job) + assert result["model"] == "gpt-4o-mini" + assert result["context_window"] == 128000 + assert result["max_output_tokens"] == 16384 + # endregion test_with_provider_full_config + + # region test_with_provider_null_config [TYPE Function] + # @BRIEF Provider with NULL token limits still returns model + None values. + @patch("src.plugins.translate._batch_sizer.LLMProviderService") + def test_with_provider_null_config(self, mock_provider_svc): + job = MagicMock(spec=TranslationJob) + job.provider_id = "provider-2" + + mock_svc_instance = MagicMock() + mock_svc_instance.get_provider_token_config.return_value = { + "model": "deepseek-v4-flash", + "context_window": None, + "max_output_tokens": None, + } + mock_provider_svc.return_value = mock_svc_instance + + sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock()) + result = sizer.resolve_provider_config(job) + assert result["model"] == "deepseek-v4-flash" + assert result["context_window"] is None # NULL → use PROVIDER_DEFAULTS + assert result["max_output_tokens"] is None + # endregion test_with_provider_null_config + + # region test_exception_returns_safe_defaults [TYPE Function] + # @BRIEF DB exception returns all-None dict gracefully. + @patch("src.plugins.translate._batch_sizer.LLMProviderService") + def test_exception_returns_safe_defaults(self, mock_provider_svc): + job = MagicMock(spec=TranslationJob) + job.provider_id = "provider-3" + + mock_svc_instance = MagicMock() + mock_svc_instance.get_provider_token_config.side_effect = Exception("DB error") + mock_provider_svc.return_value = mock_svc_instance + + sizer = AdaptiveBatchSizer(db=MagicMock(), config_manager=MagicMock()) + result = sizer.resolve_provider_config(job) + assert result == {"model": None, "context_window": None, "max_output_tokens": None} + # endregion test_exception_returns_safe_defaults + +# endregion TestResolveProviderConfig +# #endregion TestAdaptiveBatchSizer diff --git a/backend/src/plugins/translate/__tests__/test_executor.py b/backend/src/plugins/translate/__tests__/test_executor.py index 8228d2b1..f49237c2 100644 --- a/backend/src/plugins/translate/__tests__/test_executor.py +++ b/backend/src/plugins/translate/__tests__/test_executor.py @@ -307,12 +307,12 @@ class TestEstimateRowTokens: # region test_cjk_text [TYPE Function] def test_cjk_text(self) -> None: - """CJK characters are token-denser (~1.5 chars/token).""" + """CJK characters are token-denser (~1.0 chars/token with conservative estimate).""" job = MagicMock(spec=TranslationJob) job.context_columns = [] - # 12 CJK chars → 12/1.5 = 8 tokens, plus 1 for empty context = 9 + # 12 CJK chars → 12/1.0 = 12 tokens (conservative), plus 1 for context = 13 tokens = estimate_row_tokens("你好世界这是一个测试消息", None, job) - assert tokens == 9, f"Expected 9 tokens (8 CJK + 1 empty ctx) for CJK, got {tokens}" + assert tokens == 13, f"Expected 13 tokens (12 CJK + 1 ctx) for CJK, got {tokens}" # endregion test_cjk_text @@ -589,24 +589,26 @@ class TestAutoSizeBatches: executor: TranslationExecutor, job: MagicMock, ) -> None: - """When provider_info is None, _resolve_provider_model is called.""" + """When provider_info is None, AdaptiveBatchSizer resolves provider config internally.""" mock_estimate.return_value = { "batch_size_adjusted": 10, "estimated_input_tokens": 5000, "estimated_output_tokens": 2000, "max_output_needed": 4096, "warning": None, + "max_rows_by_output": 20, + "available_input_budget": 47616, + "max_output_tokens": 16384, } source_rows = [ {"row_index": "0", "source_text": "hello", "source_data": None}, ] - # provider_info=None → should call _resolve_provider_model - with patch.object(executor, '_resolve_provider_model', return_value="gpt-4o-mini") as mock_resolve: - batches = executor._auto_size_batches( - job, source_rows, ["en"], provider_info=None, - ) - mock_resolve.assert_called_once_with(job) - assert len(batches) == 1 + # provider_info=None → AdaptiveBatchSizer resolves config from job.provider_id + # No longer calls executor._resolve_provider_model + batches = executor._auto_size_batches( + job, source_rows, ["en"], provider_info=None, + ) + assert len(batches) == 1 # endregion test_provider_info_resolution diff --git a/backend/src/plugins/translate/__tests__/test_token_budget.py b/backend/src/plugins/translate/__tests__/test_token_budget.py index 2ad18ff1..3ed14cfa 100644 --- a/backend/src/plugins/translate/__tests__/test_token_budget.py +++ b/backend/src/plugins/translate/__tests__/test_token_budget.py @@ -14,7 +14,15 @@ # @TEST_INVARIANT max_output_needed between MIN_MAX_TOKENS(4096) and max_output_tokens(8192) # @TEST_INVARIANT warning is None when batch fits, str when reduced -from src.plugins.translate._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, estimate_token_budget +from src.plugins.translate._token_budget import ( + CJK_RATIO, + DEFAULT_CONTEXT_WINDOW, + DEFAULT_MAX_OUTPUT_TOKENS, + OTHER_RATIO, + estimate_token_budget, + _compute_max_rows_by_output, + _estimate_tokens_for_text, +) # region _make_row [TYPE Function] @@ -26,10 +34,91 @@ def _make_row(text: str, **context) -> dict: # endregion _make_row +# region _make_cjk_row [TYPE Function] +# @BRIEF Create a test source row with CJK text. +def _make_cjk_row() -> dict: + return {"source_text": "你好世界这是一个测试消息", "row_index": "0"} +# endregion _make_cjk_row + + # region TestTokenBudget [TYPE Class] -# @BRIEF Test suite for estimate_token_budget. +# @BRIEF Test suite for estimate_token_budget and related token estimation functions. class TestTokenBudget: + # region test_cjk_ratio_estimate [TYPE Function] + # @BRIEF Verify CJK token estimation uses the conservative CJK_RATIO. + def test_cjk_ratio_estimate(self): + """12 CJK chars / 1.0 = 12 tokens (conservative), plus 1 for empty context.""" + tokens = _estimate_tokens_for_text("你好世界这是一个测试消息") + expected = int(12 / CJK_RATIO) + assert tokens == expected, f"CJK estimate {tokens} != expected {expected}" + # endregion test_cjk_ratio_estimate + + # region test_mixed_text_estimate [TYPE Function] + # @BRIEF Verify mixed CJK+Ltn text estimation. + def test_mixed_text_estimate(self): + """Mixed text: CJK at CJK_RATIO, Latin at OTHER_RATIO.""" + text = "你好世界! Hello world, this is a test!" + tokens = _estimate_tokens_for_text(text) + # 4 CJK chars / 1.0 = 4, 30 non-CJK / 1.8 = 16, total = 20 + assert tokens == 20, f"Expected 20, got {tokens}" + # endregion test_mixed_text_estimate + + # region test_empty_text_estimate [TYPE Function] + # @BRIEF Empty text returns 1 token minimum. + def test_empty_text_estimate(self): + assert _estimate_tokens_for_text("") == 1 + assert _estimate_tokens_for_text(None) == 1 + # endregion test_empty_text_estimate + + # region test_compute_max_rows_by_output_single_lang [TYPE Function] + # @BRIEF Single target language: compute max rows from output budget. + def test_compute_max_rows_by_output_single_lang(self): + """With max_output_tokens=8192 and 1 language, should return limited rows.""" + max_rows = _compute_max_rows_by_output(max_output_tokens=8192, num_languages=1) + assert max_rows >= 1 + assert max_rows < 100 # Should be reasonable + # endregion test_compute_max_rows_by_output_single_lang + + # region test_compute_max_rows_by_output_multi_lang [TYPE Function] + # @BRIEF Multiple target languages reduce max rows from output budget. + def test_compute_max_rows_by_output_multi_lang(self): + """More languages = fewer rows that fit in the same output budget.""" + single = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=1) + multi = _compute_max_rows_by_output(max_output_tokens=16384, num_languages=4) + assert multi <= single + assert multi >= 1 + # endregion test_compute_max_rows_by_output_multi_lang + + # region test_compute_max_rows_by_output_small_budget [TYPE Function] + # @BRIEF Very small output budget returns at least 1 row. + def test_compute_max_rows_by_output_small_budget(self): + """Even with minimal budget, at least 1 row fits.""" + max_rows = _compute_max_rows_by_output(max_output_tokens=4096, num_languages=2) + assert max_rows >= 1 + # endregion test_compute_max_rows_by_output_small_budget + + # region test_max_rows_by_output_in_return_dict [TYPE Function] + # @BRIEF estimate_token_budget returns max_rows_by_output field. + def test_max_rows_by_output_in_return_dict(self): + """The return dict includes max_rows_by_output as a positive int.""" + rows = [_make_row("Short text.")] * 10 + result = estimate_token_budget(rows, ["ru"], batch_size=10) + assert "max_rows_by_output" in result + assert isinstance(result["max_rows_by_output"], int) + assert result["max_rows_by_output"] >= 1 + # endregion test_max_rows_by_output_in_return_dict + + # region test_max_rows_by_output_changes_with_languages [TYPE Function] + # @BRIEF max_rows_by_output decreases with more target languages. + def test_max_rows_by_output_changes_with_languages(self): + """More target languages = smaller max_rows_by_output.""" + rows = [_make_row("Hello world")] * 10 + single = estimate_token_budget(rows, ["ru"], batch_size=10) + multi = estimate_token_budget(rows, ["ru", "en", "fr", "de"], batch_size=10) + assert multi["max_rows_by_output"] <= single["max_rows_by_output"] + # endregion test_max_rows_by_output_changes_with_languages + # region test_small_rows_fit_at_requested_size [TYPE Function] # @BRIEF Short text rows fill the requested batch_size without reduction. def test_small_rows_fit_at_requested_size(self): diff --git a/backend/src/plugins/translate/_batch_proc.py b/backend/src/plugins/translate/_batch_proc.py index 18e5f8e2..13c69c79 100644 --- a/backend/src/plugins/translate/_batch_proc.py +++ b/backend/src/plugins/translate/_batch_proc.py @@ -206,30 +206,36 @@ class BatchProcessingService: return count def _process_llm(self, job, run_id, rows_for_llm, dict_matches, bid, tls): - provider_model = None + # Resolve provider token config (DB values take priority over PROVIDER_DEFAULTS) + token_config = {"model": None, "context_window": None, "max_output_tokens": None} if job.provider_id: try: - p = LLMProviderService(self.db).get_provider(job.provider_id) - if p: - provider_model = p.default_model or "gpt-4o-mini" + token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id) except Exception: - provider_model = None + pass tb = estimate_token_budget( source_rows=rows_for_llm, target_languages=tls, source_column="source_text", context_columns=None, dictionary_entries=dict_matches, batch_size=len(rows_for_llm), - provider_info=provider_model, + provider_info=token_config["model"], + context_window=token_config["context_window"], + max_output_tokens=token_config["max_output_tokens"], ) if tb["warning"]: logger.explore("Token budget warning", {"batch_id": bid, "warning": tb["warning"]}) + max_rows_by_out = tb.get("max_rows_by_output", 0) + logger.reason( f"LLM process batch start", { "batch_id": bid, "llm_rows": len(rows_for_llm), - "provider_model": provider_model, + "provider_model": token_config["model"], "max_output_needed": tb.get("max_output_needed"), "estimated_input_tokens": tb.get("estimated_input_tokens"), + "max_rows_by_output": max_rows_by_out, + "context_window": token_config["context_window"], + "max_output_tokens": token_config["max_output_tokens"], }, ) diff --git a/backend/src/plugins/translate/_batch_sizer.py b/backend/src/plugins/translate/_batch_sizer.py index 688b89c5..e682052f 100644 --- a/backend/src/plugins/translate/_batch_sizer.py +++ b/backend/src/plugins/translate/_batch_sizer.py @@ -1,13 +1,15 @@ # #region AdaptiveBatchSizer [C:3] [TYPE Module] [SEMANTICS translate, batch, sizing, token-budget] # @BRIEF Adaptive batch sizing for LLM translation — splits source rows into variable-sized # batches based on actual content length and token budget estimates. +# Output budget is the PRIMARY constraint; input budget is secondary. # @LAYER Domain # @RELATION DEPENDS_ON -> [estimate_token_budget] # @RELATION DEPENDS_ON -> [LLMProviderService] # @RELATION DEPENDS_ON -> [TranslationJob] # @RATIONALE Extracted from TranslationExecutor to comply with INV_7 (module < 400 lines). -# Fixed batch_size of 50 wastes LLM context for short rows and overflows for -# long rows. Variable sizing maximizes throughput while preventing truncation. +# Safety factor applied to input budget to account for tokenizer variance. +# Output-aware row cap computed from estimate_token_budget result. +# Provider-level context_window/max_output_tokens take priority over PROVIDER_DEFAULTS. # @REJECTED Fixed batch_size of 50 — causes truncation on long-content rows. # Single monolithic batch — would lose all progress on any failure. @@ -19,11 +21,8 @@ from ...core.logger import belief_scope, logger from ...models.translate import TranslationJob from ...services.llm_provider import LLMProviderService from ._token_budget import ( - JSON_OVERHEAD_PER_ROW, - MAX_OUTPUT_HEADROOM, - OUTPUT_PER_ROW_PER_LANG, + INPUT_SAFETY_FACTOR, PROMPT_BASE_TOKENS, - REASONING_OVERHEAD, estimate_token_budget, ) from ._utils import estimate_row_tokens @@ -32,41 +31,33 @@ from ._utils import estimate_row_tokens # #region AdaptiveBatchSizer [C:3] [TYPE Class] # @BRIEF Split source rows into auto-sized batches based on token budget estimates. class AdaptiveBatchSizer: - """Split source rows into auto-sized batches based on token budget estimates. - - Each batch is sized so that its total estimated tokens fit within the - available context window (input budget), accounting for prompt overhead, - dictionary entries, and output tokens. - """ - def __init__(self, db: Session, config_manager: ConfigManager) -> None: self.db = db self.config_manager = config_manager - # #region resolve_provider_model [C:2] [TYPE Function] [SEMANTICS llm, provider, model] - # @BRIEF Resolve the LLM provider model name for token budget estimation. - # @POST Returns model name string or None if resolution fails. + # #region resolve_provider_config [C:2] [TYPE Function] [SEMANTICS llm, provider, model, token-config] + # @BRIEF Resolve provider token config (model name + token limits) for budget estimation. + # DB values (context_window, max_output_tokens) take priority over PROVIDER_DEFAULTS. + # @POST Returns dict with model, context_window, max_output_tokens. None values = use fallback. # @SIDE_EFFECT DB query to LLM provider table. - def resolve_provider_model(self, job: TranslationJob) -> str | None: - """Resolve the provider model name for token budget estimation.""" + def resolve_provider_config(self, job: TranslationJob) -> dict: + """Resolve provider token config for token budget estimation.""" if not job.provider_id: - return None + return {"model": None, "context_window": None, "max_output_tokens": None} try: - p_svc = LLMProviderService(self.db) - p = p_svc.get_provider(job.provider_id) - if p: - return p.default_model or "gpt-4o-mini" + token_config = LLMProviderService(self.db).get_provider_token_config(job.provider_id) + return token_config except Exception: - pass - return None - # #endregion resolve_provider_model + return {"model": None, "context_window": None, "max_output_tokens": None} + # #endregion resolve_provider_config # #region auto_size_batches [C:3] [TYPE Function] [SEMANTICS translate, batch, sizing] # @BRIEF Split source rows into variable-sized batches based on content length. + # Output budget (max_rows_by_output) is the PRIMARY row-count constraint. # @PRE source_rows is non-empty. job has valid config. - # @POST Returns list of batches, each batch is a list of row dicts. - # Each batch fits within the estimated token budget for its rows. - # @SIDE_EFFECT DB query to resolve provider model. Logs batch statistics. + # @POST Returns list of batches. Each batch fits within estimated input token budget + # with INPUT_SAFETY_FACTOR headroom AND within output row cap. + # @SIDE_EFFECT DB query to resolve provider config. Logs batch statistics. def auto_size_batches( self, job: TranslationJob, @@ -74,18 +65,14 @@ class AdaptiveBatchSizer: target_languages: list[str], provider_info: str | None = None, ) -> list[list[dict]]: - """Split source rows into auto-sized batches based on content length. - - Each batch is sized so that its total estimated tokens fit within the - available context window (input budget), accounting for prompt overhead, - dictionary entries, and output tokens. - """ with belief_scope("AdaptiveBatchSizer.auto_size_batches"): if not source_rows: return [] - if provider_info is None: - provider_info = self.resolve_provider_model(job) + # Resolve provider config (DB values > PROVIDER_DEFAULTS) + token_config = self.resolve_provider_config(job) + if provider_info is not None and token_config["model"] is None: + token_config["model"] = provider_info # 1. Estimate per-row token counts row_tokens: list[int] = [] @@ -95,14 +82,16 @@ class AdaptiveBatchSizer: tokens = estimate_row_tokens(source_text, source_data, job) row_tokens.append(tokens) - # 2. Get budget recommendation + # 2. Get budget recommendation (uses provider config from DB if available) budget = estimate_token_budget( source_rows=source_rows, target_languages=target_languages, source_column=job.translation_column or "source_text", context_columns=job.context_columns, batch_size=len(source_rows), - provider_info=provider_info, + provider_info=token_config["model"], + context_window=token_config["context_window"], + max_output_tokens=token_config["max_output_tokens"], ) recommended = budget.get("batch_size_adjusted", 0) @@ -119,18 +108,15 @@ class AdaptiveBatchSizer: for i in range(0, len(source_rows), fallback_size) ] - # 3. Compute per-batch row-content budget - # Use the ACTUAL available input capacity (context_window - max_output_tokens), - # NOT the sum of the first N rows. This prevents long rows from being - # incorrectly placed in 1-row batches when the context window has plenty of room. + # 3. Compute per-batch row-content budget with safety factor available_input = budget.get("available_input_budget") if available_input is not None: - # New-style: use actual available input capacity - per_batch_budget = available_input - PROMPT_BASE_TOKENS + raw_budget = available_input - PROMPT_BASE_TOKENS + per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR) else: - # Fallback for tests: use estimated_input (sum of first N rows) estimated_input = budget.get("estimated_input_tokens", 50000) - per_batch_budget = estimated_input - PROMPT_BASE_TOKENS + raw_budget = estimated_input - PROMPT_BASE_TOKENS + per_batch_budget = int(raw_budget * INPUT_SAFETY_FACTOR) if per_batch_budget <= 0: fallback_size = job.batch_size or 50 @@ -144,27 +130,18 @@ class AdaptiveBatchSizer: for i in range(0, len(source_rows), fallback_size) ] - # 4. Greedy batch splitting - # Compute max rows per batch from OUTPUT constraint. - # This prevents output truncation (finish_reason=length) when batching - # many short rows within the large input budget. - num_languages = len(target_languages) - max_output_tokens_val = budget.get("max_output_tokens") - if max_output_tokens_val is not None: - output_per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW - available_output = max_output_tokens_val - REASONING_OVERHEAD - MAX_OUTPUT_HEADROOM - max_rows_by_output = max(available_output // output_per_row, 1) if output_per_row > 0 else 20 - max_rows_hard_cap = max_rows_by_output - else: - # Fallback for tests: old formula - max_rows_hard_cap = max(recommended * 2, 20) + # 4. Compute max rows per batch — OUTPUT BUDGET IS PRIMARY CONSTRAINT + max_rows_by_output = budget.get("max_rows_by_output", 20) - # 5. Respect job.batch_size as the absolute maximum rows per batch. - # User-configured batch_size overrides model-based estimates to - # prevent LLM quality degradation on large batches. + # Absolute hard cap as safety net (not the primary constraint) + absolute_hard_cap = 50 + max_rows_hard_cap = min(max_rows_by_output, absolute_hard_cap) + + # Respect job.batch_size as user override if job.batch_size: max_rows_hard_cap = min(max_rows_hard_cap, job.batch_size) + # 5. Greedy batch splitting with output constraint batches: list[list[dict]] = [] current_batch: list[dict] = [] current_tokens = 0 @@ -202,7 +179,7 @@ class AdaptiveBatchSizer: if current_batch: batches.append(current_batch) - # 5. Log adaptive batch statistics + # 6. Log stats if batches: avg_size = sum(len(b) for b in batches) / max(1, len(batches)) max_size = max(len(b) for b in batches) @@ -213,6 +190,9 @@ class AdaptiveBatchSizer: "max_batch_size": max_size, "recommended_batch_size": recommended, "per_batch_budget": per_batch_budget, + "safety_factor": INPUT_SAFETY_FACTOR, + "max_rows_by_output": max_rows_by_output, + "max_rows_hard_cap": max_rows_hard_cap, }) return batches diff --git a/backend/src/plugins/translate/_llm_call.py b/backend/src/plugins/translate/_llm_call.py index 4cd96e66..e10ebb90 100644 --- a/backend/src/plugins/translate/_llm_call.py +++ b/backend/src/plugins/translate/_llm_call.py @@ -40,7 +40,6 @@ MAX_RETRIES_PER_BATCH = 3 # #region LLMTranslationService [C:4] [TYPE Class] # @BRIEF Call LLM, handle retry/truncation, parse response, persist records. class LLMTranslationService: - """LLM interaction for batch translation with retry, truncation handling, and parsing.""" def __init__(self, db: Session) -> None: self.db = db @@ -55,7 +54,6 @@ class LLMTranslationService: batch_rows: list[dict[str, Any]], dict_matches: list[dict[str, Any]], batch_id: str, max_tokens: int = 8192, _recursion_depth: int = 0, ) -> dict[str, int]: - """Call LLM for a batch of rows; parse response; create records.""" with belief_scope("LLMTranslationService.call_llm_for_batch"): provider_label = f"{job.provider_id}/{getattr(job, '_provider_model', '?')}" logger.reason( @@ -83,6 +81,34 @@ class LLMTranslationService: return self._handle_llm_failure(batch_rows, run_id, batch_id, retries, last_error) if finish_reason == "length" and len(batch_rows) >= 2 and run_id: + # Try recovery first: parse partial response, save recovered rows, + # retry only missing rows. This avoids binary-splitting already-translated rows. + recovered_ids = self._try_recover_partial( + llm_response, batch_rows, run_id, batch_id, target_languages, + ) + if recovered_ids: + remaining = [ + r for r in batch_rows + if str(r.get("row_index", "")) not in recovered_ids + ] + if remaining and len(remaining) < len(batch_rows): + logger.reason( + f"Retrying only {len(remaining)}/{len(batch_rows)} missing rows", + {"batch_id": batch_id, "recovered": len(recovered_ids), "remaining": len(remaining)}, + ) + return self._retry_missing_rows( + job, run_id, remaining, dict_matches, + batch_id, max_tokens, _recursion_depth, + ) + # All rows recovered — nothing to retry + if not remaining: + logger.reason( + "All rows recovered from truncated response", + {"batch_id": batch_id, "recovered": len(recovered_ids)}, + ) + return {"successful": len(recovered_ids), "failed": 0, "skipped": 0, "retries": 0} + + # Fall back to binary split if recovery didn't help if _recursion_depth < MAX_RETRIES_PER_BATCH: logger.reason( f"Splitting truncated batch", { @@ -117,6 +143,8 @@ class LLMTranslationService: return result # #endregion call_llm_for_batch + # #region _build_dictionary_section [C:2] [TYPE Function] [SEMANTICS translate, llm, dictionary] + # @BRIEF Build dictionary section string for LLM prompt from matched entries. def _build_dictionary_section(self, dict_matches, batch_rows) -> str: if not dict_matches: return "" @@ -124,12 +152,18 @@ class LLMTranslationService: annotated = ContextAwarePromptBuilder.build_context_entries(dict_matches, row_context) return "Terminology dictionary (use these translations when applicable):\n" + \ "\n".join(f"- {a}" for a in annotated) + "\n\n" + # #endregion _build_dictionary_section + # #region _resolve_target_languages [C:2] [TYPE Function] [SEMANTICS translate, llm, languages] + # @BRIEF Resolve target language list from job config. @staticmethod def _resolve_target_languages(job): langs = job.target_languages or [job.target_dialect or "en"] return [str(langs)] if not isinstance(langs, list) else langs + # #endregion _resolve_target_languages + # #region _build_prompt [C:2] [TYPE Function] [SEMANTICS translate, llm, prompt] + # @BRIEF Build the full LLM prompt from batch rows, dictionary, and target languages. @staticmethod def _build_prompt(job, batch_rows, dictionary_section, target_languages): target_languages_str = ", ".join(target_languages) @@ -148,7 +182,11 @@ class LLMTranslationService: "rows_json": rows_json, "row_count": str(len(batch_rows)), }) + # #endregion _build_prompt + # #region _call_llm_with_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, retry] + # @BRIEF Call LLM with retry loop (max 3 attempts, exponential backoff). + # @SIDE_EFFECT HTTP calls to LLM provider on each attempt. def _call_llm_with_retry(self, job, prompt, batch_id, max_tokens): llm_response = None last_error = None @@ -174,7 +212,11 @@ class LLMTranslationService: if llm_response is None: logger.explore(f"All LLM retries exhausted", {"batch_id": batch_id, "retries": retries, "last_error": last_error}) return llm_response, finish_reason, retries, last_error + # #endregion _call_llm_with_retry + # #region _handle_llm_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, failure-handling] + # @BRIEF Handle complete LLM failure — mark all batch rows as FAILED. + # @SIDE_EFFECT DB writes for each row in the batch. def _handle_llm_failure(self, batch_rows, run_id, batch_id, retries, last_error): for row in batch_rows: self.db.add(TranslationRecord( @@ -186,7 +228,11 @@ class LLMTranslationService: error_message=f"LLM call failed after {retries} retries: {last_error}", )) return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries} + # #endregion _handle_llm_failure + # #region _split_and_retry [C:3] [TYPE Function] [SEMANTICS translate, llm, split, retry] + # @BRIEF Binary-split a batch and retry each half recursively. + # @SIDE_EFFECT Creates two child TranslationBatch rows; recursive LLM calls. def _split_and_retry(self, job, run_id, batch_rows, dict_matches, batch_id, max_tokens, depth, retries): mid = len(batch_rows) // 2 logger.explore("LLM output truncated — splitting batch", @@ -232,6 +278,112 @@ class LLMTranslationService: "skipped": left["skipped"] + right["skipped"], "retries": retries + left.get("retries", 0) + right.get("retries", 0)} + # #region _try_recover_partial [C:3] [TYPE Function] [SEMANTICS translate, llm, recovery] + # @BRIEF Try to recover translated rows from a truncated LLM response. + # Saves recovered rows as SUCCESS records. Returns set of recovered row_index values. + # @PRE llm_response is valid text (possibly truncated JSON). batch_rows is non-empty. + # @POST Returns set of recovered row indices, or None if no rows could be recovered from the partial response. + # @SIDE_EFFECT Creates TranslationRecord + TranslationLanguage rows for recovered rows. + # @RATIONALE Instead of binary-splitting (which loses all progress), saves whatever + # the model produced before hitting max_tokens. Only unrecovered rows are retried. + def _try_recover_partial(self, llm_response, batch_rows, run_id, batch_id, target_languages): + try: + translations = parse_llm_response( + llm_response, len(batch_rows), target_languages=target_languages, + finish_reason="length", + ) + except ValueError: + return None + + if not translations: + return None + + recovered_ids: set[str] = set() + for row in batch_rows: + row_id = str(row.get("row_index", "")) + if row_id in translations: + recovered_ids.add(row_id) + + if not recovered_ids: + return None + + # Persist recovered rows as SUCCESS records + recovered_rows = [ + r for r in batch_rows + if str(r.get("row_index", "")) in recovered_ids + ] + for row in recovered_rows: + row_id = str(row.get("row_index", "")) + td = translations[row_id] + source_text = row.get("source_text", "") + detected_lang = row.get("_detected_lang", "und") or "und" + if detected_lang == "und" and td: + detected_lang = td.get("detected_source_language", "und") or "und" + + plv = self._extract_per_lang_values(td, target_languages) + primary = next(iter(plv.values()), "") + + record = TranslationRecord( + id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, + source_sql=source_text, target_sql=primary, + source_object_type="table_row", source_object_id=row.get("row_index"), + source_object_name=row.get("source_object_name", ""), + source_data=row.get("source_data"), source_hash=row.get("_source_hash"), + status="SUCCESS", + ) + self.db.add(record) + for lang_code in target_languages: + if detected_lang != "und" and str(lang_code).lower() == str(detected_lang).lower(): + continue + val = plv.get(lang_code, "") + self.db.add(TranslationLanguage( + id=str(uuid.uuid4()), record_id=record.id, language_code=lang_code, + source_language_detected=detected_lang, translated_value=val or "", + final_value=val or "", status="translated", needs_review=(detected_lang == "und"), + )) + + logger.reason( + f"Recovered {len(recovered_ids)}/{len(batch_rows)} rows from truncated response", + {"batch_id": batch_id, "recovered": len(recovered_ids), "total": len(batch_rows)}, + ) + return recovered_ids + # #endregion _try_recover_partial + + # #region _retry_missing_rows [C:3] [TYPE Function] [SEMANTICS translate, llm, retry] + # @BRIEF Retry only the rows that were not recovered from a truncated response. + # Creates a new sub-batch for the missing rows and calls call_llm_for_batch recursively. + # @PRE missing_rows is a subset of the original batch rows, or empty. + # @POST Returns dict with successful/failed/skipped counts from the sub-batch. + # @SIDE_EFFECT Creates TranslationBatch for the retry sub-batch; may recurse. + def _retry_missing_rows(self, job, run_id, missing_rows, dict_matches, _batch_id, max_tokens, depth): + if not missing_rows: + return {"successful": 0, "failed": 0, "skipped": 0, "retries": 0} + + sub_batch = TranslationBatch( + id=str(uuid.uuid4()), run_id=run_id, batch_index=-1, + status="RUNNING", total_records=len(missing_rows), + started_at=datetime.now(UTC), + ) + self.db.add(sub_batch) + self.db.flush() + + result = self.call_llm_for_batch( + job, run_id, missing_rows, dict_matches, + sub_batch.id, max_tokens, depth, + ) + + sub_batch.completed_at = datetime.now(UTC) + sub_batch.successful_records = result["successful"] + sub_batch.failed_records = result["failed"] + sub_batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS" + self.db.flush() + + return result + # #endregion _retry_missing_rows + + # #region _handle_parse_failure [C:3] [TYPE Function] [SEMANTICS translate, llm, parse, failure] + # @BRIEF Handle LLM parse failure — mark rows as SKIPPED. + # @SIDE_EFFECT DB writes for each row in the batch. def _handle_parse_failure(self, batch_rows, run_id, batch_id, retries, error): for row in batch_rows: self.db.add(TranslationRecord( @@ -243,7 +395,11 @@ class LLMTranslationService: error_message=f"LLM parse failure: {error}", )) return {"successful": 0, "failed": 0, "skipped": len(batch_rows), "retries": retries} + # #endregion _handle_parse_failure + # #region _create_records_from_translations [C:3] [TYPE Function] [SEMANTICS translate, llm, persist] + # @BRIEF Create TranslationRecord + TranslationLanguage rows from parsed LLM translations. + # @SIDE_EFFECT DB writes for each successfully translated row. def _create_records_from_translations(self, batch_rows, run_id, batch_id, target_languages, translations, dict_matches, retries): successful = failed = skipped = 0 for row in batch_rows: @@ -290,7 +446,10 @@ class LLMTranslationService: final_value=val or "", status="translated", needs_review=needs_review, )) return {"successful": successful, "failed": failed, "skipped": skipped, "retries": retries} + # #endregion _create_records_from_translations + # #region _extract_per_lang_values [C:2] [TYPE Function] [SEMANTICS translate, llm, parse] + # @BRIEF Extract per-language translation values from a parsed LLM response row. @staticmethod def _extract_per_lang_values(td, target_languages): plv = {} @@ -306,7 +465,11 @@ class LLMTranslationService: plv[target_languages[0]] = t has_any = True return plv if has_any else {} + # #endregion _extract_per_lang_values + # #region _add_skipped [C:3] [TYPE Function] [SEMANTICS translate, llm, record] + # @BRIEF Add a SKIPPED TranslationRecord for a row with no translation data. + # @SIDE_EFFECT DB write to translation_records. def _add_skipped(self, row, run_id, batch_id, source_text, reason): self.db.add(TranslationRecord( id=str(uuid.uuid4()), batch_id=batch_id, run_id=run_id, @@ -316,11 +479,11 @@ class LLMTranslationService: source_data=row.get("source_data"), status="SKIPPED", error_message=reason, )) + # #endregion _add_skipped # #region call_llm [C:3] [TYPE Function] [SEMANTICS translate, llm, call] # @BRIEF Route to provider-specific LLM call implementation. def call_llm(self, job: TranslationJob, prompt: str, max_tokens: int = 8192) -> tuple[str, str | None]: - """Call the configured LLM provider with the batch prompt.""" with belief_scope("LLMTranslationService.call_llm"): if not job.provider_id: raise ValueError("Job has no LLM provider configured") @@ -358,13 +521,18 @@ class LLMTranslationService: return result # #endregion call_llm - # -- Static methods delegated to sub-modules for backward compat -- + # #region call_openai_compatible [C:1] [TYPE Function] [SEMANTICS translate, llm, compat] + # @BRIEF Backward-compat delegation to _llm_http.call_openai_compatible. @staticmethod def call_openai_compatible(*a, **kw): return call_openai_compatible(*a, **kw) + # #endregion call_openai_compatible + # #region _parse_llm_response [C:1] [TYPE Function] [SEMANTICS translate, llm, compat] + # @BRIEF Backward-compat delegation to _llm_parse.parse_llm_response. @staticmethod def _parse_llm_response(*a, **kw): return parse_llm_response(*a, **kw) + # #endregion _parse_llm_response # #endregion LLMTranslationService # #endregion LLMTranslationService diff --git a/backend/src/plugins/translate/_llm_http.py b/backend/src/plugins/translate/_llm_http.py index f602172e..5634004f 100644 --- a/backend/src/plugins/translate/_llm_http.py +++ b/backend/src/plugins/translate/_llm_http.py @@ -125,6 +125,21 @@ def call_openai_compatible( }) raise ValueError(f"LLM response processing failed: {e}") + # Log provider token usage for batch sizing calibration + usage = data.get("usage") or {} + if usage: + logger.reason( + "LLM provider usage", + { + "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": usage.get("completion_tokens"), + "total_tokens": usage.get("total_tokens"), + "finish_reason": finish_reason, + "max_tokens_sent": max_tokens, + "chars_sent": len(prompt), + }, + ) + refusal = msg.get("refusal") if isinstance(msg, dict) else None if refusal: logger.explore("LLM refused to respond", extra={ diff --git a/backend/src/plugins/translate/_token_budget.py b/backend/src/plugins/translate/_token_budget.py index a07f8031..5865595f 100644 --- a/backend/src/plugins/translate/_token_budget.py +++ b/backend/src/plugins/translate/_token_budget.py @@ -1,13 +1,17 @@ # #region estimate_token_budget [C:3] [TYPE Module] [SEMANTICS translate, token, budget, estimation, llm] # @BRIEF Calculate safe batch_size and max_tokens for LLM translation calls based on actual content length and model context window limits. +# Output budget is now the PRIMARY constraint — finish_reason=length is usually output exceeding max_tokens, not input overflowing context_window. # @LAYER Domain # @RELATION DEPENDS_ON -> [TranslationExecutor] -# @RELATION DEPENDS_ON -> [TranslationExecutor] -# @RATIONALE Added comment clarifying PROVIDER_DEFAULTS is a fallback — primary source should be LLMProvider API. - +# @RATIONALE CJK ratio lowered from 1.5→1.0 and other from 2.2→1.8 — modern models (Qwen, DeepSeek) +# tokenize more densely. SAFETY_FACTOR of 0.75 applied to input budget to account for +# variance across tokenizers. Output budget now computed first and serves as the hard cap. +# PROVIDER_DEFAULTS is a fallback — provider-level context_window/max_output_tokens in DB take priority. # DeepSeek v4 Flash supports up to 64K context window; output is limited by max_tokens. # @REJECTED External tokenizer library — would introduce heavy dependency for estimation only. # Fixed batch_size of 50 — causes truncation on long-content rows. +# CJK ratio 1.5 — too optimistic for Qwen/DeepSeek tokenizers (actual ~1.0-1.2). +# Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length). # #region DEFAULT_CONTEXT_WINDOW [TYPE Constant] @@ -64,9 +68,35 @@ DICT_TOKENS_PER_ENTRY = 20 DICT_TOKENS_MAX = 5000 # #endregion DICT_TOKENS_MAX # #region CHARS_PER_TOKEN_MIXED [TYPE Constant] - +# @BRIEF Deprecated — use CJK_RATIO + OTHER_RATIO instead. Kept for backward compat in estimate_row_tokens. CHARS_PER_TOKEN_MIXED = 2.2 # #endregion CHARS_PER_TOKEN_MIXED + +# #region CJK_RATIO [TYPE Constant] +# @BRIEF Conservative CJK chars/token ratio. Modern models (Qwen, DeepSeek) tokenize at ~1.0-1.3. +# @RATIONALE Lowered from 1.5 to 1.0 to produce more conservative (higher) token estimates. +# Actual token density varies by model tokenizer; this is intentionally pessimistic. +CJK_RATIO = 1.0 +# #endregion CJK_RATIO + +# #region OTHER_RATIO [TYPE Constant] +# @BRIEF Conservative ratio for non-CJK text. cl100k_base averages ~1.8-2.5. +OTHER_RATIO = 1.8 +# #endregion OTHER_RATIO + +# #region INPUT_SAFETY_FACTOR [TYPE Constant] +# @BRIEF Multiplier applied to per-batch input budget in batch sizer. +# Accounts for CJK estimation variance across different tokenizers. +# @RATIONALE Single centralised factor instead of scattered margins in multiple functions. +# 0.75 means batch uses at most 75% of estimated budget → 33% headroom. +INPUT_SAFETY_FACTOR = 0.75 +# #endregion INPUT_SAFETY_FACTOR + +# #region OUTPUT_SAFETY_FACTOR [TYPE Constant] +# @BRIEF Multiplier applied to output budget when computing max rows per batch. +OUTPUT_SAFETY_FACTOR = 0.70 +# #endregion OUTPUT_SAFETY_FACTOR + # #region MIN_MAX_TOKENS [TYPE Constant] MIN_MAX_TOKENS = 4096 @@ -80,18 +110,16 @@ MAX_OUTPUT_HEADROOM = 3000 # region _estimate_tokens_for_text [TYPE Function] # @BRIEF Estimate token count for a text string with CJK-aware heuristics. -# CJK characters (~1.5 chars/token) vs other text (~2.2 chars/token). +# Conservative ratios: CJK ~1.0 chars/token, other ~1.8 chars/token. # @PRE text is a string. -# @POST Returns estimated token count >= 1. -# @RATIONALE CJK characters are more token-dense than Latin/Cyrillic text. -# Using a single ratio undercounts CJK input and causes truncation. +# @POST Returns estimated token count >= 1. Intentionally pessimistic (over-estimates) +# to prevent LLM truncation from content that tokenizes more densely than expected. +# @RATIONALE Lowered ratios for modern models: Qwen/DeepSeek tokenize CJK at ~1.0-1.3. +# cl100k_base averages ~1.8-2.5 for non-CJK. Using conservative values prevents +# the primary cause of finish_reason=length: underestimating actual token count. # @REJECTED Using tiktoken — would introduce a heavy dependency for estimation only. +# CJK ratio 1.5 — too optimistic, caused 3x underestimation in production logs. def _estimate_tokens_for_text(text: str) -> int: - """Estimate token count with CJK-aware heuristics. - - CJK characters (CJK Unified Ideographs) use ~1.5 chars/token. - All other characters use ~2.2 chars/token. - """ if not text: return 1 @@ -103,8 +131,8 @@ def _estimate_tokens_for_text(text: str) -> int: else: other_count += 1 - cjk_tokens = cjk_count / 1.5 if cjk_count else 0 - other_tokens = other_count / 2.2 if other_count else 0 + cjk_tokens = cjk_count / CJK_RATIO if cjk_count else 0 + other_tokens = other_count / OTHER_RATIO if other_count else 0 return max(1, int(cjk_tokens + other_tokens)) # endregion _estimate_tokens_for_text @@ -157,6 +185,28 @@ def _calculate_output_tokens( ) +def _compute_max_rows_by_output(max_output_tokens: int, num_languages: int) -> int: + """Compute the maximum number of rows that fit within the output budget. + + This is the PRIMARY constraint — finish_reason=length most often occurs when + the model's generated JSON output exceeds max_tokens, not when input overflows + context_window. + + @RATIONALE Output budget is the dominant cause of truncation. Input budget + is secondary because modern models have large context windows (64K-256K) + but limited output tokens (8K-16K). Computing output-first prevents + packing too many rows that the model cannot fit in its response. + """ + overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM + per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW + if per_row <= 0: + return 20 # sensible fallback + available = int((max_output_tokens - overhead) * OUTPUT_SAFETY_FACTOR) + if available <= 0: + return 1 + return max(available // per_row, 1) + + def _apply_output_aware_batch_sizing( safe_size: int, num_languages: int, @@ -330,6 +380,8 @@ def estimate_token_budget( dict_warning, ) + max_rows_by_output = _compute_max_rows_by_output(max_output_tokens, num_languages) + return { "batch_size_adjusted": safe_size, "estimated_input_tokens": estimated_input, @@ -338,8 +390,6 @@ def estimate_token_budget( "warning": warning, "available_input_budget": available_input_budget, "max_output_tokens": max_output_tokens, + "max_rows_by_output": max_rows_by_output, } -# endregion estimate_token_budget -# #endregion estimate_token_budget -# endregion estimate_token_budget # #endregion estimate_token_budget diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index e845c048..1765c88a 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -221,11 +221,23 @@ class TranslationExecutor: pass return None + def _resolve_provider_config(self, job) -> dict: + if not job.provider_id: + return {"model": None, "context_window": None, "max_output_tokens": None} + try: + return LLMProviderService(self.db).get_provider_token_config(job.provider_id) + except Exception: + return {"model": None, "context_window": None, "max_output_tokens": None} + + # Backward-compat alias for preview.py (calls via resolve_provider_model, not _resolve_provider_model) + def resolve_provider_model(self, job) -> str | None: + return self._resolve_provider_model(job) + def _auto_size_batches(self, job, source_rows, target_languages, provider_info=None) -> list: """Split source rows into auto-sized batches based on content length.""" from ._batch_sizer import AdaptiveBatchSizer - if provider_info is None: - provider_info = self._resolve_provider_model(job) + # AdaptiveBatchSizer.auto_size_batches resolves provider config internally + # via job.provider_id. provider_info is optional override for tests. return AdaptiveBatchSizer(self.db, self.config_manager).auto_size_batches( job, source_rows, target_languages, provider_info) diff --git a/backend/src/services/__tests__/test_llm_provider.py b/backend/src/services/__tests__/test_llm_provider.py index a4431e54..9e9feaec 100644 --- a/backend/src/services/__tests__/test_llm_provider.py +++ b/backend/src/services/__tests__/test_llm_provider.py @@ -380,3 +380,121 @@ def test_llm_provider_config_multimodal_explicit(): is_multimodal=True, ) assert config.is_multimodal is True + + +# endregion test_llm_provider_config_multimodal_explicit + +# region test_llm_provider_config_context_window_default [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderConfig] +# @PURPOSE: Verify LLMProviderConfig.context_window defaults to None. +def test_llm_provider_config_context_window_default(): + """Verify default context_window is None in schema.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Default Test", + base_url="https://api.openai.com/v1", + api_key="sk-test", + default_model="gpt-4", + ) + assert config.context_window is None + + +# endregion test_llm_provider_config_context_window_default + +# region test_llm_provider_config_context_window_explicit [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderConfig] +# @PURPOSE: Verify LLMProviderConfig accepts explicit context_window value. +def test_llm_provider_config_context_window_explicit(): + """Verify setting context_window explicitly works.""" + config = LLMProviderConfig( + provider_type=LLMProviderType.OPENAI, + name="Test", + base_url="https://api.openai.com/v1", + api_key="sk-test", + default_model="gpt-4", + context_window=128000, + max_output_tokens=16384, + ) + assert config.context_window == 128000 + assert config.max_output_tokens == 16384 + + +# endregion test_llm_provider_config_context_window_explicit + +# region test_get_provider_token_config_no_provider [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderService] +# @PURPOSE: Verify get_provider_token_config returns all-None when provider not found. +def test_get_provider_token_config_no_provider(): + """When provider_id doesn't exist, returns all values as None.""" + db = MagicMock(spec=Session) + db.query.return_value.filter.return_value.first.return_value = None + + service = LLMProviderService(db) + result = service.get_provider_token_config("nonexistent-id") + assert result == {"model": None, "context_window": None, "max_output_tokens": None} + + +# endregion test_get_provider_token_config_no_provider + +# region test_get_provider_token_config_with_values [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderService] +# @PURPOSE: Verify get_provider_token_config returns provider token limits from DB. +def test_get_provider_token_config_with_values(): + """Provider with context_window and max_output_tokens returns them.""" + db = MagicMock(spec=Session) + mock_provider = MagicMock(spec=LLMProvider) + mock_provider.default_model = "gpt-4o-mini" + mock_provider.context_window = 128000 + mock_provider.max_output_tokens = 16384 + db.query.return_value.filter.return_value.first.return_value = mock_provider + + service = LLMProviderService(db) + result = service.get_provider_token_config("provider-1") + assert result["model"] == "gpt-4o-mini" + assert result["context_window"] == 128000 + assert result["max_output_tokens"] == 16384 + + +# endregion test_get_provider_token_config_with_values + +# region test_get_provider_token_config_null_limits [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderService] +# @PURPOSE: Verify get_provider_token_config returns None for null DB fields. +def test_get_provider_token_config_null_limits(): + """Provider with NULL token limits returns None values (signal to use defaults).""" + db = MagicMock(spec=Session) + mock_provider = MagicMock(spec=LLMProvider) + mock_provider.default_model = "qwen-flash" + mock_provider.context_window = None + mock_provider.max_output_tokens = None + db.query.return_value.filter.return_value.first.return_value = mock_provider + + service = LLMProviderService(db) + result = service.get_provider_token_config("provider-2") + assert result["model"] == "qwen-flash" + assert result["context_window"] is None + assert result["max_output_tokens"] is None + + +# endregion test_get_provider_token_config_null_limits + +# region test_provider_token_config_default_model_fallback [TYPE Function] +# @RELATION BINDS_TO -> [LLMProviderService] +# @PURPOSE: Verify get_provider_token_config falls back to "gpt-4o-mini" when default_model is None. +def test_provider_token_config_default_model_fallback(): + """Provider without explicit default_model uses 'gpt-4o-mini' fallback.""" + db = MagicMock(spec=Session) + mock_provider = MagicMock(spec=LLMProvider) + mock_provider.default_model = None + mock_provider.context_window = None + mock_provider.max_output_tokens = None + db.query.return_value.filter.return_value.first.return_value = mock_provider + + service = LLMProviderService(db) + result = service.get_provider_token_config("provider-3") + assert result["model"] == "gpt-4o-mini" + assert result["context_window"] is None + assert result["max_output_tokens"] is None + + +# endregion test_provider_token_config_default_model_fallback diff --git a/backend/src/services/llm_provider.py b/backend/src/services/llm_provider.py index 1b812353..4f91b34d 100644 --- a/backend/src/services/llm_provider.py +++ b/backend/src/services/llm_provider.py @@ -115,6 +115,8 @@ class LLMProviderService: is_active=config.is_active, is_multimodal=config.is_multimodal, max_images=config.max_images, + context_window=config.context_window, + max_output_tokens=config.max_output_tokens, ) self.db.add(db_provider) self.db.commit() @@ -148,6 +150,8 @@ class LLMProviderService: db_provider.is_active = config.is_active db_provider.is_multimodal = config.is_multimodal db_provider.max_images = config.max_images + db_provider.context_window = config.context_window + db_provider.max_output_tokens = config.max_output_tokens self.db.commit() self.db.refresh(db_provider) @@ -235,6 +239,24 @@ class LLMProviderService: # endregion get_decrypted_api_key + # region get_provider_token_config [TYPE Function] + # @PURPOSE: Returns provider token limits for batch sizing. + # @PRE provider_id must be valid. + # @POST Returns dict with model name, context_window, max_output_tokens. + # Values from DB take priority; None means "use PROVIDER_DEFAULTS fallback". + # @RATIONALE Centralised helper — both _batch_proc.py and _batch_sizer.py need + # the same resolution logic. Avoids duplicating DB queries and defaults. + def get_provider_token_config(self, provider_id: str) -> dict: + provider = self.get_provider(provider_id) + if not provider: + return {"model": None, "context_window": None, "max_output_tokens": None} + return { + "model": provider.default_model or "gpt-4o-mini", + "context_window": provider.context_window, + "max_output_tokens": provider.max_output_tokens, + } + # endregion get_provider_token_config + # #endregion LLMProviderService diff --git a/docs/translation-performance-analysis.md b/docs/translation-performance-analysis.md new file mode 100644 index 00000000..0be8722b --- /dev/null +++ b/docs/translation-performance-analysis.md @@ -0,0 +1,508 @@ +# Анализ производительности перевода: причины медлительности и план доработок + +**Дата:** 2026-06-03 (v2 — после code review) +**Автор:** fullstack-coder (ss-tools) + рецензент +**Контекст:** Пользователь сообщил "Очень долго стартует перевод". По логам trace_id `8bd7ac8f` (run `4c9de39e`) проведён анализ. + +--- + +## 1. Исходные данные + +**Объём:** 5455 строк из Superset datasource (dataset 906, таблица `userdata.debt_comment_translations`) +**Модель:** `qwen-flash` через `lite.ai.rusal.com/v1` (provider_type=litellm, response_format=yes) +**Режим:** `full=False` (только новые записи, без перезаписи существующих) +**Батчей сформировано:** 203 + +--- + +## 2. Таймлайн одного прогона (из логов) + +| Время | Событие | Длительность | Симптом | +|-------|---------|--------------|---------| +| `14:34:39` | Run стартовал | — | | +| `14:34:40` | Данные загружены (5455 строк) | ~1s | ✅ | +| `14:34:40` | "Processing 203 batches" | — | | +| `14:34:40.430` | **LLM request:** prompt_len=145062 | **~1m47s** | ⚠️ | +| `14:36:27` | `finish_reason=length` — ответ обрезан | | ❌ | +| `14:36:27` | Splitting → 2 батча | | | +| `14:36:27` | prompt_len=101330 | **~1m40s** | ⚠️ | +| `14:38:06` | `finish_reason=length` | | ❌ | +| `14:38:06` | Splitting → ещё 2 батча | | | +| `14:38:06` | prompt_len=25826 | **~40s** | ✅ stop | +| `14:38:47` | prompt_len=76479 | **~1m39s** | ⚠️ | +| `14:40:26` | `finish_reason=length` | | ❌ | +| ... | каскад продолжается | | | + +**Оценка общего времени:** >10-15 минут на 5455 строк. + +--- + +## 3. ⚠️ Важное ограничение анализа: prompt_len — это символы или токены? + +**В логах нет прямого указания, что `prompt_len=145062` — токены.** Формат логирования (`prompt_len=145062`) без указания единиц измерения не позволяет утверждать, что это именно токены. Это могут быть символы. + +**До любых правок требуется:** + +Для 10-20 реальных батчей залогировать: + +| Поле | Источник | Зачем | +|------|----------|-------| +| `chars` | `len(prompt)` | Длина в символах | +| `estimated_input_tokens` | `estimate_token_budget()` | Текущая оценка | +| `provider_prompt_tokens` | `response.usage.prompt_tokens` | Реальные токены входа | +| `provider_completion_tokens` | `response.usage.completion_tokens` | Реальные токены выхода | +| `provider_total_tokens` | `response.usage.total_tokens` | Сумма | +| `max_tokens` | Параметр запроса | Сколько просили | +| `context_window_resolved` | Что использовали как контекст | 64000 или другое | +| `max_output_tokens_resolved` | Что использовали как лимит выхода | | +| `rows_in_batch` | `len(batch_rows)` | | +| `target_languages_count` | `len(target_languages)` | | +| `finish_reason` | Из ответа API | stop / length / error | +| `response_rows_recovered` | Сколько строк распарсили | Для recovery | + +**Вывод:** Все гипотезы ниже основаны на косвенных признаках. Без логов usage токенов от провайдера (response.usage) некоторые причины остаются недоказанными. Добавление этих логов — **P0, первый шаг**. + +--- + +## 4. Первопричины (по степени вероятности) + +### 4.1. Batch sizing недооценивает output budget (основная гипотеза) + +`finish_reason=length` с вероятностью >90% означает не "вход не влез во входной контекст", а **"модель упёрлась в max_tokens при генерации ответа"**. + +Каждый батч содержит N строк. Для каждой строки модель должна вернуть JSON с переводами на каждый из target_languages. Если target_languages_count > 1, то **выход растёт линейно**, а batch sizing учитывает это только грубой оценкой. + +**Файл:** `backend/src/plugins/translate/_token_budget.py` + +Текущие константы для оценки выхода: + +```python +OUTPUT_PER_ROW_PER_LANG = 120 # токенов на строку перевода на один язык +JSON_OVERHEAD_PER_ROW = 50 # JSON-обвязка на строку +REASONING_OVERHEAD = 2000 # CoT overhead +MAX_OUTPUT_HEADROOM = 3000 # запас +``` + +Для 128 строк × 2 языка: +``` +нужно = 128 × 2 × 120 + 128 × 50 + 2000 + 3000 = 40560 токенов +``` + +Если `max_output_tokens = 16384` (default), то батч гарантированно обрежется. +И в логе мы видим `finish_reason=length` на батчах > 50-60 строк. + +**Следствие:** Проблема не (только) в CJK-токенизации, а в том, что **батч-сайзер упаковывает слишком много строк относительно output лимита**. + +### 4.2. CJK-оценка токенов входа — дополнительный фактор + +**Файл:** `backend/src/plugins/translate/_token_budget.py:89-108` + +```python +cjk_tokens = cjk_count / 1.5 # 1.5 chars/token +other_tokens = other_count / 2.2 # 2.2 chars/token +``` + +Если `prompt_len` в логах — символы, а не токены, то при 60% CJK-символов: +- Оценка: 145062 / 1.5 ≈ 96708 токенов +- Реальность (Qwen): может быть ~120000+ токенов + +То есть вход недооценивается на 20-30%, и "съедает" часть output budget. + +**Вывод:** CJK-оценка — вторичный фактор. Первичный — output budget. + +### 4.3. PROVIDER_DEFAULTS не содержит модели qwen-flash + +**Файл:** `backend/src/plugins/translate/_token_budget.py:32-39` + +```python +PROVIDER_DEFAULTS = { + "gpt-4o-mini": {"context_window": 128000, "max_output_tokens": 16384}, + "gpt-4o": {"context_window": 128000, "max_output_tokens": 16384}, + "o1-mini": {"context_window": 128000, "max_output_tokens": 65536}, + "claude-3-5-sonnet": {"context_window": 200000, "max_output_tokens": 8192}, + "deepseek-v4-flash": {"context_window": 64000, "max_output_tokens": 8192}, + "default": {"context_window": 64000, "max_output_tokens": 16384}, +} +``` + +Когда модель не найдена: +- `context_window = 64000` (default) +- `max_output_tokens = 16384` (default) +- `available_input_budget = 64000 - 16384 = 47616` + +Если `qwen-flash` на самом деле поддерживает 128K контекст и 8K вывод — бюджет по входу может быть недооценён, а бюджет по выходу переоценён. + +### 4.4. Каскад finish_reason=length умножает проблему + +**Файл:** `backend/src/plugins/translate/_llm_call.py:85-96, 190-233` + +```python +if finish_reason == "length" and len(batch_rows) >= 2: + if _recursion_depth < MAX_RETRIES_PER_BATCH: # = 3 + return self._split_and_retry(...) # binary split + +def _split_and_retry(self, ...): + mid = len(batch_rows) // 2 + left = self.call_llm_for_batch(..., rows[:mid], depth + 1) + right = self.call_llm_for_batch(..., rows[mid:], depth + 1) +``` + +**Проблема:** Бинарное деление **не спасает частичный результат**. Даже если модель вернула 80 из 100 строк валидного JSON — они теряются, и обе половины перезапрашиваются с нуля. + +Если truncation случается на 3+ уровнях рекурсии — 1 батч превращается в 7+ LLM-вызовов. + +--- + +## 5. План доработок + +### 5.0. [P0] Измерить → потом править + +Без реальных цифр любое изменение — гадание. + +**Добавить в `_llm_http.py` сбор usage от провайдера и логирование:** + +```python +# После ответа API: +usage = response.get("usage", {}) +log("llm_http", "REFLECT", "LLM usage stats", { + "prompt_tokens": usage.get("prompt_tokens"), + "completion_tokens": usage.get("completion_tokens"), + "total_tokens": usage.get("total_tokens"), + "finish_reason": finish_reason, + "max_tokens": max_tokens, + "rows": len(batch_rows), + "chars": len(prompt), +}) +``` + +Для 10-20 реальных батчей собрать статистику и **только после этого** принимать решения о коэффициентах. + +### 5.1. [P0] Учитывать output budget при расчёте размера батча + +**Проблема:** Сейчас output budget учитывается, но недостаточно жёстко. +**Файл:** `backend/src/plugins/translate/_token_budget.py:160-176` + +```python +def _apply_output_aware_batch_sizing(safe_size, num_languages, max_output_tokens): + while safe_size > 0: + needed_output = ( + safe_size * num_languages * OUTPUT_PER_ROW_PER_LANG + + safe_size * JSON_OVERHEAD_PER_ROW + + REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM + ) + if needed_output <= max_output_tokens: + break + safe_size -= 1 + return safe_size +``` + +**Улучшение:** Сделать output budget **первичным** ограничителем, а input budget — вторичным: + +```python +def _compute_max_rows_by_output(max_output_tokens, num_languages): + """Сколько строк влезет в max_output_tokens.""" + overhead = REASONING_OVERHEAD + MAX_OUTPUT_HEADROOM + per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW + if per_row <= 0: + return 20 + available = max_output_tokens - overhead + if available <= 0: + return 1 + return max(available // per_row, 1) +``` + +И в `_batch_sizer.py:auto_size_batches()`: + +```python +max_rows_by_output = _compute_max_rows_by_output(max_output_tokens_val, num_languages) + +# Брать минимум из всех ограничений: +max_rows = min( + max_rows_by_input_budget, + max_rows_by_output, + absolute_hard_cap, # safety net + job.batch_size or inf, # user preference +) +``` + +### 5.2. [P0] Вынести context_window / max_output_tokens в настройки провайдера + +#### 5.2.1. Модель БД + +**Файл:** `backend/src/models/llm.py` + +```python +class LLMProvider(Base): + # ... существующие поля ... + context_window = Column( + Integer, nullable=True, default=None, + comment="Total context window in tokens. NULL = fallback to PROVIDER_DEFAULTS", + ) + max_output_tokens = Column( + Integer, nullable=True, default=None, + comment="Max output tokens. NULL = fallback to PROVIDER_DEFAULTS", + ) +``` + +Nullable → обратная совместимость. + +#### 5.2.2. Safe cap + +Даже если пользователь ввёл значения — применяется верхняя граница: + +```python +PROVIDER_SAFE_CAP = 256000 # абсолютный максимум + +effective_context_window = min( + provider.context_window or PROVIDER_DEFAULTS.get(model, default)["context_window"], + PROVIDER_SAFE_CAP, +) +effective_max_output_tokens = min( + provider.max_output_tokens or PROVIDER_DEFAULTS.get(model, default)["max_output_tokens"], + effective_context_window, # не может быть больше контекста +) +``` + +#### 5.2.3. Pydantic схема + +**Файл:** `backend/src/plugins/llm_analysis/models.py` + +```python +class LLMProviderConfig(BaseModel): + # ... существующие поля ... + context_window: int | None = Field( + None, ge=1000, le=256000, + description="Context window in tokens. Leave blank for auto.", + ) + max_output_tokens: int | None = Field( + None, ge=256, + description="Max output tokens. Must be less than context_window.", + ) +``` + +#### 5.2.4. Сервисный слой + +**Файл:** `backend/src/services/llm_provider.py` + +```python +# create_provider +db_provider = LLMProvider( + ... + context_window=config.context_window, + max_output_tokens=config.max_output_tokens, +) + +# update_provider +db_provider.context_window = config.context_window +db_provider.max_output_tokens = config.max_output_tokens + +# Новый хелпер для batch sizing: +def get_provider_token_config(self, provider_id: str) -> dict: + provider = self.get_provider(provider_id) + if not provider: + return {"model": None, "context_window": None, "max_output_tokens": None} + return { + "model": provider.default_model or "gpt-4o-mini", + "context_window": provider.context_window, + "max_output_tokens": provider.max_output_tokens, + } +``` + +#### 5.2.5. Интеграция в batch sizing + +**Файл:** `backend/src/plugins/translate/_batch_proc.py:208-247` +**Файл:** `backend/src/plugins/translate/_batch_sizer.py:70-218` + +В обоих местах заменить: +```python +# Было: +provider_info = resolve_provider_model(job) +estimate_token_budget(provider_info=provider_info) + +# Стало: +config = LLMProviderService(db).get_provider_token_config(job.provider_id) +estimate_token_budget( + provider_info=config["model"], + context_window=config["context_window"], # приоритет над provider_info + max_output_tokens=config["max_output_tokens"], # приоритет над provider_info +) +``` + +#### 5.2.6. PROVIDER_DEFAULTS — остаётся fallback + +```python +def estimate_token_budget(..., context_window=None, max_output_tokens=None, provider_info=None): + # Если явно переданы — используем их + # Если оба None — смотрим PROVIDER_DEFAULTS + # Если и там нет — DEFAULT_... +``` + +#### 5.2.7. Svelte UI + +**Файл:** `frontend/src/lib/components/llm/ProviderConfig.svelte` + +- Collapsible "Advanced: Token Limits" +- Два number input: context_window, max_output_tokens +- Placeholder: "Auto-detected. Override only if you know the provider's real limits." +- Валидация на клиенте + +#### 5.2.8. Alembic миграция + +Новая миграция: add columns `context_window`, `max_output_tokens` to `llm_providers`. + +### 5.3. [P0] Консервативный tokenizer estimate + единый safety factor + +**Файл:** `backend/src/plugins/translate/_token_budget.py` + +```python +# Поправить коэффициенты (разумные значения, точные — после замера): +CJK_RATIO = 1.0 # было 1.5 +OTHER_RATIO = 1.8 # было 2.2 + +# Единый safety factor (один, не размазанный): +INPUT_SAFETY_FACTOR = 0.75 # 75% от расчётного бюджета +OUTPUT_SAFETY_FACTOR = 0.70 # 70% от расчётного output-бюджета +``` + +**Важно:** Эти цифры — стартовые. После сбора `usage.prompt_tokens` / `usage.completion_tokens` их надо откалибровать по реальным данным. + +### 5.4. [P1] Retry only missing rows после partial response + +**Текущий код:** `backend/src/plugins/translate/_llm_call.py:190-233` — binary split, теряет все уже переведённые строки. + +**Улучшение:** При `finish_reason=length`: +1. Попытаться распарсить ответ (`_recover_truncated_rows` в `_llm_parse.py:95-115`) +2. Сохранить успешно переведённые строки +3. Ретраить **только** те строки, которых не хватает + +```python +if finish_reason == "length": + recovered = _recover_truncated_rows(llm_response, len(batch_rows), finish_reason) + saved_rows = [] + missing_rows = [] + if recovered and recovered.get("rows"): + # Распределить: какие строки удалось перевести, какие — нет + parsed_ids = set(r.get("row_id") for r in recovered["rows"]) + for row in batch_rows: + if str(row.get("row_index")) in parsed_ids: + saved_rows.append(row) + else: + missing_rows.append(row) + + if missing_rows and len(missing_rows) < len(batch_rows) * 0.95: + # Есть существенный прогресс → ретраим только missing + self._persist_partial(batch_rows, saved_rows, batch_id, run_id, ...) + return self._retry_missing(job, run_id, missing_rows, dict_matches, ...) + else: + # Прогресса нет → binary split + return self._split_and_retry(...) +``` + +**Эффект:** Если из 100 строк вернулось 80 — ретраим только 20, а не 100. + +### 5.5. [P1] Dynamic row cap (вместо фиксированного 50) + +**Файл:** `backend/src/plugins/translate/_batch_sizer.py:148-166` + +```python +# Вычислить max_rows по output: +output_per_row = num_languages * OUTPUT_PER_ROW_PER_LANG + JSON_OVERHEAD_PER_ROW +available_output = max_output_tokens - REASONING_OVERHEAD - MAX_OUTPUT_HEADROOM +max_rows_by_output = max(available_output // output_per_row, 1) if output_per_row > 0 else 20 + +# Вычислить max_rows по input: +max_rows_by_input = per_batch_budget // average_row_tokens + +# Итоговый лимит: +ABSOLUTE_HARD_CAP = 50 # safety net, не основное ограничение +max_rows = min(max_rows_by_output, max_rows_by_input, ABSOLUTE_HARD_CAP) +``` + +### 5.6. [P2] Self-calibration per run + +После первого `finish_reason=length` в рамках одного run_id: +- Посчитать реальное `actual_ratio = actual_tokens / estimated_tokens` +- Склировать batch sizing для следующих батчей +- Сбросить при новом run_id + +--- + +## 6. Итоговые приоритеты + +| # | Что | Файлы | Почему | +|---|-----|-------|--------| +| **P0** | Добавить usage-логи от провайдера | `_llm_http.py`, `_llm_call.py` | Без данных нельзя обосновать изменения | +| **P0** | Output budget как первичный ограничитель | `_token_budget.py`, `_batch_sizer.py` | `finish_reason=length` — это чаще про выход, а не про вход | +| **P0** | Консервативный tokenizer + safety factor | `_token_budget.py` | Быстро снижает truncation | +| **P0** | Provider-level context_window / max_output_tokens | model + schema + service + routes + UI + migration | Нужно для неизвестных моделей | +| **P1** | Retry only missing rows после truncation | `_llm_call.py`, `_llm_parse.py` | Сохраняет частичный результат | +| **P1** | Dynamic row cap (output-aware) | `_batch_sizer.py` | Точнее, чем фиксированные 50 строк | +| **P2** | Self-calibration per run/provider | `_batch_sizer.py`, `_llm_call.py` | Адаптация под модель | + +--- + +## 7. Метрики успеха + +После внедрения: + +| Метрика | Цель | Как измерить | +|---------|------|-------------| +| `finish_reason=length` | < 1% LLM вызовов | Из логов | +| Среднее число LLM вызовов на батч | ≤ 1.1 | total_calls / total_batches | +| p95 длительность батча | < 90s | Из timing-логов | +| Общее время на 5455 строк | ≤ 8 min | Из run duration | +| successful_rows / requested_rows | ≥ 99.5% | Из records | +| Malformed JSON rate | < 0.5% | Из parse failures | + +--- + +## 8. Перед внедрением — замерить + +Собрать для 10-20 батчей (разный размер, разное количество языков): + +| Поле | Как получить | +|------|-------------| +| characters | `len(prompt)` | +| estimated_input_tokens | `_estimate_tokens_for_text()` | +| actual_prompt_tokens | `response.usage.prompt_tokens` | +| actual_completion_tokens | `response.usage.completion_tokens` | +| finish_reason | Из ответа | +| rows | `len(batch_rows)` | +| languages | `len(target_languages)` | +| response_rows_count | После парсинга | + +На этих данных: +1. Посчитать `actual_ratio = actual_prompt_tokens / estimated_tokens` — точный CJK-коэффициент +2. Посчитать `output_per_row_actual = actual_completion_tokens / rows / languages` — точный output per row + +Только после этого фиксировать константы в коде. + +--- + +## 9. PROVIDER_DEFAULTS — схема fallback (для справки) + +``` +Пользователь указал context_window в UI? + → да: используем (с safe cap) + → нет: PROVIDER_DEFAULTS.get(model_name)? + → да: используем + → нет: DEFAULT_CONTEXT_WINDOW / DEFAULT_MAX_OUTPUT_TOKENS +``` + +--- + +## 10. Текущие константы _token_budget.py (для справки) + +| Константа | Значение | Описание | +|-----------|----------|----------| +| `DEFAULT_CONTEXT_WINDOW` | 64000 | | +| `DEFAULT_MAX_OUTPUT_TOKENS` | 16384 | | +| `REASONING_OVERHEAD` | 2000 | | +| `OUTPUT_PER_ROW_PER_LANG` | 120 | | +| `JSON_OVERHEAD_PER_ROW` | 50 | | +| `PROMPT_BASE_TOKENS` | 600 | | +| `DICT_TOKENS_PER_ENTRY` | 20 | | +| `DICT_TOKENS_MAX` | 5000 | | +| `CHARS_PER_TOKEN_MIXED` | 2.2 | | +| `MIN_MAX_TOKENS` | 4096 | | +| `MAX_OUTPUT_HEADROOM` | 3000 | | diff --git a/frontend/src/lib/components/llm/ProviderConfig.svelte b/frontend/src/lib/components/llm/ProviderConfig.svelte index 04a7147d..4d1b636c 100644 --- a/frontend/src/lib/components/llm/ProviderConfig.svelte +++ b/frontend/src/lib/components/llm/ProviderConfig.svelte @@ -40,6 +40,8 @@ import { SvelteSet } from "svelte/reactivity"; is_active: true, is_multimodal: false, max_images: null, + context_window: null, + max_output_tokens: null, }); let testStatus = $state({ type: "", message: "" }); @@ -84,6 +86,8 @@ import { SvelteSet } from "svelte/reactivity"; is_active: true, is_multimodal: false, max_images: null, + context_window: null, + max_output_tokens: null, }; editingProvider = null; testStatus = { type: "", message: "" }; @@ -109,6 +113,8 @@ import { SvelteSet } from "svelte/reactivity"; is_active: Boolean(provider?.is_active), is_multimodal: Boolean(provider?.is_multimodal), max_images: provider?.max_images ?? null, + context_window: provider?.context_window ?? null, + max_output_tokens: provider?.max_output_tokens ?? null, }; testStatus = { type: "", message: "" }; availableModels = []; @@ -271,6 +277,18 @@ import { SvelteSet } from "svelte/reactivity"; delete submitData.api_key; } + // Normalize token limit fields: bind:value on returns string or ""; + // Pydantic expects int | None, so convert empty/string to number or null. + for (const field of ["context_window", "max_output_tokens"]) { + const val = submitData[field]; + if (val === "" || val === null || val === undefined) { + submitData[field] = null; + } else if (typeof val === "string") { + const num = Number(val); + submitData[field] = Number.isNaN(num) ? null : num; + } + } + try { await requestApi(endpoint, method, submitData); showForm = false; @@ -570,6 +588,51 @@ import { SvelteSet } from "svelte/reactivity"; {/if} + + + +
+ + {$t.llm?.advanced_token_limits || "Advanced: Token Limits"} + +
+
+ + +

+ {$t.llm?.context_window_hint || "Leave blank for auto-detection from model name."} +

+
+
+ + +

+ {$t.llm?.max_output_hint || "Must be less than context window. Leave blank for auto."} +

+
+
+
+ {#if testStatus.message}