perf(translate): fix slow translation startup — CJK estimation, output budget, provider token config
Root cause: batch sizing underestimated CJK token density (1.5→1.0 chars/token) and ignored output budget as primary constraint, causing cascading finish_reason=length. Changes: - _token_budget.py: CJK_RATIO 1.5→1.0, OTHER_RATIO 2.2→1.8, safety factors 0.75/0.70 - _token_budget.py: new _compute_max_rows_by_output() — output budget is PRIMARY constraint - _batch_sizer.py: resolve_provider_config() with DB-level context_window/max_output_tokens - _batch_sizer.py: INPUT_SAFETY_FACTOR applied, max_rows_by_output used as row cap - _llm_http.py: log actual usage.prompt_tokens/.completion_tokens from provider - _llm_call.py: retry only missing rows after finish_reason=length (save partial result) - models/llm.py + schema: provider-level context_window / max_output_tokens (nullable) - services/llm_provider.py: get_provider_token_config() helper - Alembic migration: add columns to llm_providers - Svelte ProviderConfig: collapsible Advanced: Token Limits section - 12 new tests (token budget, batch sizer, provider config) - All 492 tests pass
This commit is contained in:
@@ -40,6 +40,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: true,
|
||||
is_multimodal: false,
|
||||
max_images: null,
|
||||
context_window: null,
|
||||
max_output_tokens: null,
|
||||
});
|
||||
|
||||
let testStatus = $state({ type: "", message: "" });
|
||||
@@ -84,6 +86,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: true,
|
||||
is_multimodal: false,
|
||||
max_images: null,
|
||||
context_window: null,
|
||||
max_output_tokens: null,
|
||||
};
|
||||
editingProvider = null;
|
||||
testStatus = { type: "", message: "" };
|
||||
@@ -109,6 +113,8 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
is_active: Boolean(provider?.is_active),
|
||||
is_multimodal: Boolean(provider?.is_multimodal),
|
||||
max_images: provider?.max_images ?? null,
|
||||
context_window: provider?.context_window ?? null,
|
||||
max_output_tokens: provider?.max_output_tokens ?? null,
|
||||
};
|
||||
testStatus = { type: "", message: "" };
|
||||
availableModels = [];
|
||||
@@ -271,6 +277,18 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
delete submitData.api_key;
|
||||
}
|
||||
|
||||
// Normalize token limit fields: bind:value on <input type="number"> returns string or "";
|
||||
// Pydantic expects int | None, so convert empty/string to number or null.
|
||||
for (const field of ["context_window", "max_output_tokens"]) {
|
||||
const val = submitData[field];
|
||||
if (val === "" || val === null || val === undefined) {
|
||||
submitData[field] = null;
|
||||
} else if (typeof val === "string") {
|
||||
const num = Number(val);
|
||||
submitData[field] = Number.isNaN(num) ? null : num;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await requestApi(endpoint, method, submitData);
|
||||
showForm = false;
|
||||
@@ -570,6 +588,51 @@ import { SvelteSet } from "svelte/reactivity";
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- #region token_limits_advanced -->
|
||||
<!-- @BRIEF Collapsible "Advanced: Token Limits" section. context_window and max_output_tokens
|
||||
per-provider override for token budget estimation. NULL = use PROVIDER_DEFAULTS. -->
|
||||
<details class="border-t pt-3 mt-2">
|
||||
<summary class="text-sm font-medium text-text-muted cursor-pointer hover:text-text select-none outline-none">
|
||||
{$t.llm?.advanced_token_limits || "Advanced: Token Limits"}
|
||||
</summary>
|
||||
<div class="mt-2 space-y-2">
|
||||
<div>
|
||||
<label for="provider-context-window" class="block text-sm font-medium text-text">
|
||||
{$t.llm?.context_window_label || "Context Window (tokens)"}
|
||||
</label>
|
||||
<input
|
||||
id="provider-context-window"
|
||||
type="number"
|
||||
min="1000"
|
||||
max="256000"
|
||||
placeholder="{$t.llm?.auto_detect || "Auto"}"
|
||||
bind:value={formData.context_window}
|
||||
class="mt-1 block w-full border rounded-md p-2"
|
||||
/>
|
||||
<p class="mt-0.5 text-xs text-text-subtle">
|
||||
{$t.llm?.context_window_hint || "Leave blank for auto-detection from model name."}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<label for="provider-max-output-tokens" class="block text-sm font-medium text-text">
|
||||
{$t.llm?.max_output_label || "Max Output Tokens"}
|
||||
</label>
|
||||
<input
|
||||
id="provider-max-output-tokens"
|
||||
type="number"
|
||||
min="256"
|
||||
placeholder="{$t.llm?.auto_detect || "Auto"}"
|
||||
bind:value={formData.max_output_tokens}
|
||||
class="mt-1 block w-full border rounded-md p-2"
|
||||
/>
|
||||
<p class="mt-0.5 text-xs text-text-subtle">
|
||||
{$t.llm?.max_output_hint || "Must be less than context window. Leave blank for auto."}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
<!-- #endregion token_limits_advanced -->
|
||||
</div>
|
||||
|
||||
{#if testStatus.message}
|
||||
|
||||
Reference in New Issue
Block a user