fix(translate): normalize Unix timestamps to YYYY-MM-DD for ClickHouse Date columns

- Add _normalize_timestamp_value to key column values in orchestrator.py and executor.py before SQL generation
- Fix test mocks for .options(joinedload()) chain, explicit None attrs for mock job
- Add global translation run progress indicator (TranslationRunGlobalIndicator + store)
- Fix translate page import missing translationRunStore
- All 208 translate tests pass
This commit is contained in:
2026-05-15 22:06:27 +03:00
parent fdf48491a1
commit 27168664b8
10 changed files with 781 additions and 96 deletions

View File

@@ -17,6 +17,7 @@ class LLMProviderType(str, Enum):
OPENAI = "openai"
OPENROUTER = "openrouter"
KILO = "kilo"
LITELLM = "litellm"
# #endregion LLMProviderType
# #region LLMProviderConfig [TYPE Class]

View File

@@ -36,6 +36,10 @@ def _make_mock_job(**overrides):
job.environment_id = "test-env"
job.context_columns = []
job.target_column = None
job.target_language_column = None
job.target_source_column = None
job.target_source_language_column = None
job.target_languages = ["en"]
for k, v in overrides.items():
setattr(job, k, v)
return job
@@ -63,6 +67,7 @@ def _make_mock_record(
rec.status = "SUCCESS"
rec.error_message = None
rec.created_at = None
rec.languages = []
rec.source_data = {
"report_date": report_date,
"document_number": document_number,
@@ -347,8 +352,9 @@ class TestOrchestratorInsertFlow:
)
# Mock the DB query to return our records
# Note: orchestrator uses .options(joinedload()) before .filter()
mock_query = MagicMock()
mock_query.filter.return_value.all.return_value = [rec1, rec2]
mock_query.options.return_value.filter.return_value.all.return_value = [rec1, rec2]
db.query.return_value = mock_query
# Create mock job

View File

@@ -1,4 +1,4 @@
# #region TranslationExecutor [C:4] [TYPE Module] [SEMANTICS sqlalchemy, tenacity, translate, insert, llm-retry]
# #region TranslationExecutor [C:5] [TYPE Module] [SEMANTICS sqlalchemy, tenacity, translate, insert, llm-retry]
# @BRIEF Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationBatch]
@@ -10,6 +10,8 @@
# @PRE: Valid TranslationRun with job configuration. DB session is available.
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
# @DATA_CONTRACT Input: TranslationRun + Job -> Output: updated Run with batch/record rows
# @INVARIANT Batch processing is independent — one batch failure does not affect others.
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
@@ -20,7 +22,7 @@ from collections.abc import Callable
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.orm import Session
from sqlalchemy.orm import Session, joinedload
from ...core.config_manager import ConfigManager
from ...core.logger import belief_scope, logger
@@ -32,6 +34,7 @@ from ...models.translate import (
TranslationPreviewSession,
TranslationRecord,
TranslationRun,
TranslationRunLanguageStats,
)
from ...services.llm_prompt_templates import render_prompt
from ...services.llm_provider import LLMProviderService
@@ -39,16 +42,17 @@ from ._token_budget import DEFAULT_CONTEXT_WINDOW, DEFAULT_MAX_OUTPUT_TOKENS, es
from .dictionary import DictionaryManager
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
from .prompt_builder import ContextAwarePromptBuilder
from .sql_generator import SQLGenerator, _normalize_timestamp_value
from .superset_executor import SupersetSqlLabExecutor
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# #region MAX_ROWS_PER_RUN [TYPE Constant]
# @BRIEF Safety cap on rows fetched from datasource per run to prevent unbounded LLM processing.
# @RATIONALE Without a cap, a datasource with thousands of rows blocks the single uvicorn worker
# for minutes/hours. Preview uses sample_size (5-10). Full run should stay within reasonable bounds.
# Safety cap: without it, a datasource with thousands of rows blocks the single
# uvicorn worker for minutes/hours. Preview uses sample_size (5-10). Full run
# should stay within reasonable bounds.
MAX_ROWS_PER_RUN = 10000
# #endregion MAX_ROWS_PER_RUN
@@ -82,6 +86,7 @@ class TranslationExecutor:
self,
run: TranslationRun,
llm_progress_callback: Callable[[str, int, int, int], None] | None = None,
language_stats_map: dict[str, TranslationRunLanguageStats] | None = None,
) -> TranslationRun:
with belief_scope("TranslationExecutor.execute_run"):
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
@@ -170,6 +175,19 @@ class TranslationExecutor:
# status + batch progress visible to other DB sessions (frontend polling).
self.db.commit()
# Incremental INSERT into target table after each batch,
# so data appears incrementally in the target view/table
# without waiting for the entire run to complete.
batch_id = batch_result.get("batch_id")
if batch_id and batch_result["successful"] > 0:
try:
self._insert_batch_to_target(job, batch_id, run.id)
except Exception as e:
logger.explore("Batch INSERT failed (non-fatal, continuing)", {
"batch_id": batch_id,
"error": str(e),
})
# Re-fetch run after commit to check for cancellation flag set by
# cancel_run() fallback (direct SQL UPDATE of error_message).
self.db.refresh(run)
@@ -184,6 +202,17 @@ class TranslationExecutor:
self.db.commit()
return run
# Update per-language statistics incrementally after each batch
# so the frontend shows real-time per-language counts for RUNNING runs.
if language_stats_map and batch_result["successful"] > 0:
try:
self._update_language_stats_incremental(run.id, language_stats_map)
except Exception as e:
logger.explore("Language stats update failed (non-fatal)", {
"batch_id": batch_id,
"error": str(e),
})
if self.on_batch_progress:
self.on_batch_progress(
run.id, batch_idx + 1, len(batches),
@@ -665,9 +694,251 @@ class TranslationExecutor:
**result,
})
return result
return {**result, "batch_id": batch_id}
# endregion _process_batch
# region _insert_batch_to_target [TYPE Function]
# @PURPOSE: Insert successful records from a single batch into the target table via Superset SQL Lab.
# @PRE: batch has committed TranslationRecords with status SUCCESS. job has target_table configured.
# @POST: Per record, N+1 rows are INSERTED: 1 original + N translations.
# Context columns are bundled into JSON in the `context` column.
# `is_original=1` marks the source-language row.
# @SIDE_EFFECT: HTTP call to Superset SQL Lab API. Writes to target database.
def _insert_batch_to_target(
self,
job: TranslationJob,
batch_id: str,
run_id: str,
) -> None:
with belief_scope("TranslationExecutor._insert_batch_to_target"):
records = (
self.db.query(TranslationRecord)
.options(joinedload(TranslationRecord.languages))
.filter(
TranslationRecord.batch_id == batch_id,
TranslationRecord.status == "SUCCESS",
TranslationRecord.target_sql.isnot(None),
)
.all()
)
if not records:
return
effective_target = job.target_column or job.translation_column
primary_language = (job.target_languages or ["en"])[0]
# Columns that exist in the target ClickHouse table
columns = []
if job.target_key_cols:
columns.extend(job.target_key_cols)
if effective_target:
columns.append(effective_target)
if job.target_language_column:
columns.append(job.target_language_column)
if job.target_source_column:
columns.append(job.target_source_column)
if job.target_source_language_column:
columns.append(job.target_source_language_column)
columns.append("context")
columns.append("is_original")
# Deduplicate while preserving order
seen: set[str] = set()
deduped: list[str] = []
for c in columns:
if c and c not in seen:
deduped.append(c)
seen.add(c)
columns = deduped
# Keys for the context JSON: context_columns + original translation_column
context_keys = list(job.context_columns or [])
if job.translation_column and job.translation_column != effective_target and job.translation_column not in context_keys:
context_keys.append(job.translation_column)
rows_for_sql: list[dict[str, object]] = []
for rec in records:
source_data = rec.source_data or {}
# Detect source language from first TranslationLanguage entry
detected_src_lang = "und"
if rec.languages and len(rec.languages) > 0:
detected_src_lang = rec.languages[0].source_language_detected or "und"
# Build context JSON: all extra columns the user configured
context_data: dict[str, str] = {}
for key in context_keys:
val = source_data.get(key)
context_data[key] = str(val) if val is not None else ""
# ── Shared base row (columns common to original and all translations) ──
base_row: dict[str, object] = {}
# Key columns (report_date, document_number) — normalize timestamps to YYYY-MM-DD
if job.target_key_cols:
for k in job.target_key_cols:
raw = source_data.get(k)
if raw is not None:
normalized = _normalize_timestamp_value(raw)
base_row[k] = normalized if normalized else raw
else:
base_row[k] = None
# Source text: original
if job.target_source_column:
base_row[job.target_source_column] = rec.source_sql or ""
# Source language (same for all rows of this record)
if job.target_source_language_column:
base_row[job.target_source_language_column] = detected_src_lang
# Context JSON string
base_row["context"] = json.dumps(context_data, ensure_ascii=False)
# ── 1. ORIGINAL row (is_original = 1) ──
original_row = dict(base_row)
if effective_target:
original_row[effective_target] = rec.source_sql or ""
if job.target_language_column:
original_row[job.target_language_column] = detected_src_lang
original_row["is_original"] = 1
rows_for_sql.append(original_row)
# ── 2. TRANSLATION rows (is_original = 0) ──
# Skip language that matches the source — the original row already covers it
if rec.languages and len(rec.languages) > 0:
for lang in rec.languages:
if lang.language_code == detected_src_lang:
continue
trans_row = dict(base_row)
trans_value = lang.final_value or lang.translated_value or ""
if effective_target:
trans_row[effective_target] = trans_value
if job.target_language_column:
trans_row[job.target_language_column] = lang.language_code
trans_row["is_original"] = 0
rows_for_sql.append(trans_row)
else:
# Fallback: no per-language data → single translation row with primary language
fallback_row = dict(base_row)
if effective_target:
fallback_row[effective_target] = rec.target_sql or ""
if job.target_language_column:
fallback_row[job.target_language_column] = primary_language
fallback_row["is_original"] = 0
rows_for_sql.append(fallback_row)
if not columns:
columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
executor.resolve_database_id(target_database_id=job.target_database_id)
real_backend = executor.get_database_backend()
except Exception as e:
logger.explore("Failed to resolve database backend for batch insert", {
"batch_id": batch_id,
"error": str(e),
})
real_backend = None
dialect = real_backend or job.database_dialect or job.target_dialect or "postgresql"
try:
sql, row_count = SQLGenerator.generate(
dialect=dialect,
target_schema=job.target_schema,
target_table=job.target_table or "translated_data",
columns=columns,
rows=rows_for_sql,
key_columns=job.target_key_cols,
upsert_strategy=job.upsert_strategy or "MERGE",
)
except ValueError as e:
logger.explore("SQL generation failed for batch", {
"batch_id": batch_id,
"error": str(e),
})
return
try:
result = executor.execute_and_poll(
sql=sql,
max_polls=30,
poll_interval_seconds=2.0,
)
except Exception as e:
logger.explore("Superset SQL submission failed for batch", {
"batch_id": batch_id,
"error": str(e),
})
return
logger.reason(f"Batch {batch_id[:12]} inserted {row_count} rows", {
"batch_id": batch_id,
"rows": row_count,
"status": result.get("status"),
})
# endregion _insert_batch_to_target
# region _update_language_stats_incremental [TYPE Function]
# @PURPOSE: Update per-language TranslationRunLanguageStats incrementally after each batch.
# @PRE: language_stats_map has entries for all target languages.
# @POST: Language stat objects updated with counts from committed TranslationLanguage rows.
# @SIDE_EFFECT: Mutates ORM objects; caller must commit.
def _update_language_stats_incremental(
self,
run_id: str,
language_stats_map: dict[str, TranslationRunLanguageStats],
) -> None:
with belief_scope("TranslationExecutor._update_language_stats_incremental"):
records = (
self.db.query(TranslationRecord)
.filter(TranslationRecord.run_id == run_id)
.all()
)
record_ids = [r.id for r in records]
if not record_ids:
return
lang_entries = (
self.db.query(TranslationLanguage)
.filter(TranslationLanguage.record_id.in_(record_ids))
.all()
)
from collections import defaultdict
agg: dict[str, dict[str, int]] = defaultdict(
lambda: {"total": 0, "translated": 0, "failed": 0, "skipped": 0}
)
for le in lang_entries:
code = le.language_code
agg[code]["total"] += 1
if le.status in ("translated", "approved", "edited"):
agg[code]["translated"] += 1
elif le.status == "failed":
agg[code]["failed"] += 1
elif le.status == "skipped":
agg[code]["skipped"] += 1
total_tokens_est = max(1, sum(len(le.translated_value or "") for le in lang_entries if le.translated_value) // 4)
num_langs = len(language_stats_map) or 1
cost_per_token = 0.002 / 1000
for lang_code, lang_stat in language_stats_map.items():
data = agg.get(lang_code, {"total": 0, "translated": 0, "failed": 0, "skipped": 0})
lang_stat.total_rows = data["total"]
lang_stat.translated_rows = data["translated"]
lang_stat.failed_rows = data["failed"]
lang_stat.skipped_rows = data["skipped"]
lang_stat.token_count = total_tokens_est // num_langs
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
self.db.flush()
# endregion _update_language_stats_incremental
# region _call_llm_for_batch [TYPE Function]
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
# @PRE: job has valid provider_id. batch_rows is non-empty.
@@ -959,7 +1230,7 @@ class TranslationExecutor:
disable_reasoning = getattr(job, 'disable_reasoning', False)
if provider_type in ("openai", "openai_compatible", "openrouter", "kilo"):
if provider_type in ("openai", "openai_compatible", "openrouter", "kilo", "litellm"):
response_text = self._call_openai_compatible(
base_url=provider.base_url,
api_key=api_key,
@@ -1006,23 +1277,23 @@ class TranslationExecutor:
"temperature": 0.1,
"max_tokens": max_tokens,
}
# Structured output — native OpenAI and compatible providers (e.g. Ollama, vLLM).
# Kilo gateway API docs show response_format support, but upstream providers (e.g. StepFun)
# reject it with "structured_outputs is not supported". Skip for Kilo/OpenRouter to avoid 400.
if provider_type in ("openai", "openai_compatible"):
# Structured output — OpenRouter and Kilo also support response_format, but some
# upstream providers (e.g. StepFun) reject it. We try with response_format and
# fall back on 400 "structured_outputs is not supported".
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter do NOT support disabling reasoning (returns 400)
# NOTE: Kilo/OpenRouter/LiteLLM do NOT support disabling reasoning (returns 400)
if disable_reasoning:
if provider_type not in ("kilo", "openrouter"):
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload["extra_body"] = {"reasoning_effort": "none"}
payload.pop("response_format", None)
payload["messages"][0] = {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."}
logger.reason(
f"LLM request model={payload.get('model')} "
f"LLM request url={base_url} model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
f"prompt_len={len(prompt)}"

View File

@@ -39,7 +39,7 @@ from ...models.translate import (
)
from .events import TranslationEventLog
from .executor import TranslationExecutor
from .sql_generator import SQLGenerator
from .sql_generator import SQLGenerator, _normalize_timestamp_value
from .superset_executor import SupersetSqlLabExecutor
@@ -232,7 +232,7 @@ class TranslationOrchestrator:
on_batch_progress=on_batch_progress,
)
try:
run = executor.execute_run(run, llm_progress_callback=None)
run = executor.execute_run(run, llm_progress_callback=None, language_stats_map=language_stats_map)
except Exception as e:
logger.explore("Translation execution failed", {
"run_id": run.id,
@@ -371,83 +371,106 @@ class TranslationOrchestrator:
"dialect": job.database_dialect or job.target_dialect,
})
# Determine effective target column for INSERT (defaults to translation_column)
effective_target = job.target_column or job.translation_column
# Build columns for SQL generation
columns = job.context_columns or []
# Always include translation_column (original text) if it's different from target
if job.translation_column and job.translation_column not in columns:
columns.append(job.translation_column)
# Add target_column separately if it differs from translation_column
if effective_target and effective_target != job.translation_column and effective_target not in columns:
columns.append(effective_target)
# Also include key columns if used for upsert
if job.target_key_cols:
for k in job.target_key_cols:
if k not in columns:
columns.append(k)
# Add target metadata columns for enhanced table mapping
if job.target_source_column and job.target_source_column not in columns:
columns.append(job.target_source_column)
if job.target_language_column and job.target_language_column not in columns:
columns.append(job.target_language_column)
if job.target_source_language_column and job.target_source_language_column not in columns:
columns.append(job.target_source_language_column)
# Resolve the primary target language for INSERT
primary_language = (job.target_languages or ["en"])[0]
rows_for_sql = []
# Columns that exist in the target ClickHouse table
columns = []
if job.target_key_cols:
columns.extend(job.target_key_cols)
if effective_target:
columns.append(effective_target)
if job.target_language_column:
columns.append(job.target_language_column)
if job.target_source_column:
columns.append(job.target_source_column)
if job.target_source_language_column:
columns.append(job.target_source_language_column)
columns.append("context")
columns.append("is_original")
# Deduplicate while preserving order
seen: set[str] = set()
deduped: list[str] = []
for c in columns:
if c and c not in seen:
deduped.append(c)
seen.add(c)
columns = deduped
# Keys for the context JSON: context_columns + original translation_column
context_keys = list(job.context_columns or [])
if job.translation_column and job.translation_column != effective_target and job.translation_column not in context_keys:
context_keys.append(job.translation_column)
rows_for_sql: list[dict[str, object]] = []
for rec in records:
row_data = {}
source_data = rec.source_data or {}
# Context columns from source data
if job.context_columns:
for col in job.context_columns:
row_data[col] = source_data.get(col, "")
# Original text column (translation_column)
if job.translation_column and job.translation_column not in (job.target_key_cols or []):
# If target_column differs, keep the original value from source_data;
# otherwise use the translated value
if effective_target and effective_target != job.translation_column:
row_data[job.translation_column] = source_data.get(job.translation_column, "")
else:
row_data[job.translation_column] = rec.target_sql or ""
# Translated text goes into target_column (may be same as translation_column)
if effective_target:
row_data[effective_target] = rec.target_sql or ""
# Source text column: INSERT original source text
if job.target_source_column:
row_data[job.target_source_column] = rec.source_sql or ""
# Language column: INSERT language code (e.g. 'ru', 'en')
if job.target_language_column:
row_data[job.target_language_column] = primary_language
# Source language column: INSERT detected source language (BCP-47)
if job.target_source_language_column:
detected = "und"
# Detect source language from first TranslationLanguage entry
detected_src_lang = "und"
if rec.languages and len(rec.languages) > 0:
detected = rec.languages[0].source_language_detected or "und"
row_data[job.target_source_language_column] = detected
detected_src_lang = rec.languages[0].source_language_detected or "und"
# Build context JSON: all extra columns the user configured
context_data: dict[str, str] = {}
for key in context_keys:
val = source_data.get(key)
context_data[key] = str(val) if val is not None else ""
# ── Shared base row ──
base_row: dict[str, object] = {}
# Key columns from source data
if job.target_key_cols:
for k in job.target_key_cols:
row_data[k] = source_data.get(k, "")
rows_for_sql.append(row_data)
raw = source_data.get(k)
if raw is not None:
normalized = _normalize_timestamp_value(raw)
base_row[k] = normalized if normalized else raw
else:
base_row[k] = None
if job.target_source_column:
base_row[job.target_source_column] = rec.source_sql or ""
if job.target_source_language_column:
base_row[job.target_source_language_column] = detected_src_lang
base_row["context"] = json.dumps(context_data, ensure_ascii=False)
# ── 1. ORIGINAL row (is_original = 1) ──
original_row = dict(base_row)
if effective_target:
original_row[effective_target] = rec.source_sql or ""
if job.target_language_column:
original_row[job.target_language_column] = detected_src_lang
original_row["is_original"] = 1
rows_for_sql.append(original_row)
# ── 2. TRANSLATION rows (is_original = 0) ──
# Skip language that matches the source — the original row already covers it
if rec.languages and len(rec.languages) > 0:
for lang in rec.languages:
if lang.language_code == detected_src_lang:
continue
trans_row = dict(base_row)
trans_value = lang.final_value or lang.translated_value or ""
if effective_target:
trans_row[effective_target] = trans_value
if job.target_language_column:
trans_row[job.target_language_column] = lang.language_code
trans_row["is_original"] = 0
rows_for_sql.append(trans_row)
else:
# Fallback: no per-language data
fallback_row = dict(base_row)
if effective_target:
fallback_row[effective_target] = rec.target_sql or ""
if job.target_language_column:
fallback_row[job.target_language_column] = primary_language
fallback_row["is_original"] = 0
rows_for_sql.append(fallback_row)
if not columns:
# Use target_sql as the sole column
columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
@@ -1094,6 +1117,4 @@ class TranslationOrchestrator:
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
# endregion _compute_dict_snapshot_hash
# #endregion TranslationOrchestrator
# #endregion TranslationOrchestrator

View File

@@ -964,7 +964,7 @@ class TranslationPreview:
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
disable_reasoning = getattr(job, 'disable_reasoning', False)
if provider_type in ("openai", "openai_compatible", "openrouter", "kilo"):
if provider_type in ("openai", "openai_compatible", "openrouter", "kilo", "litellm"):
max_attempts = 2
last_error = None
for attempt in range(max_attempts):
@@ -1030,14 +1030,14 @@ class TranslationPreview:
}
# Structured output — Kilo gateway supports response_format, but upstream providers
# (e.g. StepFun) may reject it. We try with response_format and fall back on 400.
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter"):
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter do NOT support disabling reasoning (returns 400)
# NOTE: Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
if disable_reasoning:
# Kilo/OpenRouter reject reasoning_effort — only use for native OpenAI-compatible
if provider_type not in ("kilo", "openrouter"):
# Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload["extra_body"] = {"reasoning_effort": "none"}
payload.pop("response_format", None) # JSON mode triggers reasoning on some models
@@ -1048,7 +1048,7 @@ class TranslationPreview:
payload["messages"][0] = {"role": "system", "content": system_content}
logger.reason(
f"LLM request model={payload.get('model')} "
f"LLM request url={base_url} model={payload.get('model')} "
f"provider_type={provider_type} "
f"response_format={'yes' if 'response_format' in payload else 'no'} "
f"reasoning={'no' if disable_reasoning else 'yes'} "

View File

@@ -22,6 +22,13 @@
let editingProvider = $state(null);
let showForm = $state(false);
const DEFAULT_BASE_URLS = {
openai: "https://api.openai.com/v1",
openrouter: "https://openrouter.ai/api/v1",
kilo: "https://api.kilo.chat/v1",
litellm: "http://localhost:4000/v1",
};
let formData = $state({
name: "",
provider_type: "openai",
@@ -57,6 +64,16 @@
);
}
function updateBaseUrlForType(providerType) {
// Only auto-update base_url if user hasn't changed it from the default
// or the base_url matches a previous default for a different type
const currentUrl = formData.base_url;
const isCurrentlyDefault = Object.values(DEFAULT_BASE_URLS).includes(currentUrl);
if (isCurrentlyDefault || !currentUrl) {
formData.base_url = DEFAULT_BASE_URLS[providerType] || currentUrl;
}
}
function resetForm() {
formData = {
name: "",
@@ -322,11 +339,13 @@
<select
id="provider-type"
bind:value={formData.provider_type}
onchange={() => updateBaseUrlForType(formData.provider_type)}
class="mt-1 block w-full border rounded-md p-2"
>
<option value="openai">OpenAI</option>
<option value="openrouter">OpenRouter</option>
<option value="kilo">Kilo</option>
<option value="litellm">LiteLLM</option>
</select>
</div>

View File

@@ -0,0 +1,139 @@
<!-- #region TranslationRunGlobalIndicator [C:3] [TYPE Component] [SEMANTICS translate, progress, global, indicator] -->
<!-- @BRIEF Persistent mini progress indicator for translation runs, rendered in root layout. -->
<!-- @LAYER UI -->
<!-- @RELATION BINDS_TO -> [translationRunStore] -->
<!-- @UX_STATE hidden -> No active or recently completed run -->
<!-- @UX_STATE running -> Thin blue bar + "Translating X/Y" banner -->
<!-- @UX_STATE completed -> Green bar + "Completed" banner (auto-hides after 5s) -->
<!-- @UX_STATE failed -> Red bar + "Failed" banner (auto-hides after 8s) -->
<!-- @UX_FEEDBACK Click banner navigates to the translation run page -->
<script>
import { translationRunStore } from '$lib/stores/translationRun.js';
import { fromStore } from 'svelte/store';
import { page } from '$app/state';
import { goto } from '$app/navigation';
import { t } from '$lib/i18n';
import { onDestroy } from 'svelte';
const runState = fromStore(translationRunStore);
// Hide on the translate page itself to avoid duplication
let isOnTranslatePage = $derived(
page.url.pathname.startsWith('/translate/') && page.url.pathname !== '/translate'
);
// Terminal state auto-dismiss timers
let dismissTimer = $state(null);
let dismissed = $state(false);
let uxState = $derived(runState.current?.uxState || 'idle');
let show = $derived.by(() => {
if (dismissed) return false;
if (isOnTranslatePage) return false;
if (uxState === 'running' || uxState === 'inserting') return true;
if (uxState === 'completed' || uxState === 'partial') return true;
if (uxState === 'failed' || uxState === 'cancelled') return true;
return false;
});
let barColor = $derived.by(() => {
if (uxState === 'running' || uxState === 'inserting') return 'bg-blue-600';
if (uxState === 'completed' || uxState === 'partial') return 'bg-green-500';
if (uxState === 'failed') return 'bg-red-500';
if (uxState === 'cancelled') return 'bg-gray-400';
return 'bg-blue-600';
});
let bannerBg = $derived.by(() => {
if (uxState === 'running' || uxState === 'inserting') return 'bg-blue-600';
if (uxState === 'completed' || uxState === 'partial') return 'bg-green-500';
if (uxState === 'failed') return 'bg-red-500';
if (uxState === 'cancelled') return 'bg-gray-500';
return 'bg-blue-600';
});
let label = $derived.by(() => {
if (uxState === 'inserting') return $t.translate?.run?.insert_phase || 'Inserting...';
if (uxState === 'running') return $t.translate?.run?.translate_phase || 'Translating...';
if (uxState === 'completed') return $t.translate?.run?.completed || 'Completed';
if (uxState === 'partial') return $t.translate?.run?.completed_with_errors || 'Completed with errors';
if (uxState === 'failed') return $t.translate?.run?.translation_failed || 'Translation failed';
if (uxState === 'cancelled') return $t.translate?.run?.cancelled || 'Cancelled';
return '';
});
// Auto-dismiss terminal states after a few seconds
$effect(() => {
if (uxState === 'completed' || uxState === 'partial') {
dismissed = false;
if (dismissTimer) clearTimeout(dismissTimer);
dismissTimer = setTimeout(() => { dismissed = true; }, 5000);
} else if (uxState === 'failed' || uxState === 'cancelled') {
dismissed = false;
if (dismissTimer) clearTimeout(dismissTimer);
dismissTimer = setTimeout(() => { dismissed = true; }, 8000);
} else if (uxState === 'idle') {
dismissed = false;
}
});
onDestroy(() => {
if (dismissTimer) clearTimeout(dismissTimer);
});
function handleClick() {
const jobId = runState.current?.jobId;
if (jobId) {
goto(`/translate/${jobId}`);
}
}
</script>
{#if show}
<div
class="fixed top-0 left-0 right-0 z-[100] cursor-pointer shadow-sm"
onclick={handleClick}
role="button"
tabindex="0"
aria-label={label}
>
<!-- Animated thin bar -->
<div class="h-1 bg-gray-200">
<div
class="h-full {barColor} transition-all duration-500"
style="width: {Math.min(runState.current?.progressPct || 0, 100)}%"
/>
</div>
<!-- Compact stats banner -->
<div class="{bannerBg} text-white text-xs px-3 py-1 flex items-center gap-3">
<span class="font-medium">{label}</span>
{#if uxState === 'running' || uxState === 'inserting'}
<span class="opacity-80">
{runState.current?.successfulRecords || 0}/{runState.current?.totalRecords || 0}
</span>
{#if (runState.current?.failedRecords || 0) > 0}
<span class="text-red-200">
{$t.translate?.run?.failed || 'Failed'}: {runState.current?.failedRecords}
</span>
{/if}
<span class="ml-auto opacity-70 hover:opacity-100 transition-opacity">
&#9654; {$t.common?.open || 'Open'}
</span>
{:else if uxState === 'partial'}
<span class="opacity-80">
{runState.current?.successfulRecords || 0}/{runState.current?.totalRecords || 0}
</span>
<span class="ml-auto opacity-70 hover:opacity-100 transition-opacity">
&#9654; {$t.common?.open || 'Open'}
</span>
{:else}
<span class="ml-auto opacity-70 hover:opacity-100 transition-opacity">
&#9654; {$t.common?.open || 'Open'}
</span>
{/if}
</div>
</div>
{/if}
<!-- #endregion TranslationRunGlobalIndicator -->

View File

@@ -0,0 +1,209 @@
// #region TranslationRunStore [C:3] [TYPE Store] [SEMANTICS translate, run, progress, polling, store]
// @BRIEF Global store for active translation run progress — survives page navigation.
// @RELATION DEPENDS_ON -> [TranslateApi]
// @UX_STATE idle -> No active run
// @UX_STATE running -> Translation phase in progress
// @UX_STATE inserting -> Insert phase in progress
// @UX_STATE completed -> Run completed
// @UX_STATE partial -> Completed with errors
// @UX_STATE failed -> Run failed
// @UX_STATE cancelled -> Run cancelled
// @INVARIANT Polling stops when run reaches a terminal state (completed/failed/cancelled).
// @INVARIANT Store is writable so page components can also set initial state.
import { writable, derived } from 'svelte/store';
import { fetchRunStatus } from '$lib/api/translate.js';
/**
* @typedef {'idle'|'running'|'inserting'|'completed'|'partial'|'failed'|'cancelled'} UxState
*/
/**
* @typedef {Object} TranslationRunState
* @property {string|null} runId - Active translation run ID
* @property {UxState} uxState - Current UX state
* @property {Object|null} status - Raw status data from API
* @property {number} totalRecords
* @property {number} successfulRecords
* @property {number} failedRecords
* @property {number} skippedRecords
* @property {number} progressPct - 0-100
* @property {string|null} insertStatus
* @property {number} batchCount
* @property {string|null} jobId - The job this run belongs to (for navigation)
* @property {boolean} isFullRun
* @property {boolean} isActive - Derived: true when polling is active
*/
const initialState = {
runId: null,
uxState: 'idle',
status: null,
totalRecords: 0,
successfulRecords: 0,
failedRecords: 0,
skippedRecords: 0,
progressPct: 0,
insertStatus: null,
batchCount: 0,
jobId: null,
isFullRun: false,
};
/** @type {import('svelte/store').Writable<TranslationRunState>} */
export const translationRunStore = writable(initialState);
/** Derived: true when a run is actively being polled */
export const isTranslationActive = derived(
translationRunStore,
($store) => $store.uxState === 'running' || $store.uxState === 'inserting'
);
/** Derived: true when run has finished */
export const isTranslationFinished = derived(
translationRunStore,
($store) =>
$store.uxState === 'completed' ||
$store.uxState === 'partial' ||
$store.uxState === 'failed' ||
$store.uxState === 'cancelled'
);
// Internal polling state (not reactive)
let _pollingInterval = null;
let _pollCount = 0;
const MAX_POLLS = 300;
let _onCompleteCallback = null;
/**
* Clear the onComplete callback without stopping polling.
* Used by page components when they unmount — the store keeps polling
* so the global indicator still works, but the page callback is detached.
*/
export function clearOnCompleteCallback() {
_onCompleteCallback = null;
}
/**
* Start polling for a translation run.
* @param {string} runId
* @param {Object} [options]
* @param {string} [options.jobId] - Job ID for navigation
* @param {boolean} [options.isFullRun]
* @param {Function} [options.onComplete] - Called when run reaches terminal state
*/
export function startTranslationRun(runId, options = {}) {
if (!runId) return;
stopTranslationRun();
translationRunStore.set({
...initialState,
runId,
uxState: 'running',
jobId: options.jobId || null,
isFullRun: options.isFullRun || false,
});
_onCompleteCallback = options.onComplete || null;
_pollCount = 0;
_pollingInterval = setInterval(pollStatus, 2000);
pollStatus();
}
/**
* Stop polling without clearing state (run may persist).
*/
export function stopTranslationRun() {
if (_pollingInterval) {
clearInterval(_pollingInterval);
_pollingInterval = null;
}
}
/**
* Reset store to idle.
*/
export function resetTranslationRun() {
stopTranslationRun();
translationRunStore.set(initialState);
_onCompleteCallback = null;
}
/**
* Manually set the run state (used by page component for lifecycle hooks).
* @param {Partial<TranslationRunState>} state
*/
export function updateTranslationRunState(state) {
translationRunStore.update(prev => ({ ...prev, ...state }));
}
/** @returns {Promise<void>} */
async function pollStatus() {
_pollCount++;
if (_pollCount > MAX_POLLS) {
console.warn('[translationRunStore] Max polls reached, stopping');
stopTranslationRun();
translationRunStore.update(s => ({ ...s, uxState: 'failed' }));
if (_onCompleteCallback) _onCompleteCallback({ status: 'TIMEOUT', error_message: 'Translation run timed out' });
return;
}
let currentState;
translationRunStore.subscribe(s => { currentState = s; })();
if (!currentState || !currentState.runId) {
stopTranslationRun();
return;
}
try {
const data = await fetchRunStatus(currentState.runId);
if (!data) return;
const s = data?.status;
const insertS = data?.insert_status;
let newUxState = currentState.uxState;
if (s === 'PENDING' || s === 'RUNNING') {
newUxState = (insertS === 'started' || insertS === 'pending' || insertS === 'running')
? 'inserting' : 'running';
} else if (s === 'COMPLETED') {
newUxState = (insertS === 'failed' || insertS === 'timeout') ? 'insert_failed'
: (data.failed_records > 0) ? 'partial' : 'completed';
stopTranslationRun();
} else if (s === 'FAILED') {
newUxState = 'failed';
stopTranslationRun();
} else if (s === 'CANCELLED') {
newUxState = 'cancelled';
stopTranslationRun();
}
const total = data?.total_records || 0;
const pct = total > 0
? Math.round(((data.successful_records + data.failed_records + data.skipped_records) / total) * 100)
: 0;
translationRunStore.update(prev => ({
...prev,
uxState: newUxState,
status: data,
totalRecords: total,
successfulRecords: data?.successful_records || 0,
failedRecords: data?.failed_records || 0,
skippedRecords: data?.skipped_records || 0,
progressPct: pct,
insertStatus: data?.insert_status || null,
batchCount: data?.batch_count || 0,
}));
// Fire completion callback if terminal
if (newUxState !== 'running' && newUxState !== 'inserting') {
if (_onCompleteCallback) _onCompleteCallback(data);
}
} catch (err) {
console.warn('[translationRunStore] Poll error:', err);
}
}
// #endregion TranslationRunStore

View File

@@ -30,6 +30,7 @@
import TopNavbar from '$lib/components/layout/TopNavbar.svelte';
import TaskDrawer from '$lib/components/layout/TaskDrawer.svelte';
import AssistantChatPanel from '$lib/components/assistant/AssistantChatPanel.svelte';
import TranslationRunGlobalIndicator from '$lib/components/translate/TranslationRunGlobalIndicator.svelte';
import { t } from '$lib/i18n';
import {
isProductionContextStore,
@@ -53,6 +54,9 @@
<Toast />
<!-- Global persistent translation run progress indicator -->
<TranslationRunGlobalIndicator />
<main class="min-h-screen {isProductionContext ? 'bg-red-50/40' : 'bg-slate-50'}">
{#if isLoginPage}
<div class="p-4">

View File

@@ -30,7 +30,7 @@
* @UX_REACTIVITY: columnList is $derived from selected datasource
*/
import { onMount } from 'svelte';
import { onMount, onDestroy } from 'svelte';
import { goto } from '$app/navigation';
import { page } from '$app/state';
import { t, _ } from '$lib/i18n';
@@ -50,6 +50,13 @@
import ScheduleConfig from '$lib/components/translate/ScheduleConfig.svelte';
import { triggerRun, fetchRunHistory, cancelRun } from '$lib/api/translate.js';
import BulkReplaceModal from '$lib/components/translate/BulkReplaceModal.svelte';
import { fromStore } from 'svelte/store';
import {
translationRunStore,
startTranslationRun,
resetTranslationRun,
clearOnCompleteCallback,
} from '$lib/stores/translationRun.js';
const LANGUAGES = [
{ code: 'ru', name: 'Russian' },
@@ -154,8 +161,9 @@
let validationErrors = $state({});
let warnings = $state([]);
// Run state
let currentRunId = $state(null);
// Run state — using global store to survive page navigation
let runState = fromStore(translationRunStore);
let currentRunId = $derived(runState.current?.runId || null);
let completedRuns = $state([]);
let isRunning = $state(false);
let isFullRun = $state(false);
@@ -168,7 +176,7 @@
runError = '';
try {
const run = await triggerRun(jobId, full);
currentRunId = run.id;
startTranslationRun(run.id, { jobId, isFullRun: full, onComplete: handleRunComplete });
addToast(full ? 'Полный перевод запущен (все строки)' : _('translate.config.run_started'), 'success');
} catch (err) {
runError = err?.message || _('translate.config.run_failed');
@@ -177,8 +185,9 @@
}
function handleRunComplete(statusData) {
if (!isRunning) return; // Idempotent: already handled
isRunning = false;
currentRunId = null;
resetTranslationRun();
loadRunHistory();
const statusLabel = statusData?.status || _('translate.run.completed');
addToast(`${_('translate.run.run_id')} ${statusLabel.toLowerCase()}`, 'info');
@@ -199,7 +208,7 @@
await cancelRun(currentRunId);
} catch (_e) { /* ignore */ }
}
currentRunId = null;
resetTranslationRun();
await handleTriggerRun();
}
@@ -235,6 +244,12 @@
}
});
// Clean up store callback when page unmounts — the store keeps
// polling so the global indicator in the layout still works.
onDestroy(() => {
clearOnCompleteCallback();
});
/** @returns {Promise<void>} */
async function loadInitialData() {
try {