feat: implement LLM table translation service with searchable datasource dropdown, i18n, sidebar, and RBAC
- Add backend plugin modules: preview, executor, orchestrator, events, sql_generator, superset_executor, dictionary, scheduler, metrics - Add API routes for translate jobs, runs, dictionaries, preview, schedule, metrics, corrections - Add ORM models (12 tables) and Pydantic schemas (15 DTOs) - Register translate router in app.py with RBAC permission guards - Add frontend pages: job list, job config, dictionary list/editor, history - Add 7 reusable Svelte 5 components: Preview, RunProgress, RunResult, TermCorrection, BulkCorrection, ScheduleConfig, MetricsDashboard - Add searchable datasource dropdown with Superset API integration - Add i18n (en/ru) with ~300 keys across common, config, jobs, dictionaries, preview namespaces - Add Translation sidebar category with Jobs/Dictionaries/History sub-items - Hide health monitor error toast via suppressToast API option - Add 69 backend tests and 44 frontend test files - Fix: SupersetClient env resolution (string -> Environment object) - Fix: Dataset detail API returns proper database dict - Fix: Database dialect extraction fallback when metadata incomplete
This commit is contained in:
626
backend/src/plugins/translate/executor.py
Normal file
626
backend/src/plugins/translate/executor.py
Normal file
@@ -0,0 +1,626 @@
|
||||
# [DEF:TranslationExecutor:Module]
|
||||
# @COMPLEXITY: 4
|
||||
# @SEMANTICS: translate, executor, batch, llm
|
||||
# @PURPOSE: Process translation in batches: fetch source rows, call LLM, persist TranslationBatch and TranslationRecord rows.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationBatch]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRecord]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun]
|
||||
# @RELATION: DEPENDS_ON -> [LLMProviderService]
|
||||
# @RELATION: DEPENDS_ON -> [DictionaryManager]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationPreview]
|
||||
# @PRE: Valid TranslationRun with job configuration. DB session is available.
|
||||
# @POST: TranslationBatch and TranslationRecord rows are created. Run status is updated.
|
||||
# @SIDE_EFFECT: Calls LLM provider; creates DB rows; updates run statistics.
|
||||
# @RATIONALE: Batch processing with retry — independent batches allow partial recovery.
|
||||
# @REJECTED: Single monolithic LLM call — would lose all progress on any failure.
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Callable
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...models.translate import (
|
||||
TranslationJob,
|
||||
TranslationRun,
|
||||
TranslationBatch,
|
||||
TranslationRecord,
|
||||
TranslationPreviewSession,
|
||||
TranslationPreviewRecord,
|
||||
)
|
||||
from ...services.llm_provider import LLMProviderService
|
||||
from ...services.llm_prompt_templates import render_prompt
|
||||
from .dictionary import DictionaryManager
|
||||
from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
||||
|
||||
|
||||
# [DEF:MAX_RETRIES_PER_BATCH:Constant]
|
||||
# @PURPOSE: Maximum number of retries for a single batch before marking it failed.
|
||||
MAX_RETRIES_PER_BATCH = 3
|
||||
|
||||
|
||||
# [DEF:TranslationExecutor:Class]
|
||||
# @COMPLEXITY: 4
|
||||
# @PURPOSE: Process translation batches: fetch source rows, filter dict, call LLM, persist results.
|
||||
# @PRE: DB session and config manager available.
|
||||
# @POST: Batches and records created with status tracking.
|
||||
# @SIDE_EFFECT: LLM API calls; DB writes.
|
||||
class TranslationExecutor:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Session,
|
||||
config_manager: ConfigManager,
|
||||
current_user: Optional[str] = None,
|
||||
on_batch_progress: Optional[Callable[[str, int, int, int, int], None]] = None,
|
||||
):
|
||||
self.db = db
|
||||
self.config_manager = config_manager
|
||||
self.current_user = current_user
|
||||
self.on_batch_progress = on_batch_progress
|
||||
self._current_run_id: Optional[str] = None
|
||||
|
||||
# [DEF:execute_run:Function]
|
||||
# @PURPOSE: Run full translation execution for a TranslationRun.
|
||||
# @PRE: run is in PENDING or RUNNING status with valid job config.
|
||||
# @POST: Run is populated with batches and records.
|
||||
# @SIDE_EFFECT: LLM API calls; DB batch writes.
|
||||
def execute_run(
|
||||
self,
|
||||
run: TranslationRun,
|
||||
llm_progress_callback: Optional[Callable[[str, int, int, int], None]] = None,
|
||||
) -> TranslationRun:
|
||||
with belief_scope("TranslationExecutor.execute_run"):
|
||||
job = self.db.query(TranslationJob).filter(TranslationJob.id == run.job_id).first()
|
||||
if not job:
|
||||
raise ValueError(f"Job '{run.job_id}' not found for run '{run.id}'")
|
||||
|
||||
logger.reason("Starting translation execution", {
|
||||
"run_id": run.id,
|
||||
"job_id": job.id,
|
||||
"batch_size": job.batch_size,
|
||||
})
|
||||
|
||||
# Mark run as RUNNING
|
||||
run.status = "RUNNING"
|
||||
run.started_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
# Fetch source rows from the accepted preview session
|
||||
source_rows = self._fetch_source_rows(job.id, run.id)
|
||||
if not source_rows:
|
||||
logger.explore("No source rows to translate", {"run_id": run.id})
|
||||
run.status = "COMPLETED"
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
return run
|
||||
|
||||
total_rows = len(source_rows)
|
||||
run.total_records = total_rows
|
||||
|
||||
# Split into batches
|
||||
batch_size = job.batch_size or 50
|
||||
batches = [
|
||||
source_rows[i:i + batch_size]
|
||||
for i in range(0, total_rows, batch_size)
|
||||
]
|
||||
|
||||
logger.reason(f"Processing {len(batches)} batches", {
|
||||
"run_id": run.id,
|
||||
"total_rows": total_rows,
|
||||
"batch_size": batch_size,
|
||||
})
|
||||
|
||||
successful_records = 0
|
||||
failed_records = 0
|
||||
skipped_records = 0
|
||||
|
||||
for batch_idx, batch_rows in enumerate(batches):
|
||||
batch_result = self._process_batch(
|
||||
job=job,
|
||||
run_id=run.id,
|
||||
batch_index=batch_idx,
|
||||
batch_rows=batch_rows,
|
||||
)
|
||||
successful_records += batch_result["successful"]
|
||||
failed_records += batch_result["failed"]
|
||||
skipped_records += batch_result["skipped"]
|
||||
|
||||
# Update run stats incrementally
|
||||
run.successful_records = successful_records
|
||||
run.failed_records = failed_records
|
||||
run.skipped_records = skipped_records
|
||||
self.db.flush()
|
||||
|
||||
if self.on_batch_progress:
|
||||
self.on_batch_progress(
|
||||
run.id, batch_idx + 1, len(batches),
|
||||
successful_records, total_rows,
|
||||
)
|
||||
|
||||
# Update final run status
|
||||
if failed_records == 0 and skipped_records == 0:
|
||||
run.status = "COMPLETED"
|
||||
elif successful_records == 0:
|
||||
run.status = "FAILED"
|
||||
else:
|
||||
run.status = "COMPLETED" # Partial success
|
||||
|
||||
run.completed_at = datetime.now(timezone.utc)
|
||||
self.db.flush()
|
||||
|
||||
logger.reflect("Translation execution complete", {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"total": total_rows,
|
||||
"successful": successful_records,
|
||||
"failed": failed_records,
|
||||
"skipped": skipped_records,
|
||||
})
|
||||
|
||||
return run
|
||||
# [/DEF:execute_run:Function]
|
||||
|
||||
# [DEF:_fetch_source_rows:Function]
|
||||
# @PURPOSE: Fetch source rows from the accepted preview session for this job.
|
||||
# @PRE: job_id exists.
|
||||
# @POST: Returns list of dicts with source data.
|
||||
def _fetch_source_rows(self, job_id: str, run_id: str) -> List[Dict[str, Any]]:
|
||||
with belief_scope("TranslationExecutor._fetch_source_rows"):
|
||||
# Get the latest APPLIED preview session
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "APPLIED",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
logger.explore("No accepted preview session found", {"job_id": job_id})
|
||||
return []
|
||||
|
||||
# Fetch APPROVED or all records from the session
|
||||
records = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(
|
||||
TranslationPreviewRecord.session_id == session.id,
|
||||
TranslationPreviewRecord.status.in_(["APPROVED", "PENDING"]),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
|
||||
source_rows = []
|
||||
for rec in records:
|
||||
source_rows.append({
|
||||
"row_index": rec.source_object_id or "0",
|
||||
"source_text": rec.source_sql or "",
|
||||
"approved_translation": rec.target_sql if rec.status == "APPROVED" else None,
|
||||
"source_object_name": rec.source_object_name or "",
|
||||
})
|
||||
|
||||
logger.reason(f"Fetched {len(source_rows)} source rows from preview", {
|
||||
"run_id": run_id,
|
||||
"session_id": session.id,
|
||||
})
|
||||
return source_rows
|
||||
# [/DEF:_fetch_source_rows:Function]
|
||||
|
||||
# [DEF:_process_batch:Function]
|
||||
# @PURPOSE: Process a single batch: filter dict, build prompt, call LLM, persist records.
|
||||
# @PRE: job and batch_rows are valid.
|
||||
# @POST: TranslationBatch and TranslationRecord rows are created.
|
||||
# @SIDE_EFFECT: LLM API call.
|
||||
def _process_batch(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
run_id: str,
|
||||
batch_index: int,
|
||||
batch_rows: List[Dict[str, Any]],
|
||||
) -> Dict[str, int]:
|
||||
with belief_scope("TranslationExecutor._process_batch"):
|
||||
batch_start = time.monotonic()
|
||||
|
||||
# Create batch record
|
||||
batch = TranslationBatch(
|
||||
id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
batch_index=batch_index,
|
||||
status="RUNNING",
|
||||
total_records=len(batch_rows),
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
self.db.add(batch)
|
||||
self.db.flush()
|
||||
batch_id = batch.id
|
||||
|
||||
result = {"successful": 0, "failed": 0, "skipped": 0, "retries": 0}
|
||||
|
||||
# Extract source texts for dict filtering
|
||||
source_texts = [
|
||||
row.get("source_text", "")
|
||||
for row in batch_rows
|
||||
if row.get("source_text")
|
||||
]
|
||||
|
||||
# Filter dictionary entries
|
||||
dict_matches = DictionaryManager.filter_for_batch(
|
||||
self.db, source_texts, job.id
|
||||
)
|
||||
|
||||
# For each row, determine if we need LLM translation or can use approved translation
|
||||
rows_for_llm = []
|
||||
pre_translated = []
|
||||
|
||||
for row in batch_rows:
|
||||
if row.get("approved_translation"):
|
||||
pre_translated.append(row)
|
||||
else:
|
||||
rows_for_llm.append(row)
|
||||
|
||||
# Handle pre-translated (approved) rows
|
||||
for row in pre_translated:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=row.get("approved_translation"),
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
result["successful"] += 1
|
||||
|
||||
# Process rows needing LLM translation
|
||||
if rows_for_llm:
|
||||
llm_result = self._call_llm_for_batch(
|
||||
job=job,
|
||||
run_id=run_id,
|
||||
batch_rows=rows_for_llm,
|
||||
dict_matches=dict_matches,
|
||||
batch_id=batch_id,
|
||||
)
|
||||
result["successful"] += llm_result["successful"]
|
||||
result["failed"] += llm_result["failed"]
|
||||
result["skipped"] += llm_result["skipped"]
|
||||
result["retries"] += llm_result.get("retries", 0)
|
||||
|
||||
# Update batch status
|
||||
batch.successful_records = result["successful"]
|
||||
batch.failed_records = result["failed"]
|
||||
batch.completed_at = datetime.now(timezone.utc)
|
||||
batch.status = "COMPLETED" if result["failed"] == 0 else "COMPLETED_WITH_ERRORS"
|
||||
self.db.flush()
|
||||
|
||||
batch_latency = int((time.monotonic() - batch_start) * 1000)
|
||||
logger.reason(f"Batch {batch_index} complete", {
|
||||
"batch_id": batch_id,
|
||||
"latency_ms": batch_latency,
|
||||
**result,
|
||||
})
|
||||
|
||||
return result
|
||||
# [/DEF:_process_batch:Function]
|
||||
|
||||
# [DEF:_call_llm_for_batch:Function]
|
||||
# @PURPOSE: Call LLM for a batch of rows requiring translation. Parse structured JSON response.
|
||||
# @PRE: job has valid provider_id. batch_rows is non-empty.
|
||||
# @POST: Returns dict with successful/failed/skipped counts. Creates TranslationRecord rows.
|
||||
# @SIDE_EFFECT: HTTP call to LLM provider.
|
||||
def _call_llm_for_batch(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
run_id: str,
|
||||
batch_rows: List[Dict[str, Any]],
|
||||
dict_matches: List[Dict[str, Any]],
|
||||
batch_id: str,
|
||||
) -> Dict[str, int]:
|
||||
with belief_scope("TranslationExecutor._call_llm_for_batch"):
|
||||
# Build dictionary section
|
||||
dictionary_section = ""
|
||||
if dict_matches:
|
||||
glossary_lines = []
|
||||
for m in dict_matches:
|
||||
glossary_lines.append(
|
||||
f"- '{m['source_term']}' -> '{m['target_term']}'"
|
||||
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
|
||||
)
|
||||
dictionary_section = (
|
||||
"Terminology dictionary (use these translations when applicable):\n"
|
||||
+ "\n".join(glossary_lines)
|
||||
+ "\n\n"
|
||||
)
|
||||
|
||||
# Build rows JSON for LLM
|
||||
rows_json = json.dumps([
|
||||
{
|
||||
"row_id": str(row.get("row_index", idx)),
|
||||
"text": row.get("source_text", ""),
|
||||
}
|
||||
for idx, row in enumerate(batch_rows)
|
||||
], indent=2)
|
||||
|
||||
# Build prompt
|
||||
prompt = render_prompt(
|
||||
DEFAULT_EXECUTION_PROMPT_TEMPLATE,
|
||||
{
|
||||
"source_language": job.source_dialect or "SQL",
|
||||
"target_language": job.target_language or job.target_dialect or "en",
|
||||
"source_dialect": job.source_dialect or "",
|
||||
"target_dialect": job.target_dialect or "",
|
||||
"translation_column": job.translation_column or "",
|
||||
"dictionary_section": dictionary_section,
|
||||
"rows_json": rows_json,
|
||||
"row_count": str(len(batch_rows)),
|
||||
},
|
||||
)
|
||||
|
||||
# Call LLM with retry
|
||||
llm_response = None
|
||||
last_error = None
|
||||
retries = 0
|
||||
|
||||
for attempt in range(1, MAX_RETRIES_PER_BATCH + 1):
|
||||
try:
|
||||
llm_response = self._call_llm(job, prompt)
|
||||
break
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
retries += 1
|
||||
logger.explore(f"LLM call failed (attempt {attempt})", {
|
||||
"batch_id": batch_id,
|
||||
"error": last_error,
|
||||
"attempt": attempt,
|
||||
})
|
||||
if attempt < MAX_RETRIES_PER_BATCH:
|
||||
time.sleep(2 ** attempt) # Exponential backoff
|
||||
else:
|
||||
logger.explore("LLM call exhausted retries", {
|
||||
"batch_id": batch_id,
|
||||
"last_error": last_error,
|
||||
})
|
||||
|
||||
if llm_response is None:
|
||||
# All retries failed — mark all rows as failed
|
||||
for row in batch_rows:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=None,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="FAILED",
|
||||
error_message=f"LLM call failed after {retries} retries: {last_error}",
|
||||
)
|
||||
self.db.add(record)
|
||||
return {"successful": 0, "failed": len(batch_rows), "skipped": 0, "retries": retries}
|
||||
|
||||
# Parse LLM response
|
||||
try:
|
||||
translations = self._parse_llm_response(llm_response, len(batch_rows))
|
||||
except ValueError as e:
|
||||
# Parse failure — mark all rows as SKIPPED
|
||||
logger.explore("LLM response parse failed", {
|
||||
"batch_id": batch_id,
|
||||
"error": str(e),
|
||||
"response_preview": llm_response[:500] if llm_response else "",
|
||||
})
|
||||
skipped = len(batch_rows)
|
||||
for row in batch_rows:
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=None,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message=f"LLM parse failure: {e}",
|
||||
)
|
||||
self.db.add(record)
|
||||
return {
|
||||
"successful": 0,
|
||||
"failed": 0,
|
||||
"skipped": skipped,
|
||||
"retries": retries,
|
||||
}
|
||||
|
||||
successful = 0
|
||||
failed = 0
|
||||
skipped = 0
|
||||
|
||||
for row in batch_rows:
|
||||
row_id = str(row.get("row_index", ""))
|
||||
translation = translations.get(row_id)
|
||||
|
||||
if translation is None:
|
||||
# NULL translation — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message="NULL translation returned by LLM",
|
||||
)
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
if translation.strip() == "":
|
||||
# Empty translation — skip
|
||||
skipped += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql="",
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SKIPPED",
|
||||
error_message="Empty translation returned by LLM",
|
||||
)
|
||||
self.db.add(record)
|
||||
continue
|
||||
|
||||
successful += 1
|
||||
record = TranslationRecord(
|
||||
id=str(uuid.uuid4()),
|
||||
batch_id=batch_id,
|
||||
run_id=run_id,
|
||||
source_sql=row.get("source_text", ""),
|
||||
target_sql=translation,
|
||||
source_object_type="table_row",
|
||||
source_object_id=row.get("row_index"),
|
||||
source_object_name=row.get("source_object_name", ""),
|
||||
status="SUCCESS",
|
||||
)
|
||||
self.db.add(record)
|
||||
|
||||
return {
|
||||
"successful": successful,
|
||||
"failed": failed,
|
||||
"skipped": skipped,
|
||||
"retries": retries,
|
||||
}
|
||||
# [/DEF:_call_llm_for_batch:Function]
|
||||
|
||||
# [DEF:_call_llm:Function]
|
||||
# @PURPOSE: Call the configured LLM provider with the batch prompt.
|
||||
# @PRE: job has valid provider_id.
|
||||
# @POST: Returns raw LLM response string.
|
||||
# @SIDE_EFFECT: HTTP call to LLM provider.
|
||||
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
|
||||
with belief_scope("TranslationExecutor._call_llm"):
|
||||
if not job.provider_id:
|
||||
raise ValueError("Job has no LLM provider configured")
|
||||
|
||||
provider_svc = LLMProviderService(self.db)
|
||||
provider = provider_svc.get_provider(job.provider_id)
|
||||
if not provider:
|
||||
raise ValueError(f"LLM provider '{job.provider_id}' not found")
|
||||
|
||||
api_key = provider_svc.get_decrypted_api_key(job.provider_id)
|
||||
if not api_key:
|
||||
raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'")
|
||||
|
||||
model = provider.default_model or "gpt-4o-mini"
|
||||
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
|
||||
|
||||
if provider_type in ("openai", "openai_compatible"):
|
||||
return self._call_openai_compatible(
|
||||
base_url=provider.base_url,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported provider type '{provider_type}'")
|
||||
# [/DEF:_call_llm:Function]
|
||||
|
||||
# [DEF:_call_openai_compatible:Function]
|
||||
# @PURPOSE: Call OpenAI-compatible API for batch translation.
|
||||
# @PRE: Valid API endpoint, key, model, and prompt.
|
||||
# @POST: Returns response text.
|
||||
# @SIDE_EFFECT: HTTP POST to LLM API.
|
||||
@staticmethod
|
||||
def _call_openai_compatible(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
) -> str:
|
||||
with belief_scope("TranslationExecutor._call_openai_compatible"):
|
||||
import requests as http_requests
|
||||
|
||||
url = f"{base_url.rstrip('/')}/chat/completions"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
|
||||
response = http_requests.post(url, headers=headers, json=payload, timeout=180)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
choices = data.get("choices", [])
|
||||
if not choices:
|
||||
raise ValueError("LLM returned no choices")
|
||||
|
||||
content = choices[0].get("message", {}).get("content", "")
|
||||
if not content:
|
||||
raise ValueError("LLM returned empty content")
|
||||
|
||||
return content
|
||||
# [/DEF:_call_openai_compatible:Function]
|
||||
|
||||
# [DEF:_parse_llm_response:Function]
|
||||
# @PURPOSE: Parse LLM JSON response into dict of row_id -> translation.
|
||||
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
||||
# @POST: Returns dict mapping row_id to translation text.
|
||||
@staticmethod
|
||||
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
|
||||
with belief_scope("TranslationExecutor._parse_llm_response"):
|
||||
try:
|
||||
data = json.loads(response_text)
|
||||
except json.JSONDecodeError:
|
||||
# Try to extract from markdown code block
|
||||
import re
|
||||
match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except json.JSONDecodeError:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
else:
|
||||
raise ValueError("LLM response was not valid JSON")
|
||||
|
||||
rows = data.get("rows", [])
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError("LLM response missing 'rows' array")
|
||||
|
||||
translations: Dict[str, str] = {}
|
||||
for item in rows:
|
||||
row_id = str(item.get("row_id", ""))
|
||||
translation = item.get("translation")
|
||||
if translation is None:
|
||||
# Skip NULL translations — they'll be handled by caller
|
||||
continue
|
||||
if row_id:
|
||||
translations[row_id] = str(translation)
|
||||
|
||||
return translations
|
||||
# [/DEF:_parse_llm_response:Function]
|
||||
|
||||
|
||||
# [/DEF:TranslationExecutor:Class]
|
||||
# [/DEF:TranslationExecutor:Module]
|
||||
Reference in New Issue
Block a user