busya
306c5ae742
- Convert legacy [DEF🆔Type] anchors to #region/#endregion across 329 files
- Reinstate _normalize_timestamp_value in sql_generator.py
- Fix MarkerLogger→logger migration in events.py (molecular CoT markers)
- Fix dataset_review orchestrator dependencies (_build_execution_snapshot)
- Fix config_manager stale-record deletion (moved to save path only)
- Add 77 missing [/DEF:] closers in 5 unbalanced test files
- Update assistant_chat.integration.test.js for #region format
- Apply molecular-cot-logging markers (REASON/REFLECT/EXPLORE) via logger.* methods
822 lines
35 KiB
Python
822 lines
35 KiB
Python
# #region TranslationPreview [C:4] [TYPE Module] [SEMANTICS translate, preview, llm, session]
|
|
# @BRIEF Preview session management: fetch sample rows from Superset, send to LLM with context + filtered dictionary, return side-by-side results.
|
|
# @LAYER: Domain
|
|
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
|
|
# @RELATION DEPENDS_ON -> [TranslationPreviewSession:Class]
|
|
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord:Class]
|
|
# @RELATION DEPENDS_ON -> [LLMProviderService]
|
|
# @RELATION DEPENDS_ON -> [DictionaryManager]
|
|
# @RELATION DEPENDS_ON -> [SupersetClient]
|
|
# @RELATION DEPENDS_ON -> [render_prompt]
|
|
# @RELATION DEPENDS_ON -> [ConfigManager]
|
|
# @PRE: Database session and config manager are available.
|
|
# @POST: Preview sessions are created with LLM-translated rows; records can be approved/edited/rejected.
|
|
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
|
# @RATIONALE: C4 because preview is stateful with approve/edit/reject lifecycle and LLM API calls.
|
|
# @REJECTED: Transient preview state (in-memory/session-scoped) — would lose decisions on restart and cannot gate execution reliably.
|
|
|
|
from typing import Any, Dict, List, Optional, Tuple
|
|
from sqlalchemy.orm import Session
|
|
from datetime import datetime, timezone, timedelta
|
|
import uuid
|
|
import json
|
|
import hashlib
|
|
import re
|
|
|
|
from ...core.logger import logger, belief_scope
|
|
from ...core.config_manager import ConfigManager
|
|
from ...core.superset_client import SupersetClient
|
|
from ...models.translate import (
|
|
TranslationJob,
|
|
TranslationPreviewSession,
|
|
TranslationPreviewRecord,
|
|
TranslationJobDictionary,
|
|
)
|
|
from ...services.llm_provider import LLMProviderService
|
|
from ...services.llm_prompt_templates import render_prompt
|
|
from .dictionary import DictionaryManager
|
|
|
|
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
|
|
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
|
|
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
|
|
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
|
"Source dialect: {source_dialect}\n"
|
|
"Target dialect: {target_dialect}\n"
|
|
"Column to translate: {translation_column}\n\n"
|
|
"{dictionary_section}"
|
|
"For each row, provide an accurate translation of the text.\n\n"
|
|
"Rows to translate:\n{rows_json}\n\n"
|
|
"Respond with a JSON object in this exact format:\n"
|
|
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
|
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
|
)
|
|
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
|
|
|
|
|
|
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
|
|
# @BRIEF Default prompt template for LLM translation preview.
|
|
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
|
|
"Translate the following database content from {source_language} to {target_language}.\n\n"
|
|
"Source dialect: {source_dialect}\n"
|
|
"Target dialect: {target_dialect}\n"
|
|
"Column to translate: {translation_column}\n\n"
|
|
"{dictionary_section}"
|
|
"For each row, provide an accurate translation of the '{translation_column}' value.\n"
|
|
"Consider the context columns when determining the meaning of the text.\n\n"
|
|
"Rows to translate:\n{rows_json}\n\n"
|
|
"Respond with a JSON object in this exact format:\n"
|
|
'{{"rows": [{{"row_id": "<row_index>", "translation": "<translated_text>"}}]}}\n'
|
|
"Each row_id must match the index provided. Return exactly {row_count} entries."
|
|
)
|
|
# #endregion DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
|
|
|
|
|
# #region TokenEstimator [TYPE Class]
|
|
# @BRIEF Estimate token counts and costs for LLM translation operations.
|
|
# @RATIONALE: Token estimation uses a heuristic (chars/token ratio) since exact tokenization depends on the LLM model.
|
|
# @REJECTED: Using an external tokenizer library would introduce a heavy dependency for estimation only.
|
|
class TokenEstimator:
|
|
"""Estimate token counts and costs for LLM operations."""
|
|
|
|
CHARS_PER_TOKEN_ESTIMATE: float = 4.0
|
|
OUTPUT_TOKENS_PER_ROW_ESTIMATE: int = 50
|
|
TOKEN_COST_PER_1K: float = 0.002 # Default cost per 1K tokens
|
|
|
|
# [DEF:estimate_prompt_tokens:Function]
|
|
# @PURPOSE: Estimate token count for a prompt string.
|
|
# @PRE: prompt is a non-empty string.
|
|
# @POST: Returns estimated token count (integer).
|
|
@staticmethod
|
|
def estimate_prompt_tokens(prompt: str) -> int:
|
|
if not prompt:
|
|
return 0
|
|
return max(1, int(len(prompt) / TokenEstimator.CHARS_PER_TOKEN_ESTIMATE))
|
|
# [/DEF:estimate_prompt_tokens:Function]
|
|
|
|
# [DEF:estimate_output_tokens:Function]
|
|
# @PURPOSE: Estimate output token count for translating N rows.
|
|
# @PRE: row_count >= 0.
|
|
# @POST: Returns estimated output token count.
|
|
@staticmethod
|
|
def estimate_output_tokens(row_count: int) -> int:
|
|
return row_count * TokenEstimator.OUTPUT_TOKENS_PER_ROW_ESTIMATE
|
|
# [/DEF:estimate_output_tokens:Function]
|
|
|
|
# [DEF:estimate_cost:Function]
|
|
# @PURPOSE: Estimate cost for a given number of tokens.
|
|
# @PRE: total_tokens >= 0.
|
|
# @POST: Returns estimated cost in USD.
|
|
@staticmethod
|
|
def estimate_cost(total_tokens: int, cost_per_1k: Optional[float] = None) -> float:
|
|
rate = cost_per_1k if cost_per_1k is not None else TokenEstimator.TOKEN_COST_PER_1K
|
|
return round((total_tokens / 1000) * rate, 6)
|
|
# [/DEF:estimate_cost:Function]
|
|
|
|
|
|
# #endregion TokenEstimator
|
|
|
|
|
|
# #region TranslationPreview [C:4] [TYPE Class]
|
|
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
|
|
# @PRE: Database session and config manager are available.
|
|
# @POST: Preview sessions created with persisted records; full execution gates on accepted session.
|
|
# @SIDE_EFFECT: Fetches sample data from Superset; calls LLM provider; creates DB rows.
|
|
class TranslationPreview:
|
|
|
|
def __init__(
|
|
self,
|
|
db: Session,
|
|
config_manager: ConfigManager,
|
|
current_user: Optional[str] = None,
|
|
):
|
|
self.db = db
|
|
self.config_manager = config_manager
|
|
self.current_user = current_user
|
|
|
|
# [DEF:preview_rows:Function]
|
|
# @COMPLEXITY: 4
|
|
# @PURPOSE: Fetch sample rows from Superset dataset, send to LLM for translation, create preview session with records.
|
|
# @PRE: job_id exists and job has source_datasource_id, translation_column configured.
|
|
# @POST: Returns TranslationPreviewResponse with records, cost estimation, and persistent session.
|
|
# @SIDE_EFFECT: Fetches data from Superset; calls LLM; creates TranslationPreviewSession and TranslationPreviewRecord rows.
|
|
def preview_rows(
|
|
self,
|
|
job_id: str,
|
|
sample_size: int = 10,
|
|
prompt_template: Optional[str] = None,
|
|
env_id: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
with belief_scope("TranslationPreview.preview_rows"):
|
|
logger.reason("Starting preview for job", {"job_id": job_id, "sample_size": sample_size})
|
|
|
|
# 1. Load job
|
|
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
|
if not job:
|
|
raise ValueError(f"Translation job '{job_id}' not found")
|
|
if not job.source_datasource_id:
|
|
raise ValueError("Job must have a source datasource configured for preview")
|
|
if not job.translation_column:
|
|
raise ValueError("Job must have a translation column configured for preview")
|
|
|
|
# 2. Compute config hash and dict snapshot hash
|
|
config_hash = self._compute_config_hash(job)
|
|
dict_snapshot_hash = self._compute_dict_snapshot_hash(job_id)
|
|
|
|
# 3. Fetch sample rows from Superset
|
|
logger.reason("Fetching sample rows from Superset", {
|
|
"datasource_id": job.source_datasource_id,
|
|
"sample_size": sample_size,
|
|
"translation_column": job.translation_column,
|
|
})
|
|
source_rows = self._fetch_sample_rows(
|
|
job=job,
|
|
sample_size=sample_size,
|
|
env_id=env_id,
|
|
)
|
|
if not source_rows:
|
|
raise ValueError("No rows returned from datasource for preview")
|
|
|
|
actual_row_count = len(source_rows)
|
|
logger.reason(f"Fetched {actual_row_count} sample row(s)")
|
|
|
|
# Debug: log first row keys and translation column value
|
|
if source_rows:
|
|
first_row = source_rows[0]
|
|
logger.reason(
|
|
f"First source row keys={list(first_row.keys())} "
|
|
f"translation_col={job.translation_column} "
|
|
f"val='{first_row.get(job.translation_column, '')}'"
|
|
)
|
|
|
|
# 4. Build prompt context from rows
|
|
all_source_texts = []
|
|
row_meta: List[Dict[str, Any]] = []
|
|
for idx, row in enumerate(source_rows):
|
|
translation_value = str(row.get(job.translation_column, "") or "")
|
|
context_values = {}
|
|
if job.context_columns:
|
|
for col in job.context_columns:
|
|
context_values[col] = str(row.get(col, "") or "")
|
|
all_source_texts.append(translation_value)
|
|
row_meta.append({
|
|
"row_index": idx,
|
|
"source_text": translation_value,
|
|
"context_data": context_values,
|
|
"source_row": row,
|
|
})
|
|
|
|
# 5. Filter dictionary entries for this batch
|
|
dict_matches = DictionaryManager.filter_for_batch(
|
|
self.db, all_source_texts, job_id
|
|
)
|
|
|
|
# Build dictionary glossary section
|
|
dictionary_section = ""
|
|
if dict_matches:
|
|
glossary_lines = []
|
|
for m in dict_matches:
|
|
glossary_lines.append(
|
|
f"- '{m['source_term']}' -> '{m['target_term']}'"
|
|
f"{' (' + m['context_notes'] + ')' if m.get('context_notes') else ''}"
|
|
)
|
|
dictionary_section = (
|
|
"Terminology dictionary (use these translations when applicable):\n"
|
|
+ "\n".join(glossary_lines)
|
|
+ "\n\n"
|
|
)
|
|
|
|
# 6. Build LLM prompt
|
|
rows_json = json.dumps([
|
|
{"row_id": str(m["row_index"]), "text": m["source_text"], "context": m["context_data"]}
|
|
for m in row_meta
|
|
], indent=2)
|
|
|
|
template = prompt_template or DEFAULT_PREVIEW_PROMPT_TEMPLATE
|
|
prompt = render_prompt(template, {
|
|
"source_language": job.source_dialect or "SQL",
|
|
"target_language": job.target_language or job.target_dialect or "en",
|
|
"source_dialect": job.source_dialect or "",
|
|
"target_dialect": job.target_dialect or "",
|
|
"translation_column": job.translation_column or "",
|
|
"dictionary_section": dictionary_section,
|
|
"rows_json": rows_json,
|
|
"row_count": str(actual_row_count),
|
|
})
|
|
|
|
# 7. Estimate tokens/cost for sample and full dataset
|
|
sample_prompt_tokens = TokenEstimator.estimate_prompt_tokens(prompt)
|
|
sample_output_tokens = TokenEstimator.estimate_output_tokens(actual_row_count)
|
|
sample_total_tokens = sample_prompt_tokens + sample_output_tokens
|
|
sample_cost = TokenEstimator.estimate_cost(sample_total_tokens)
|
|
|
|
# Estimate full dataset cost (if we knew total rows)
|
|
total_est_tokens = TokenEstimator.estimate_prompt_tokens(
|
|
prompt.replace(str(actual_row_count), "{total}")
|
|
) + TokenEstimator.estimate_output_tokens(sample_size * 10) # rough extrapolation
|
|
total_est_cost = TokenEstimator.estimate_cost(total_est_tokens)
|
|
|
|
# 8. Call LLM
|
|
logger.reason("Calling LLM for preview translation", {
|
|
"provider_id": job.provider_id,
|
|
"row_count": actual_row_count,
|
|
"estimated_tokens": sample_total_tokens,
|
|
})
|
|
llm_response = self._call_llm(
|
|
job=job,
|
|
prompt=prompt,
|
|
)
|
|
|
|
# 9. Parse LLM response
|
|
translations = self._parse_llm_response(llm_response, actual_row_count)
|
|
|
|
# 10. Create preview session
|
|
session = TranslationPreviewSession(
|
|
id=str(uuid.uuid4()),
|
|
job_id=job_id,
|
|
status="ACTIVE",
|
|
created_by=self.current_user,
|
|
created_at=datetime.now(timezone.utc),
|
|
expires_at=datetime.now(timezone.utc) + timedelta(hours=24),
|
|
)
|
|
self.db.add(session)
|
|
self.db.flush()
|
|
|
|
# 11. Create preview records
|
|
records = []
|
|
for meta in row_meta:
|
|
idx = meta["row_index"]
|
|
translation = translations.get(str(idx), "")
|
|
is_rejected = False
|
|
status = "PENDING"
|
|
feedback = None
|
|
|
|
record = TranslationPreviewRecord(
|
|
id=str(uuid.uuid4()),
|
|
session_id=session.id,
|
|
source_sql=meta["source_text"],
|
|
target_sql=translation,
|
|
source_object_type="table_row",
|
|
source_object_id=str(idx),
|
|
source_object_name=f"Row {idx + 1}",
|
|
status=status,
|
|
feedback=feedback,
|
|
created_at=datetime.now(timezone.utc),
|
|
)
|
|
self.db.add(record)
|
|
self.db.flush()
|
|
records.append({
|
|
"id": record.id,
|
|
"source_sql": record.source_sql,
|
|
"target_sql": record.target_sql,
|
|
"source_object_type": record.source_object_type,
|
|
"source_object_id": record.source_object_id,
|
|
"source_object_name": record.source_object_name,
|
|
"status": record.status,
|
|
"feedback": record.feedback,
|
|
})
|
|
|
|
self.db.commit()
|
|
|
|
result = {
|
|
"id": session.id,
|
|
"job_id": job_id,
|
|
"status": "ACTIVE",
|
|
"created_by": self.current_user,
|
|
"created_at": session.created_at.isoformat(),
|
|
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
|
"records": records,
|
|
"cost_estimate": {
|
|
"sample_size": actual_row_count,
|
|
"sample_prompt_tokens": sample_prompt_tokens,
|
|
"sample_output_tokens": sample_output_tokens,
|
|
"sample_total_tokens": sample_total_tokens,
|
|
"sample_cost": sample_cost,
|
|
"estimated_total_rows": actual_row_count * 10,
|
|
"estimated_tokens": total_est_tokens,
|
|
"estimated_cost": total_est_cost,
|
|
},
|
|
"config_hash": config_hash,
|
|
"dict_snapshot_hash": dict_snapshot_hash,
|
|
}
|
|
|
|
logger.reflect("Preview completed", {
|
|
"session_id": session.id,
|
|
"row_count": actual_row_count,
|
|
"sample_cost": sample_cost,
|
|
})
|
|
return result
|
|
# [/DEF:preview_rows:Function]
|
|
|
|
# [DEF:update_preview_row:Function]
|
|
# @PURPOSE: Approve, edit, or reject an individual preview row.
|
|
# @PRE: session_id and row_id exist, session is ACTIVE.
|
|
# @POST: PreviewRecord status is updated.
|
|
def update_preview_row(
|
|
self,
|
|
job_id: str,
|
|
row_id: str,
|
|
action: str,
|
|
translation: Optional[str] = None,
|
|
feedback: Optional[str] = None,
|
|
) -> Dict[str, Any]:
|
|
with belief_scope("TranslationPreview.update_preview_row"):
|
|
# Find the active session for this job
|
|
session = (
|
|
self.db.query(TranslationPreviewSession)
|
|
.filter(
|
|
TranslationPreviewSession.job_id == job_id,
|
|
TranslationPreviewSession.status == "ACTIVE",
|
|
)
|
|
.order_by(TranslationPreviewSession.created_at.desc())
|
|
.first()
|
|
)
|
|
if not session:
|
|
raise ValueError(f"No active preview session for job '{job_id}'")
|
|
|
|
record = (
|
|
self.db.query(TranslationPreviewRecord)
|
|
.filter(
|
|
TranslationPreviewRecord.id == row_id,
|
|
TranslationPreviewRecord.session_id == session.id,
|
|
)
|
|
.first()
|
|
)
|
|
if not record:
|
|
raise ValueError(f"Preview record '{row_id}' not found in active session")
|
|
|
|
if action == "approve":
|
|
record.status = "APPROVED"
|
|
elif action == "reject":
|
|
record.status = "REJECTED"
|
|
elif action == "edit":
|
|
record.status = "APPROVED"
|
|
if translation is not None:
|
|
record.target_sql = translation
|
|
else:
|
|
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
|
|
|
if feedback is not None:
|
|
record.feedback = feedback
|
|
|
|
self.db.commit()
|
|
self.db.refresh(record)
|
|
|
|
logger.reason(f"Preview row {action}d", {
|
|
"row_id": row_id,
|
|
"session_id": session.id,
|
|
"status": record.status,
|
|
})
|
|
|
|
return {
|
|
"id": record.id,
|
|
"source_sql": record.source_sql,
|
|
"target_sql": record.target_sql,
|
|
"status": record.status,
|
|
"feedback": record.feedback,
|
|
}
|
|
# [/DEF:update_preview_row:Function]
|
|
|
|
# [DEF:accept_preview_session:Function]
|
|
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
|
|
# @PRE: job_id has an ACTIVE preview session.
|
|
# @POST: Session status changes to APPLIED.
|
|
# @SIDE_EFFECT: Future full execution calls will check for accepted session.
|
|
def accept_preview_session(self, job_id: str) -> Dict[str, Any]:
|
|
with belief_scope("TranslationPreview.accept_preview_session"):
|
|
session = (
|
|
self.db.query(TranslationPreviewSession)
|
|
.filter(
|
|
TranslationPreviewSession.job_id == job_id,
|
|
TranslationPreviewSession.status == "ACTIVE",
|
|
)
|
|
.order_by(TranslationPreviewSession.created_at.desc())
|
|
.first()
|
|
)
|
|
if not session:
|
|
raise ValueError(f"No active preview session for job '{job_id}'")
|
|
|
|
session.status = "APPLIED"
|
|
self.db.commit()
|
|
self.db.refresh(session)
|
|
|
|
logger.reason("Preview session accepted", {
|
|
"session_id": session.id,
|
|
"job_id": job_id,
|
|
})
|
|
|
|
# Get records for response
|
|
records = (
|
|
self.db.query(TranslationPreviewRecord)
|
|
.filter(TranslationPreviewRecord.session_id == session.id)
|
|
.all()
|
|
)
|
|
|
|
return {
|
|
"id": session.id,
|
|
"job_id": job_id,
|
|
"status": "APPLIED",
|
|
"created_by": session.created_by,
|
|
"created_at": session.created_at.isoformat(),
|
|
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
|
"records": [
|
|
{
|
|
"id": r.id,
|
|
"source_sql": r.source_sql,
|
|
"target_sql": r.target_sql,
|
|
"status": r.status,
|
|
"feedback": r.feedback,
|
|
}
|
|
for r in records
|
|
],
|
|
}
|
|
# [/DEF:accept_preview_session:Function]
|
|
|
|
# [DEF:get_preview_session:Function]
|
|
# @PURPOSE: Get the latest preview session for a job with its records.
|
|
# @PRE: job_id exists.
|
|
# @POST: Returns session data with records or raises ValueError.
|
|
def get_preview_session(self, job_id: str) -> Dict[str, Any]:
|
|
with belief_scope("TranslationPreview.get_preview_session"):
|
|
session = (
|
|
self.db.query(TranslationPreviewSession)
|
|
.filter(TranslationPreviewSession.job_id == job_id)
|
|
.order_by(TranslationPreviewSession.created_at.desc())
|
|
.first()
|
|
)
|
|
if not session:
|
|
raise ValueError(f"No preview session found for job '{job_id}'")
|
|
|
|
records = (
|
|
self.db.query(TranslationPreviewRecord)
|
|
.filter(TranslationPreviewRecord.session_id == session.id)
|
|
.all()
|
|
)
|
|
|
|
return {
|
|
"id": session.id,
|
|
"job_id": job_id,
|
|
"status": session.status,
|
|
"created_by": session.created_by,
|
|
"created_at": session.created_at.isoformat(),
|
|
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
|
"records": [
|
|
{
|
|
"id": r.id,
|
|
"source_sql": r.source_sql,
|
|
"target_sql": r.target_sql,
|
|
"source_object_type": r.source_object_type,
|
|
"source_object_id": r.source_object_id,
|
|
"source_object_name": r.source_object_name,
|
|
"status": r.status,
|
|
"feedback": r.feedback,
|
|
}
|
|
for r in records
|
|
],
|
|
}
|
|
# [/DEF:get_preview_session:Function]
|
|
|
|
# [DEF:_fetch_sample_rows:Function]
|
|
# @COMPLEXITY: 4
|
|
# @PURPOSE: Fetch sample rows from the Superset dataset for preview.
|
|
# @PRE: job has source_datasource_id and translation_column.
|
|
# @POST: Returns list of dicts with row data.
|
|
# @SIDE_EFFECT: Calls Superset chart data endpoint.
|
|
def _fetch_sample_rows(self, job: TranslationJob, sample_size: int = 10, env_id: Optional[str] = None) -> List[Dict[str, Any]]:
|
|
with belief_scope("TranslationPreview._fetch_sample_rows"):
|
|
# Determine environment: prefer explicit env_id, then job.environment_id, then job.source_dialect (legacy)
|
|
environments = self.config_manager.get_environments()
|
|
target_env_id = env_id or job.environment_id or job.source_dialect or ""
|
|
env_config = next(
|
|
(e for e in environments if e.id == target_env_id),
|
|
None,
|
|
)
|
|
if not env_config:
|
|
logger.explore("Could not find environment for datasource", {
|
|
"env_id": target_env_id,
|
|
})
|
|
# Fallback: try first environment
|
|
if environments:
|
|
env_config = environments[0]
|
|
logger.explore("Falling back to first available environment", {
|
|
"env_id": env_config.id,
|
|
})
|
|
else:
|
|
raise ValueError("No Superset environments configured")
|
|
|
|
client = SupersetClient(env_config)
|
|
|
|
# Fetch dataset detail to build proper query context
|
|
dataset_detail = client.get_dataset_detail(int(job.source_datasource_id))
|
|
|
|
# Build query context for chart data endpoint.
|
|
# Virtual columns (e.g. comment_text_ru) are NOT resolved when:
|
|
# - result_type="query" (physical columns only)
|
|
# - query_mode="raw" (virtual columns unavailable in raw mode)
|
|
# Solution: remove both result_type="query" AND query_mode="raw",
|
|
# use aggregate mode with no metrics — this resolves virtual columns.
|
|
query_context = client.build_dataset_preview_query_context(
|
|
dataset_id=int(job.source_datasource_id),
|
|
dataset_record=dataset_detail,
|
|
template_params={},
|
|
effective_filters=[],
|
|
)
|
|
|
|
# Modify: use result_type="samples" which returns sample data
|
|
# including all columns (physical + virtual), without needing
|
|
# explicit column objects that trigger validation errors.
|
|
queries = query_context.get("queries", [])
|
|
if queries:
|
|
queries[0]["row_limit"] = sample_size
|
|
queries[0].pop("result_type", None)
|
|
queries[0].pop("columns", None)
|
|
queries[0]["metrics"] = []
|
|
query_context["result_type"] = "samples"
|
|
form_data = query_context.get("form_data", {})
|
|
form_data.pop("query_mode", None)
|
|
|
|
try:
|
|
response = client.network.request(
|
|
method="POST",
|
|
endpoint="/api/v1/chart/data",
|
|
data=json.dumps(query_context),
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
except Exception as e:
|
|
logger.explore("Chart data API failed", {"error": str(e)})
|
|
raise ValueError(f"Failed to fetch sample data from Superset: {e}")
|
|
|
|
# Parse response
|
|
rows = self._extract_data_rows(response)
|
|
logger.reason(f"Extracted {len(rows)} data row(s)")
|
|
|
|
# Debug: log first row keys and translation column value
|
|
if rows:
|
|
first_row = rows[0]
|
|
logger.reason(
|
|
f"Row keys={list(first_row.keys())} "
|
|
f"target_col={job.translation_column} "
|
|
f"val='{first_row.get(job.translation_column, '')}'"
|
|
)
|
|
|
|
return rows
|
|
# [/DEF:_fetch_sample_rows:Function]
|
|
|
|
# [DEF:_extract_data_rows:Function]
|
|
# @PURPOSE: Extract data rows from Superset chart data response.
|
|
# @PRE: response is a dict from Superset API.
|
|
# @POST: Returns list of row dicts.
|
|
@staticmethod
|
|
def _extract_data_rows(response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
with belief_scope("TranslationPreview._extract_data_rows"):
|
|
# Try various response formats
|
|
result = response.get("result")
|
|
if isinstance(result, list):
|
|
for item in result:
|
|
if isinstance(item, dict):
|
|
data = item.get("data")
|
|
if isinstance(data, list) and data:
|
|
return data
|
|
|
|
# Try flat result
|
|
if isinstance(result, dict):
|
|
data = result.get("data")
|
|
if isinstance(data, list) and data:
|
|
return data
|
|
|
|
# Legacy: response may have data at top level
|
|
data = response.get("data")
|
|
if isinstance(data, list) and data:
|
|
return data
|
|
|
|
# Last resort: return response itself wrapped if it looks like a list of rows
|
|
if isinstance(result, list):
|
|
return result
|
|
|
|
return []
|
|
# [/DEF:_extract_data_rows:Function]
|
|
|
|
# [DEF:_call_llm:Function]
|
|
# @COMPLEXITY: 4
|
|
# @PURPOSE: Call the configured LLM provider with the preview prompt.
|
|
# @PRE: job has a valid provider_id.
|
|
# @POST: Returns raw LLM response string.
|
|
# @SIDE_EFFECT: Makes HTTP call to LLM provider.
|
|
def _call_llm(self, job: TranslationJob, prompt: str) -> str:
|
|
with belief_scope("TranslationPreview._call_llm"):
|
|
if not job.provider_id:
|
|
raise ValueError("Job has no LLM provider configured")
|
|
|
|
provider_svc = LLMProviderService(self.db)
|
|
provider = provider_svc.get_provider(job.provider_id)
|
|
if not provider:
|
|
raise ValueError(f"LLM provider '{job.provider_id}' not found")
|
|
|
|
api_key = provider_svc.get_decrypted_api_key(job.provider_id)
|
|
if not api_key:
|
|
raise ValueError(f"Could not decrypt API key for provider '{job.provider_id}'")
|
|
|
|
# Build the API call based on provider type
|
|
model = provider.default_model or "gpt-4o-mini"
|
|
provider_type = provider.provider_type.lower() if provider.provider_type else "openai"
|
|
|
|
if provider_type in ("openai", "openai_compatible", "openrouter", "kilo"):
|
|
response_text = self._call_openai_compatible(
|
|
base_url=provider.base_url,
|
|
api_key=api_key,
|
|
model=model,
|
|
prompt=prompt,
|
|
provider_type=provider_type,
|
|
)
|
|
else:
|
|
raise ValueError(f"Unsupported provider type '{provider_type}' for preview")
|
|
|
|
logger.reason("LLM call completed", {
|
|
"provider_id": job.provider_id,
|
|
"model": model,
|
|
"response_length": len(response_text),
|
|
})
|
|
return response_text
|
|
# [/DEF:_call_llm:Function]
|
|
|
|
# [DEF:_call_openai_compatible:Function]
|
|
# @PURPOSE: Call an OpenAI-compatible API for translation.
|
|
# @PRE: base_url, api_key, model, prompt are valid.
|
|
# @POST: Returns response text.
|
|
# @SIDE_EFFECT: Makes HTTP POST to LLM API.
|
|
@staticmethod
|
|
def _call_openai_compatible(
|
|
base_url: str,
|
|
api_key: str,
|
|
model: str,
|
|
prompt: str,
|
|
provider_type: str = "openai",
|
|
) -> str:
|
|
with belief_scope("TranslationPreview._call_openai_compatible"):
|
|
import requests as http_requests
|
|
|
|
url = f"{base_url.rstrip('/')}/chat/completions"
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
}
|
|
payload = {
|
|
"model": model,
|
|
"messages": [
|
|
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
|
|
{"role": "user", "content": prompt},
|
|
],
|
|
"temperature": 0.1,
|
|
"max_tokens": 4096,
|
|
}
|
|
# Structured output (response_format) only for native OpenAI — upstream providers routed via
|
|
# Kilo/OpenRouter may not support it (e.g. StepFun returns "structured_outputs is not supported")
|
|
if provider_type in ("openai", "openai_compatible"):
|
|
payload["response_format"] = {"type": "json_object"}
|
|
|
|
logger.reason(
|
|
f"LLM request model={payload.get('model')} "
|
|
f"provider_type={provider_type} "
|
|
f"response_format={'yes' if 'response_format' in payload else 'no'} "
|
|
f"prompt_len={len(prompt)}"
|
|
)
|
|
response = http_requests.post(url, headers=headers, json=payload, timeout=120)
|
|
if not response.ok:
|
|
logger.explore(
|
|
f"LLM API error status={response.status_code} "
|
|
f"model={payload.get('model')} "
|
|
f"body={response.text[:2000]}"
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
choices = data.get("choices", [])
|
|
if not choices:
|
|
raise ValueError("LLM returned no choices")
|
|
|
|
content = choices[0].get("message", {}).get("content", "")
|
|
if not content:
|
|
raise ValueError("LLM returned empty content")
|
|
|
|
return content
|
|
# [/DEF:_call_openai_compatible:Function]
|
|
|
|
# [DEF:_parse_llm_response:Function]
|
|
# @PURPOSE: Parse the LLM JSON response into a dict of row_id -> translation.
|
|
# @PRE: response_text is valid JSON with {"rows": [...]} structure.
|
|
# @POST: Returns dict mapping string row_id to translation text.
|
|
@staticmethod
|
|
def _parse_llm_response(response_text: str, expected_count: int) -> Dict[str, str]:
|
|
with belief_scope("TranslationPreview._parse_llm_response"):
|
|
logger.reason(f"Raw LLM response length={len(response_text)} preview={response_text[:500]}")
|
|
|
|
try:
|
|
data = json.loads(response_text)
|
|
except json.JSONDecodeError:
|
|
# Try to extract JSON from markdown code block
|
|
import re
|
|
match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', response_text, re.DOTALL)
|
|
if match:
|
|
try:
|
|
data = json.loads(match.group(1))
|
|
except json.JSONDecodeError:
|
|
raise ValueError("LLM response was not valid JSON")
|
|
else:
|
|
raise ValueError("LLM response was not valid JSON")
|
|
|
|
rows = data.get("rows", [])
|
|
if not isinstance(rows, list):
|
|
logger.explore(f"LLM response has no 'rows' array, keys={list(data.keys())} text_preview={response_text[:300]}")
|
|
raise ValueError("LLM response missing 'rows' array")
|
|
|
|
translations: Dict[str, str] = {}
|
|
for item in rows:
|
|
row_id = str(item.get("row_id", ""))
|
|
translation = str(item.get("translation", ""))
|
|
if row_id:
|
|
translations[row_id] = translation
|
|
|
|
if len(translations) < expected_count:
|
|
logger.explore(
|
|
f"LLM returned fewer translations expected={expected_count} "
|
|
f"got={len(translations)} response_preview={response_text[:600]}"
|
|
)
|
|
|
|
return translations
|
|
# [/DEF:_parse_llm_response:Function]
|
|
|
|
# [DEF:_compute_config_hash:Function]
|
|
# @PURPOSE: Compute a hash of the job's current configuration for snapshot comparison.
|
|
@staticmethod
|
|
def _compute_config_hash(job: TranslationJob) -> str:
|
|
config_str = json.dumps({
|
|
"source_dialect": job.source_dialect,
|
|
"target_dialect": job.target_dialect,
|
|
"source_datasource_id": job.source_datasource_id,
|
|
"translation_column": job.translation_column,
|
|
"context_columns": job.context_columns,
|
|
"target_language": job.target_language,
|
|
"provider_id": job.provider_id,
|
|
"batch_size": job.batch_size,
|
|
"upsert_strategy": job.upsert_strategy,
|
|
}, sort_keys=True)
|
|
return hashlib.sha256(config_str.encode()).hexdigest()[:16]
|
|
# [/DEF:_compute_config_hash:Function]
|
|
|
|
# [DEF:_compute_dict_snapshot_hash:Function]
|
|
# @PURPOSE: Compute a hash of the dictionary state for snapshot comparison.
|
|
def _compute_dict_snapshot_hash(self, job_id: str) -> str:
|
|
dict_links = (
|
|
self.db.query(TranslationJobDictionary)
|
|
.filter(TranslationJobDictionary.job_id == job_id)
|
|
.all()
|
|
)
|
|
dict_ids = sorted([dl.dictionary_id for dl in dict_links])
|
|
hash_input = ",".join(dict_ids)
|
|
return hashlib.sha256(hash_input.encode()).hexdigest()[:16]
|
|
# [/DEF:_compute_dict_snapshot_hash:Function]
|
|
|
|
|
|
# #endregion TranslationPreview
|
|
|
|
# #endregion TranslationPreview
|