feat(translate): add insert_method dispatch and model/schema updates for direct DB
This commit is contained in:
@@ -76,5 +76,57 @@ class SupersetDatabasesMixin:
|
||||
_, databases = await self.get_databases(query=query)
|
||||
return databases[0] if databases else None
|
||||
# #endregion SupersetClientGetDatabaseByUuid
|
||||
# #region SupersetClientCreateDatabase [TYPE Function] [C:3]
|
||||
# @ingroup Core
|
||||
# @PURPOSE: Register a new database in Superset via REST API.
|
||||
# @POST Returns the created database dict with id.
|
||||
# @SIDE_EFFECT Makes POST request to /api/v1/database/.
|
||||
# @RELATION CALLS -> [AsyncAPIClient.request]
|
||||
async def create_database(
|
||||
self,
|
||||
database_name: str,
|
||||
sqlalchemy_uri: str,
|
||||
expose_in_sqllab: bool = True,
|
||||
allow_dml: bool = False,
|
||||
) -> dict:
|
||||
with belief_scope("SupersetClient.create_database"):
|
||||
app_logger.reason(
|
||||
"Creating database in Superset",
|
||||
extra={"database_name": database_name, "expose_in_sqllab": expose_in_sqllab},
|
||||
)
|
||||
response = await self.client.request(
|
||||
method="POST",
|
||||
endpoint="/database/",
|
||||
data={
|
||||
"database_name": database_name,
|
||||
"sqlalchemy_uri": sqlalchemy_uri,
|
||||
"expose_in_sqllab": expose_in_sqllab,
|
||||
"allow_dml": allow_dml,
|
||||
},
|
||||
)
|
||||
response = cast(dict, response)
|
||||
app_logger.reflect(
|
||||
"Database created",
|
||||
extra={"database_id": response.get("id"), "database_name": database_name},
|
||||
)
|
||||
return response
|
||||
# #endregion SupersetClientCreateDatabase
|
||||
# #region SupersetClientDeleteDatabase [TYPE Function] [C:3]
|
||||
# @ingroup Core
|
||||
# @PURPOSE: Delete a database from Superset by ID.
|
||||
# @POST Database removed from Superset.
|
||||
# @SIDE_EFFECT Makes DELETE request to /api/v1/database/<id>.
|
||||
# @RELATION CALLS -> [AsyncAPIClient.request]
|
||||
async def delete_database(self, database_id: int) -> dict:
|
||||
with belief_scope("SupersetClient.delete_database"):
|
||||
app_logger.reason("Deleting database", extra={"database_id": database_id})
|
||||
response = await self.client.request(
|
||||
method="DELETE",
|
||||
endpoint=f"/database/{database_id}",
|
||||
)
|
||||
response = cast(dict, response)
|
||||
app_logger.reflect("Database deleted", extra={"database_id": database_id})
|
||||
return response
|
||||
# #endregion SupersetClientDeleteDatabase
|
||||
# #endregion SupersetDatabasesMixin
|
||||
# #endregion SupersetDatabasesMixin
|
||||
|
||||
@@ -61,6 +61,10 @@ class TranslationJob(Base):
|
||||
environment_id = Column(String, nullable=True, comment="Superset environment ID for datasource access")
|
||||
target_database_id = Column(String, nullable=True, comment="Superset database ID for SQL Lab insert target")
|
||||
|
||||
# Direct DB insert
|
||||
insert_method = Column(String, nullable=False, default="sqllab", comment="sqllab|direct_db")
|
||||
connection_id = Column(String, nullable=True, comment="UUID referencing a DatabaseConnection in GlobalSettings.connections")
|
||||
|
||||
created_by = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=lambda: datetime.now(UTC))
|
||||
updated_at = Column(DateTime, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
|
||||
@@ -85,9 +89,11 @@ class TranslationRun(Base):
|
||||
failed_records = Column(Integer, default=0)
|
||||
skipped_records = Column(Integer, default=0)
|
||||
cache_hits = Column(Integer, default=0, comment="Number of rows served from translation cache")
|
||||
insert_status = Column(String, nullable=True, comment="Status of the Superset insert/update operation")
|
||||
superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID")
|
||||
superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output")
|
||||
insert_method = Column(String, nullable=True, comment="sqllab|direct_db — copied from job at run creation")
|
||||
connection_snapshot = Column(JSON, nullable=True, comment="Snapshot of connection metadata for audit: {name, dialect, host, port, database} — NEVER password")
|
||||
insert_status = Column(String, nullable=True, comment="Status of the insert/update operation")
|
||||
superset_execution_id = Column(String, nullable=True, comment="Superset execution/task ID (NULL for direct_db inserts)")
|
||||
superset_execution_log = Column(JSON, nullable=True, comment="Superset execution log output (NULL for direct_db inserts)")
|
||||
config_snapshot = Column(JSON, nullable=True, comment="Snapshot of job config at run creation time")
|
||||
key_hash = Column(String, nullable=True, comment="Hash of source key fields for dedup")
|
||||
config_hash = Column(String, nullable=True, comment="Hash of translation configuration state")
|
||||
|
||||
@@ -81,6 +81,9 @@ class TranslationResultAggregator:
|
||||
"skipped_records": run.skipped_records or 0,
|
||||
"cache_hits": run.cache_hits or 0,
|
||||
"insert_status": run.insert_status,
|
||||
"insert_method": run.insert_method,
|
||||
"connection_snapshot": run.connection_snapshot,
|
||||
"config_snapshot": run.config_snapshot,
|
||||
"superset_execution_id": run.superset_execution_id,
|
||||
"batch_count": batch_count,
|
||||
"language_stats": language_stats,
|
||||
|
||||
@@ -141,6 +141,8 @@ def get_run_history(
|
||||
"skipped_records": r.skipped_records or 0,
|
||||
"cache_hits": r.cache_hits or 0,
|
||||
"insert_status": r.insert_status,
|
||||
"insert_method": r.insert_method,
|
||||
"connection_snapshot": r.connection_snapshot,
|
||||
"superset_execution_id": r.superset_execution_id,
|
||||
"created_by": r.created_by,
|
||||
"created_at": r.created_at.isoformat() if r.created_at else None,
|
||||
|
||||
@@ -1,20 +1,25 @@
|
||||
# #region SQLInsertService [C:4] [TYPE Class] [SEMANTICS sql, insert, superset, translate]
|
||||
# #region SQLInsertService [C:4] [TYPE Class] [SEMANTICS sql, insert, superset, translate, direct-db]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Generate INSERT SQL from translation records and submit to Superset SQL Lab.
|
||||
# @BRIEF Generate INSERT SQL from translation records and submit to target via Superset SQL Lab OR direct DB.
|
||||
# @PRE Job has target table configured. Run has successful records.
|
||||
# @POST SQL is generated and submitted to Superset; returns execution result.
|
||||
# @SIDE_EFFECT Superset API call; event log entries.
|
||||
# @POST SQL generated and executed via Superset (sqllab) or DbExecutor (direct_db). insert_method recorded.
|
||||
# @SIDE_EFFECT Superset API call OR native DB driver call; event log entries.
|
||||
# @RELATION DEPENDS_ON -> [SQLGenerator]
|
||||
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
|
||||
# @RELATION DEPENDS_ON -> [Core.DbExecutor]
|
||||
# @RELATION DEPENDS_ON -> [Core.ConnectionService]
|
||||
# @RELATION DEPENDS_ON -> [TranslationEventLog]
|
||||
# @RELATION DEPENDS_ON -> [orchestrator_sql_rows]
|
||||
# @RATIONALE Row/column building extracted to orchestrator_sql_rows module for INV_7 compliance.
|
||||
# Direct DB execution added as parallel path — does NOT replace Superset SQL Lab.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.connection_service import ConnectionService
|
||||
from ...core.db_executor import DbExecutor
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.translate import TranslationJob, TranslationRecord, TranslationRun
|
||||
from .events import TranslationEventLog
|
||||
@@ -24,7 +29,7 @@ from .superset_executor import SupersetSqlLabExecutor
|
||||
|
||||
|
||||
class SQLInsertService:
|
||||
"""Generate INSERT SQL from successful records and submit to Superset."""
|
||||
"""Generate INSERT SQL from successful records and submit to target (Superset or direct DB)."""
|
||||
|
||||
def __init__(self, db: Session, config_manager: ConfigManager, event_log: TranslationEventLog):
|
||||
self.db = db
|
||||
@@ -32,7 +37,7 @@ class SQLInsertService:
|
||||
self.event_log = event_log
|
||||
|
||||
# region generate_and_insert_sql [TYPE Function]
|
||||
# @PURPOSE: Generate INSERT SQL and submit to Superset SQL Lab.
|
||||
# @PURPOSE: Generate INSERT SQL and execute via configured insert method (sqllab or direct_db).
|
||||
async def generate_and_insert_sql(
|
||||
self,
|
||||
job: TranslationJob,
|
||||
@@ -83,28 +88,97 @@ class SQLInsertService:
|
||||
self.event_log.log_event(
|
||||
job_id=job.id, run_id=run.id,
|
||||
event_type="INSERT_PHASE_STARTED",
|
||||
payload={"sql_length": len(sql), "row_count": row_count, "dialect": dialect},
|
||||
payload={
|
||||
"sql_length": len(sql), "row_count": row_count,
|
||||
"dialect": dialect, "insert_method": job.insert_method,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
env_id = job.environment_id or job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
await executor.resolve_database_id(target_database_id=job.target_database_id)
|
||||
result = await executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
|
||||
except Exception as e:
|
||||
logger.explore("Superset SQL submission failed", {"run_id": run.id, "error": str(e)})
|
||||
result = {"status": "failed", "error_message": str(e), "query_id": None}
|
||||
# ── Dispatch to correct executor based on insert_method ──
|
||||
insert_method = getattr(job, "insert_method", "sqllab") or "sqllab"
|
||||
|
||||
if insert_method == "direct_db" and job.connection_id:
|
||||
result = await self._execute_direct_db(job, run, sql)
|
||||
else:
|
||||
result = await self._execute_superset_sqllab(job, run, sql)
|
||||
|
||||
self.event_log.log_event(
|
||||
job_id=job.id, run_id=run.id,
|
||||
event_type="INSERT_PHASE_COMPLETED",
|
||||
payload=result,
|
||||
payload={**result, "insert_method": insert_method},
|
||||
)
|
||||
return result
|
||||
# endregion generate_and_insert_sql
|
||||
|
||||
# region _execute_superset_sqllab [TYPE Function]
|
||||
# @BRIEF Execute INSERT via Superset SQL Lab API.
|
||||
async def _execute_superset_sqllab(
|
||||
self, job: TranslationJob, run: TranslationRun, sql: str,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
env_id = job.environment_id or job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
await executor.resolve_database_id(target_database_id=job.target_database_id)
|
||||
result = await executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
|
||||
except Exception as e:
|
||||
logger.explore("Superset SQL submission failed", {"run_id": run.id, "error": str(e)})
|
||||
result = {"status": "failed", "error_message": str(e), "query_id": None}
|
||||
return result
|
||||
# endregion _execute_superset_sqllab
|
||||
|
||||
# region _execute_direct_db [TYPE Function]
|
||||
# @BRIEF Execute INSERT directly against the target database via DbExecutor.
|
||||
# @PRE job.connection_id references a valid DatabaseConnection.
|
||||
# @POST SQL executed directly. Connection snapshot stored on run for audit.
|
||||
# @SIDE_EFFECT Writes to target DB. Connection pool lifecycle managed by DbExecutor.
|
||||
async def _execute_direct_db(
|
||||
self, job: TranslationJob, run: TranslationRun, sql: str,
|
||||
) -> dict[str, Any]:
|
||||
conn_service = ConnectionService(self.config_manager)
|
||||
# Snapshot connection metadata for audit (NEVER password)
|
||||
conn = conn_service.get_connection(job.connection_id)
|
||||
if conn is None:
|
||||
return {"status": "failed", "error_message": f"Connection '{job.connection_id}' not found", "query_id": None}
|
||||
|
||||
# Store connection snapshot on run
|
||||
run.connection_snapshot = {
|
||||
"name": conn.name,
|
||||
"dialect": conn.dialect,
|
||||
"host": conn.host,
|
||||
"port": conn.port,
|
||||
"database": conn.database,
|
||||
}
|
||||
run.insert_method = "direct_db"
|
||||
self.db.commit()
|
||||
|
||||
executor = DbExecutor(conn_service)
|
||||
result = executor.execute_sql(job.connection_id, sql)
|
||||
|
||||
if result.success:
|
||||
return {
|
||||
"status": "success",
|
||||
"rows_affected": result.rows_affected,
|
||||
"execution_time_ms": result.execution_time_ms,
|
||||
"query_id": None, # Direct DB has no Superset query reference
|
||||
"connection_name": conn.name,
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "failed",
|
||||
"error_message": result.error or "Direct DB insert failed",
|
||||
"execution_time_ms": result.execution_time_ms,
|
||||
"query_id": None,
|
||||
}
|
||||
# endregion _execute_direct_db
|
||||
|
||||
# region _resolve_dialect [TYPE Function]
|
||||
async def _resolve_dialect(self, job: TranslationJob) -> str:
|
||||
if getattr(job, "insert_method", None) == "direct_db" and job.connection_id:
|
||||
conn_service = ConnectionService(self.config_manager)
|
||||
conn = conn_service.get_connection(job.connection_id)
|
||||
if conn:
|
||||
return conn.dialect
|
||||
|
||||
try:
|
||||
env_id = job.environment_id or job.source_dialect or ""
|
||||
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
|
||||
|
||||
@@ -272,13 +272,6 @@ class TranslationPreview:
|
||||
return _compute_config_hash_module(job)
|
||||
|
||||
# Delegated methods
|
||||
def update_preview_row(self, job_id: str, row_id: str, action: str, translation: str | None = None,
|
||||
feedback: str | None = None, language_code: str | None = None) -> dict[str, Any]:
|
||||
return self._session_mgr.update_preview_row(job_id, row_id, action, translation, feedback, language_code)
|
||||
|
||||
def accept_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
return self._session_mgr.accept_preview_session(job_id)
|
||||
|
||||
def get_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
return self._session_mgr.get_preview_session(job_id)
|
||||
|
||||
|
||||
@@ -1,132 +1,23 @@
|
||||
# #region PreviewSessionManager [C:3] [TYPE Class] [SEMANTICS preview, session, review, approve]
|
||||
# #region PreviewSessionManager [C:3] [TYPE Class] [SEMANTICS preview, session]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Manage preview session row-level actions: approve/edit/reject rows.
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewLanguage]
|
||||
# @BRIEF Manage preview session querying.
|
||||
# @RELATION DEPENDS_ON -> [preview_session_ops]
|
||||
# @RATIONALE Session accept/get and serialization extracted to preview_session_ops module for INV_7 compliance.
|
||||
# @RATIONALE Session get extracted to preview_session_ops module for INV_7 compliance.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.logger import belief_scope
|
||||
from ...models.translate import (
|
||||
TranslationPreviewLanguage,
|
||||
TranslationPreviewRecord,
|
||||
TranslationPreviewSession,
|
||||
)
|
||||
from .preview_session_ops import (
|
||||
accept_preview_session as _accept_preview_session,
|
||||
apply_language_action as _apply_language_action,
|
||||
apply_record_action as _apply_record_action,
|
||||
get_preview_session as _get_preview_session,
|
||||
)
|
||||
from .preview_session_ops import get_preview_session as _get_preview_session
|
||||
|
||||
|
||||
class PreviewSessionManager:
|
||||
"""Manage preview session lifecycle: row actions, accept, query."""
|
||||
"""Manage preview session lifecycle: query."""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
# region update_preview_row [TYPE Function]
|
||||
# @PURPOSE: Approve, edit, or reject an individual preview row (optionally per language).
|
||||
def update_preview_row(
|
||||
self,
|
||||
job_id: str,
|
||||
row_id: str,
|
||||
action: str,
|
||||
translation: str | None = None,
|
||||
feedback: str | None = None,
|
||||
language_code: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("PreviewSessionManager.update_preview_row"):
|
||||
session = (
|
||||
self.db.query(TranslationPreviewSession)
|
||||
.filter(
|
||||
TranslationPreviewSession.job_id == job_id,
|
||||
TranslationPreviewSession.status == "ACTIVE",
|
||||
)
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
raise ValueError(f"No active preview session for job '{job_id}'")
|
||||
|
||||
record = (
|
||||
self.db.query(TranslationPreviewRecord)
|
||||
.filter(
|
||||
TranslationPreviewRecord.id == row_id,
|
||||
TranslationPreviewRecord.session_id == session.id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not record:
|
||||
raise ValueError(f"Preview record '{row_id}' not found in active session")
|
||||
|
||||
if language_code:
|
||||
lang_entry = (
|
||||
self.db.query(TranslationPreviewLanguage)
|
||||
.filter(
|
||||
TranslationPreviewLanguage.preview_record_id == row_id,
|
||||
TranslationPreviewLanguage.language_code == language_code,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not lang_entry:
|
||||
raise ValueError(
|
||||
f"Language entry '{language_code}' not found for record '{row_id}'"
|
||||
)
|
||||
_apply_language_action(lang_entry, action, translation)
|
||||
|
||||
all_langs = (
|
||||
self.db.query(TranslationPreviewLanguage)
|
||||
.filter(TranslationPreviewLanguage.preview_record_id == row_id)
|
||||
.all()
|
||||
)
|
||||
if all(le.status in ("approved", "edited") for le in all_langs):
|
||||
record.status = "APPROVED"
|
||||
if action == "edit" and translation is not None and all_langs:
|
||||
record.target_sql = all_langs[0].final_value or all_langs[0].translated_value
|
||||
else:
|
||||
_apply_record_action(record, action, translation)
|
||||
|
||||
if feedback is not None:
|
||||
record.feedback = feedback
|
||||
|
||||
self.db.commit()
|
||||
self.db.refresh(record)
|
||||
|
||||
return {
|
||||
"id": record.id,
|
||||
"source_sql": record.source_sql,
|
||||
"target_sql": record.target_sql,
|
||||
"status": record.status,
|
||||
"feedback": record.feedback,
|
||||
"languages": [
|
||||
{
|
||||
"language_code": le.language_code,
|
||||
"source_language_detected": le.source_language_detected,
|
||||
"translated_value": le.translated_value,
|
||||
"final_value": le.final_value,
|
||||
"user_edit": le.user_edit,
|
||||
"status": le.status,
|
||||
"needs_review": le.needs_review,
|
||||
}
|
||||
for le in (record.languages or [])
|
||||
],
|
||||
}
|
||||
# endregion update_preview_row
|
||||
|
||||
# region accept_preview_session [TYPE Function]
|
||||
# @PURPOSE: Mark a preview session as accepted, which gates full execution.
|
||||
def accept_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
with belief_scope("PreviewSessionManager.accept_preview_session"):
|
||||
return _accept_preview_session(self.db, job_id)
|
||||
# endregion accept_preview_session
|
||||
|
||||
# region get_preview_session [TYPE Function]
|
||||
# @PURPOSE: Get the latest preview session for a job with its records.
|
||||
def get_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# #region preview_session_ops [C:3] [TYPE Module] [SEMANTICS preview, session, accept, get]
|
||||
# #region preview_session_ops [C:3] [TYPE Module] [SEMANTICS preview, session, query]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Preview session lifecycle operations: accept and query preview sessions.
|
||||
# @BRIEF Preview session lifecycle operations: query preview sessions.
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewSession]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
|
||||
# @RELATION DEPENDS_ON -> [preview_session_serializer]
|
||||
@@ -15,52 +15,10 @@ from ...models.translate import (
|
||||
TranslationPreviewSession,
|
||||
)
|
||||
from .preview_session_serializer import (
|
||||
apply_language_action, # noqa: F401 — re-export for preview_review.py
|
||||
apply_record_action, # noqa: F401 — re-export for preview_review.py
|
||||
serialize_preview_record as _serialize_preview_record,
|
||||
)
|
||||
|
||||
|
||||
# #region accept_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, accept]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Mark a preview session as accepted, which gates full execution.
|
||||
# @SIDE_EFFECT DB writes on session status.
|
||||
def accept_preview_session(db: Session, job_id: str, _current_user: str | None = None) -> dict[str, Any]:
|
||||
"""Mark a preview session as accepted and return the session data with records."""
|
||||
session = (
|
||||
db.query(TranslationPreviewSession)
|
||||
.filter(TranslationPreviewSession.job_id == job_id, TranslationPreviewSession.status == "ACTIVE")
|
||||
.order_by(TranslationPreviewSession.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not session:
|
||||
raise ValueError(f"No active preview session for job '{job_id}'")
|
||||
session.status = "APPLIED"
|
||||
db.commit()
|
||||
db.refresh(session)
|
||||
|
||||
records = (
|
||||
db.query(TranslationPreviewRecord)
|
||||
.filter(TranslationPreviewRecord.session_id == session.id)
|
||||
.all()
|
||||
)
|
||||
job = db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
||||
target_languages = job.target_languages or [job.target_dialect or "en"] if job else ["en"]
|
||||
if not isinstance(target_languages, list):
|
||||
target_languages = [str(target_languages)]
|
||||
return {
|
||||
"id": session.id,
|
||||
"job_id": job_id,
|
||||
"status": "APPLIED",
|
||||
"created_by": session.created_by,
|
||||
"created_at": session.created_at.isoformat(),
|
||||
"expires_at": session.expires_at.isoformat() if session.expires_at else None,
|
||||
"target_languages": target_languages,
|
||||
"records": [_serialize_preview_record(r) for r in records],
|
||||
}
|
||||
# #endregion accept_preview_session
|
||||
|
||||
|
||||
# #region get_preview_session [C:2] [TYPE Function] [SEMANTICS preview, session, query]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Get the latest preview session for a job with its records.
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
# #region preview_session_serializer [C:3] [TYPE Module] [SEMANTICS preview, record, serialize, action]
|
||||
# #region preview_session_serializer [C:2] [TYPE Module] [SEMANTICS preview, record, serialize]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Serialization and action helpers for preview sessions: record serialization and per-row approve/reject/edit actions.
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewRecord]
|
||||
# @RELATION DEPENDS_ON -> [TranslationPreviewLanguage]
|
||||
# @RATIONALE Extracted from preview_session_ops.py for INV_7 compliance (module < 150 lines).
|
||||
# @BRIEF Serialization helpers for preview sessions: record serialization.
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ...models.translate import TranslationPreviewLanguage, TranslationPreviewRecord
|
||||
from ...models.translate import TranslationPreviewRecord
|
||||
|
||||
|
||||
# #region serialize_preview_record [C:1] [TYPE Function] [SEMANTICS preview, record, serialize]
|
||||
@@ -43,62 +40,4 @@ def serialize_preview_record(r: TranslationPreviewRecord) -> dict[str, Any]:
|
||||
] if r.languages else [],
|
||||
}
|
||||
# #endregion serialize_preview_record
|
||||
|
||||
|
||||
# #region apply_language_action [C:2] [TYPE Function] [SEMANTICS preview, language, action]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Apply approve/reject/edit action to a TranslationPreviewLanguage entry.
|
||||
def apply_language_action(
|
||||
lang_entry: TranslationPreviewLanguage,
|
||||
action: str,
|
||||
translation: str | None,
|
||||
) -> None:
|
||||
"""Apply action to a per-language preview entry."""
|
||||
if action == "approve":
|
||||
lang_entry.status = "approved"
|
||||
elif action == "reject":
|
||||
lang_entry.status = "rejected"
|
||||
elif action == "edit":
|
||||
lang_entry.status = "edited"
|
||||
if translation is not None:
|
||||
lang_entry.translated_value = translation
|
||||
lang_entry.final_value = translation
|
||||
lang_entry.user_edit = translation
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
# #endregion apply_language_action
|
||||
|
||||
|
||||
# #region apply_record_action [C:2] [TYPE Function] [SEMANTICS preview, record, action]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Apply approve/reject/edit action to a TranslationPreviewRecord and its language entries.
|
||||
def apply_record_action(
|
||||
record: TranslationPreviewRecord,
|
||||
action: str,
|
||||
translation: str | None,
|
||||
) -> None:
|
||||
"""Apply action to a preview record."""
|
||||
if action == "approve":
|
||||
record.status = "APPROVED"
|
||||
for lang_entry in (record.languages or []):
|
||||
if lang_entry.status == "pending":
|
||||
lang_entry.status = "approved"
|
||||
elif action == "reject":
|
||||
record.status = "REJECTED"
|
||||
for lang_entry in (record.languages or []):
|
||||
if lang_entry.status == "pending":
|
||||
lang_entry.status = "rejected"
|
||||
elif action == "edit":
|
||||
record.status = "APPROVED"
|
||||
if translation is not None:
|
||||
record.target_sql = translation
|
||||
if record.languages:
|
||||
first_lang = record.languages[0]
|
||||
first_lang.translated_value = translation
|
||||
first_lang.final_value = translation
|
||||
first_lang.user_edit = translation
|
||||
first_lang.status = "edited"
|
||||
else:
|
||||
raise ValueError(f"Invalid action '{action}'. Use 'approve', 'reject', or 'edit'.")
|
||||
# #endregion apply_record_action
|
||||
# #endregion preview_session_serializer
|
||||
|
||||
@@ -211,6 +211,11 @@ class TranslateJobService:
|
||||
provider_id=source.provider_id, batch_size=source.batch_size,
|
||||
upsert_strategy=source.upsert_strategy, status="DRAFT",
|
||||
created_by=self.current_user,
|
||||
environment_id=source.environment_id,
|
||||
target_database_id=source.target_database_id,
|
||||
insert_method=source.insert_method or "sqllab",
|
||||
connection_id=source.connection_id,
|
||||
disable_reasoning=source.disable_reasoning or False,
|
||||
)
|
||||
self.db.add(new_job)
|
||||
self.db.flush()
|
||||
|
||||
@@ -23,6 +23,8 @@ from ...schemas.translate import (
|
||||
TargetSchemaValidationResponse,
|
||||
)
|
||||
from .superset_executor import SupersetSqlLabExecutor
|
||||
from ...core.db_executor import DbExecutor
|
||||
from ...core.connection_service import ConnectionService
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
|
||||
|
||||
@@ -230,8 +232,60 @@ async def validate_target_table_schema(
|
||||
missing_columns=expected, extra_columns=[], all_present=False,
|
||||
)
|
||||
|
||||
# 3. Запрос через Superset SQL Lab
|
||||
# 3. Запрос через Superset SQL Lab или прямую БД
|
||||
target_ref = f"{req.target_schema or 'public'}.{req.target_table}"
|
||||
|
||||
# Direct DB path — use native driver instead of SQL Lab
|
||||
if req.insert_method == "direct_db" and req.connection_id:
|
||||
logger.reason("Querying target table schema via direct DB",
|
||||
extra={"payload": {"table": target_ref, "connection_id": req.connection_id}})
|
||||
|
||||
schema = req.target_schema or "public"
|
||||
connection_svc = ConnectionService(config_manager)
|
||||
executor = DbExecutor(connection_svc)
|
||||
columns = executor.fetch_schema(req.connection_id, schema, req.target_table, config_manager)
|
||||
|
||||
if columns is None:
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=False,
|
||||
error="Failed to fetch schema via direct DB connection. Check connection and table name.",
|
||||
expected_columns=expected, actual_columns=[],
|
||||
missing_columns=expected, extra_columns=[], all_present=False,
|
||||
)
|
||||
|
||||
actual_cols = []
|
||||
for col in columns:
|
||||
actual_cols.append(TargetSchemaColumnInfo(
|
||||
name=col.name,
|
||||
data_type=col.data_type,
|
||||
is_nullable=True,
|
||||
))
|
||||
|
||||
actual_names = {c.name for c in actual_cols}
|
||||
table_exists = len(actual_cols) > 0
|
||||
|
||||
logger.reason("Direct DB schema fetched",
|
||||
extra={"payload": {"table": target_ref, "columns_found": len(actual_cols)}})
|
||||
|
||||
# Use actual_names directly (no SQL Lab parsing needed)
|
||||
missing_names = expected_names - actual_names
|
||||
extra_names = actual_names - expected_names
|
||||
|
||||
missing_cols = [c for c in expected if c.name in missing_names]
|
||||
extra_cols = [TargetSchemaColumnInfo(name=n, data_type=None) for n in extra_names]
|
||||
|
||||
all_present = len(missing_cols) == 0
|
||||
|
||||
return TargetSchemaValidationResponse(
|
||||
table_exists=table_exists,
|
||||
expected_columns=expected,
|
||||
actual_columns=actual_cols,
|
||||
missing_columns=missing_cols,
|
||||
extra_columns=extra_cols,
|
||||
all_present=all_present,
|
||||
)
|
||||
|
||||
# SQL Lab path (default)
|
||||
logger.reason("Querying target table schema",
|
||||
extra={"payload": {"table": target_ref, "env": req.environment_id, "db_id": req.target_database_id}})
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
|
||||
dictionary_ids=dict_ids or [],
|
||||
environment_id=job.environment_id,
|
||||
target_database_id=job.target_database_id,
|
||||
insert_method=job.insert_method or "sqllab",
|
||||
connection_id=job.connection_id,
|
||||
disable_reasoning=job.disable_reasoning or False,
|
||||
)
|
||||
# #endregion job_to_response
|
||||
# #endregion ServiceUtils
|
||||
|
||||
@@ -8,7 +8,7 @@ from datetime import datetime
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def _validate_bcp47_list(v: list[str] | None) -> list[str] | None:
|
||||
@@ -58,6 +58,15 @@ class TranslateJobCreate(BaseModel):
|
||||
dictionary_ids: list[str] | None = Field(default_factory=list, description="Associated terminology dictionary IDs")
|
||||
environment_id: str | None = Field(None, description="Superset environment ID")
|
||||
target_database_id: str | None = Field(None, description="Superset database ID for SQL Lab insert target")
|
||||
# Direct DB insert
|
||||
insert_method: str = Field("sqllab", description="Insert execution method: sqllab|direct_db")
|
||||
connection_id: str | None = Field(None, description="UUID referencing a DatabaseConnection in GlobalSettings.connections (required when insert_method=direct_db)")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_insert_method(self):
|
||||
if self.insert_method == "direct_db" and not self.connection_id:
|
||||
raise ValueError("connection_id is required when insert_method='direct_db'")
|
||||
return self
|
||||
# #endregion TranslateJobCreate
|
||||
|
||||
|
||||
@@ -94,6 +103,8 @@ class TranslateJobUpdate(BaseModel):
|
||||
dictionary_ids: list[str] | None = None
|
||||
environment_id: str | None = None
|
||||
target_database_id: str | None = None
|
||||
insert_method: str | None = None
|
||||
connection_id: str | None = None
|
||||
# #endregion TranslateJobUpdate
|
||||
|
||||
|
||||
@@ -132,6 +143,8 @@ class TranslateJobResponse(BaseModel):
|
||||
dictionary_ids: list[str] | None = None
|
||||
environment_id: str | None = None
|
||||
target_database_id: str | None = None
|
||||
insert_method: str = "sqllab"
|
||||
connection_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
# #endregion TranslateJobResponse
|
||||
@@ -431,6 +444,8 @@ class TranslationRunResponse(BaseModel):
|
||||
skipped_records: int = 0
|
||||
cache_hits: int = 0
|
||||
insert_status: str | None = None
|
||||
insert_method: str | None = None
|
||||
connection_snapshot: dict[str, Any] | None = None
|
||||
superset_execution_id: str | None = None
|
||||
config_snapshot: dict[str, Any] | None = None
|
||||
key_hash: str | None = None
|
||||
@@ -705,7 +720,7 @@ class InlineCorrectionSubmit(BaseModel):
|
||||
# @BRIEF Schema for requesting target table schema validation.
|
||||
class TargetSchemaValidationRequest(BaseModel):
|
||||
environment_id: str = Field(..., description="Superset environment ID")
|
||||
target_database_id: str = Field(..., description="Superset database ID for the target DB")
|
||||
target_database_id: str = Field(..., description="Superset database ID for the target DB (SQL Lab)")
|
||||
target_schema: str = Field("public", description="Target table schema (default: public)")
|
||||
target_table: str = Field(..., description="Target table name")
|
||||
# Column mapping from job config (everything that affects expected columns)
|
||||
@@ -715,6 +730,9 @@ class TargetSchemaValidationRequest(BaseModel):
|
||||
target_language_column: str | None = Field(None, description="Column for language code")
|
||||
target_source_column: str | None = Field(None, description="Column for source text")
|
||||
target_source_language_column: str | None = Field(None, description="Column for detected source language")
|
||||
# Direct DB support — when insert_method=direct_db, use connection_id instead of target_database_id
|
||||
insert_method: str | None = Field(None, description="Insert method: 'sqllab' or 'direct_db'")
|
||||
connection_id: str | None = Field(None, description="Direct DB connection ID (required when insert_method=direct_db)")
|
||||
# #endregion TargetSchemaValidationRequest
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user