282 lines
14 KiB
Python
282 lines
14 KiB
Python
# #region TranslateJobService [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, job, datasource, dialect]
|
|
# @defgroup Translate Module group.
|
|
# @BRIEF Service layer for translation job CRUD with datasource column validation and database dialect detection.
|
|
# @LAYER Domain
|
|
# @RELATION DEPENDS_ON -> [TranslationJob]
|
|
# @RELATION DEPENDS_ON -> [SupersetClient]
|
|
# @RELATION DEPENDS_ON -> [ConfigManager]
|
|
# @PRE Database session and config manager are available.
|
|
# @POST Translation jobs are created/updated/deleted with column validation and dialect caching.
|
|
# @SIDE_EFFECT Queries Superset for column metadata and database dialect at save time.
|
|
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
|
|
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
|
|
# @REJECTED Monolithic service.py — violated INV_7. Decomposed into service_datasource, service_inline_correction, service_bulk_replace, service_utils.
|
|
|
|
from datetime import UTC, datetime
|
|
import uuid
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ...core.config_manager import ConfigManager
|
|
from ...core.logger import logger
|
|
from ...models.translate import TranslationJob, TranslationJobDictionary
|
|
from ...schemas.translate import TranslateJobCreate, TranslateJobUpdate
|
|
from .dictionary_validation import _validate_bcp47
|
|
from .service_datasource import fetch_datasource_metadata
|
|
from .service_utils import _extract_dialect
|
|
|
|
|
|
# #region TranslateJobService [TYPE Class]
|
|
# @defgroup Translate Module group.
|
|
# @BRIEF Service for translation job CRUD with validation and Superset integration.
|
|
class TranslateJobService:
|
|
"""CRUD service for translation jobs with Superset datasource integration."""
|
|
|
|
def __init__(self, db: Session, config_manager: ConfigManager, current_user: str | None = None):
|
|
self.db = db
|
|
self.config_manager = config_manager
|
|
self.current_user = current_user
|
|
|
|
# region list_jobs [TYPE Function]
|
|
# @PURPOSE: List translation jobs with optional status filter and pagination.
|
|
def list_jobs(
|
|
self,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
status_filter: str | None = None,
|
|
) -> tuple[int, list[TranslationJob]]:
|
|
query = self.db.query(TranslationJob)
|
|
if status_filter:
|
|
query = query.filter(TranslationJob.status == status_filter)
|
|
total = query.count()
|
|
offset = (page - 1) * page_size
|
|
jobs = query.order_by(TranslationJob.created_at.desc()).offset(offset).limit(page_size).all()
|
|
return total, jobs
|
|
# endregion list_jobs
|
|
|
|
# region get_job [TYPE Function]
|
|
# @PURPOSE: Get a single translation job by ID.
|
|
def get_job(self, job_id: str) -> TranslationJob:
|
|
job = self.db.query(TranslationJob).filter(TranslationJob.id == job_id).first()
|
|
if not job:
|
|
raise ValueError(f"Translation job '{job_id}' not found")
|
|
return job
|
|
# endregion get_job
|
|
|
|
# region create_job [TYPE Function]
|
|
# @PURPOSE: Create a new translation job with column validation.
|
|
# @SIDE_EFFECT Validates columns via SupersetClient; caches database_dialect.
|
|
async def create_job(self, payload: TranslateJobCreate) -> TranslationJob:
|
|
logger.info(f"[TranslateJobService] Creating job '{payload.name}'")
|
|
if payload.source_datasource_id and not payload.translation_column:
|
|
raise ValueError("A translation column is required when a datasource is selected")
|
|
valid_strategies = {"MERGE", "INSERT", "UPDATE"}
|
|
if payload.upsert_strategy not in valid_strategies:
|
|
raise ValueError(
|
|
f"Invalid upsert_strategy '{payload.upsert_strategy}'. "
|
|
f"Must be one of: {', '.join(sorted(valid_strategies))}"
|
|
)
|
|
|
|
dialect = payload.database_dialect
|
|
if payload.source_datasource_id and (payload.environment_id or payload.source_dialect):
|
|
if not dialect:
|
|
try:
|
|
env_id = payload.environment_id or payload.source_dialect
|
|
_, detected_dialect = await fetch_datasource_metadata(
|
|
int(payload.source_datasource_id), env_id, self.config_manager,
|
|
)
|
|
dialect = detected_dialect
|
|
except Exception as e:
|
|
logger.warning(f"[TranslateJobService] Dialect detection failed: {e}")
|
|
dialect = payload.source_dialect
|
|
|
|
target_languages = payload.target_languages
|
|
if not target_languages and payload.target_language:
|
|
target_languages = [payload.target_language]
|
|
if target_languages:
|
|
if not isinstance(target_languages, list):
|
|
target_languages = [str(target_languages)]
|
|
for lang in target_languages:
|
|
_validate_bcp47(lang, "target_languages")
|
|
|
|
job = TranslationJob(
|
|
id=str(uuid.uuid4()), name=payload.name, description=payload.description,
|
|
source_dialect=payload.source_dialect, target_dialect=payload.target_dialect,
|
|
database_dialect=dialect, source_datasource_id=payload.source_datasource_id,
|
|
source_table=payload.source_table, target_schema=payload.target_schema,
|
|
target_table=payload.target_table, source_key_cols=payload.source_key_cols or [],
|
|
target_key_cols=payload.target_key_cols or [],
|
|
translation_column=payload.translation_column, target_column=payload.target_column,
|
|
target_language_column=payload.target_language_column,
|
|
target_source_column=payload.target_source_column,
|
|
target_source_language_column=payload.target_source_language_column,
|
|
context_columns=payload.context_columns or [], target_languages=target_languages,
|
|
provider_id=payload.provider_id, batch_size=payload.batch_size,
|
|
upsert_strategy=payload.upsert_strategy, environment_id=payload.environment_id,
|
|
target_database_id=payload.target_database_id,
|
|
status="DRAFT", created_by=self.current_user,
|
|
)
|
|
self.db.add(job)
|
|
self.db.flush()
|
|
|
|
if payload.dictionary_ids:
|
|
for dict_id in payload.dictionary_ids:
|
|
self.db.add(TranslationJobDictionary(
|
|
id=str(uuid.uuid4()), job_id=job.id, dictionary_id=dict_id,
|
|
))
|
|
|
|
self.db.commit()
|
|
self.db.refresh(job)
|
|
logger.info(f"[TranslateJobService] Created job '{job.id}'")
|
|
return job
|
|
# endregion create_job
|
|
|
|
# region update_job [TYPE Function]
|
|
# @PURPOSE: Update an existing translation job.
|
|
# @SIDE_EFFECT Re-detects database_dialect if source_datasource_id changed.
|
|
async def update_job(self, job_id: str, payload: TranslateJobUpdate) -> TranslationJob:
|
|
logger.info(f"[TranslateJobService] Updating job '{job_id}'")
|
|
job = self.get_job(job_id)
|
|
update_data = payload.model_dump(exclude_unset=True)
|
|
dict_ids = update_data.pop("dictionary_ids", None)
|
|
|
|
if "target_language" in update_data:
|
|
tl = update_data.pop("target_language")
|
|
if tl and "target_languages" not in update_data:
|
|
update_data["target_languages"] = [tl]
|
|
|
|
if update_data.get("target_languages"):
|
|
if not isinstance(update_data["target_languages"], list):
|
|
update_data["target_languages"] = [str(update_data["target_languages"])]
|
|
for lang in update_data["target_languages"]:
|
|
_validate_bcp47(lang, "target_languages")
|
|
|
|
for field, value in update_data.items():
|
|
if hasattr(job, field):
|
|
setattr(job, field, value)
|
|
|
|
if payload.source_datasource_id and not payload.database_dialect:
|
|
try:
|
|
env_id = (payload.environment_id or payload.source_dialect or job.environment_id or job.source_dialect)
|
|
_, detected_dialect = await fetch_datasource_metadata(
|
|
int(payload.source_datasource_id), env_id, self.config_manager,
|
|
)
|
|
job.database_dialect = detected_dialect
|
|
except Exception as e:
|
|
logger.warning(f"[TranslateJobService] Dialect re-detection failed: {e}")
|
|
|
|
job.updated_at = datetime.now(UTC)
|
|
|
|
if dict_ids is not None:
|
|
self.db.query(TranslationJobDictionary).filter(TranslationJobDictionary.job_id == job_id).delete()
|
|
for dict_id in dict_ids:
|
|
self.db.add(TranslationJobDictionary(
|
|
id=str(uuid.uuid4()), job_id=job_id, dictionary_id=dict_id,
|
|
))
|
|
|
|
self.db.commit()
|
|
self.db.refresh(job)
|
|
logger.info(f"[TranslateJobService] Updated job '{job_id}'")
|
|
return job
|
|
# endregion update_job
|
|
|
|
# region delete_job [TYPE Function]
|
|
# @PURPOSE: Delete a translation job and its associations.
|
|
def delete_job(self, job_id: str) -> None:
|
|
logger.info(f"[TranslateJobService] Deleting job '{job_id}'")
|
|
job = self.get_job(job_id)
|
|
self.db.query(TranslationJobDictionary).filter(TranslationJobDictionary.job_id == job_id).delete()
|
|
self.db.delete(job)
|
|
self.db.commit()
|
|
logger.info(f"[TranslateJobService] Deleted job '{job_id}'")
|
|
# endregion delete_job
|
|
|
|
# region duplicate_job [TYPE Function]
|
|
# @PURPOSE: Duplicate a translation job with a new name.
|
|
def duplicate_job(self, job_id: str, new_name: str | None = None) -> TranslationJob:
|
|
logger.info(f"[TranslateJobService] Duplicating job '{job_id}'")
|
|
source = self.get_job(job_id)
|
|
new_job = TranslationJob(
|
|
id=str(uuid.uuid4()), name=new_name or f"{source.name} (Copy)",
|
|
description=source.description, source_dialect=source.source_dialect,
|
|
target_dialect=source.target_dialect, database_dialect=source.database_dialect,
|
|
source_datasource_id=source.source_datasource_id, source_table=source.source_table,
|
|
target_schema=source.target_schema, target_table=source.target_table,
|
|
source_key_cols=source.source_key_cols, target_key_cols=source.target_key_cols,
|
|
translation_column=source.translation_column, target_column=source.target_column,
|
|
target_language_column=source.target_language_column,
|
|
target_source_column=source.target_source_column,
|
|
target_source_language_column=source.target_source_language_column,
|
|
context_columns=source.context_columns, target_languages=source.target_languages,
|
|
provider_id=source.provider_id, batch_size=source.batch_size,
|
|
upsert_strategy=source.upsert_strategy, status="DRAFT",
|
|
created_by=self.current_user,
|
|
environment_id=source.environment_id,
|
|
target_database_id=source.target_database_id,
|
|
insert_method=source.insert_method or "sqllab",
|
|
connection_id=source.connection_id,
|
|
disable_reasoning=source.disable_reasoning or False,
|
|
)
|
|
self.db.add(new_job)
|
|
self.db.flush()
|
|
|
|
old_dicts = self.db.query(TranslationJobDictionary).filter(TranslationJobDictionary.job_id == job_id).all()
|
|
for assoc in old_dicts:
|
|
self.db.add(TranslationJobDictionary(
|
|
id=str(uuid.uuid4()), job_id=new_job.id, dictionary_id=assoc.dictionary_id,
|
|
))
|
|
|
|
self.db.commit()
|
|
self.db.refresh(new_job)
|
|
logger.info(f"[TranslateJobService] Duplicated job '{job_id}' -> '{new_job.id}'")
|
|
return new_job
|
|
# endregion duplicate_job
|
|
|
|
# region get_job_dictionary_ids [TYPE Function]
|
|
# @PURPOSE: Get dictionary IDs attached to a job.
|
|
def get_job_dictionary_ids(self, job_id: str) -> list[str]:
|
|
assocs = self.db.query(TranslationJobDictionary).filter(TranslationJobDictionary.job_id == job_id).all()
|
|
return [a.dictionary_id for a in assocs]
|
|
# endregion get_job_dictionary_ids
|
|
|
|
# region fetch_available_datasources [TYPE Function]
|
|
# @PURPOSE: List available Superset datasets for translation job creation.
|
|
async def fetch_available_datasources(self, env_id: str, search: str | None = None) -> list:
|
|
from ...core.utils.client_registry import get_superset_client
|
|
env = self.config_manager.get_environment(env_id)
|
|
if not env:
|
|
raise ValueError(f"Environment '{env_id}' not found")
|
|
client = await get_superset_client(env)
|
|
_, datasets = await client.get_datasets()
|
|
result = []
|
|
for ds in datasets:
|
|
name = ds.get("table_name", "")
|
|
if search and search.lower() not in name.lower():
|
|
continue
|
|
db_info = ds.get("database", {})
|
|
backend = db_info.get("backend", "")
|
|
dialect = _extract_dialect(backend)
|
|
result.append({
|
|
"id": ds.get("id"),
|
|
"table_name": name,
|
|
"schema": ds.get("schema"),
|
|
"database_name": db_info.get("database_name", "Unknown"),
|
|
"database_id": db_info.get("id"), # <-- добавлено: ID БД для target_database_id
|
|
"database_dialect": dialect,
|
|
"description": ds.get("description", ""),
|
|
})
|
|
return result
|
|
# endregion fetch_available_datasources
|
|
# #endregion TranslateJobService
|
|
|
|
|
|
# Re-exports for backward compatibility
|
|
from .service_bulk_replace import BulkFindReplaceService # noqa: E402, F401
|
|
from .service_datasource import ( # noqa: E402, F401
|
|
detect_virtual_columns,
|
|
get_datasource_columns,
|
|
get_dialect_from_database,
|
|
)
|
|
from .service_inline_correction import InlineCorrectionService # noqa: E402, F401
|
|
# #endregion TranslateJobService
|