busya
18f88a6928
CRITICAL BUG-1: UPSERT routing generated invalid ON CONFLICT for MySQL,
MSSQL, Snowflake, Oracle, DuckDB. Now uses explicit UPSERT_SUPPORTED_DIALECTS
({'postgresql', 'redshift'}) — unknown dialects get plain INSERT.
BUG-2: _extract_dialect did not strip whitespace, inconsistent with
get_dialect_from_database. Fixed guard + normalized value.
BUG-3: Whitespace-only input (' ', '\n') returned itself instead of 'unknown'.
BUG-4: Dead code — removed 'greenplum' from POSTGRESQL_DIALECTS
(always normalized to 'postgresql').
BUG-5: MySQL identifier quoting used double quotes instead of native backticks.
Added BACKTICK_DIALECTS set.
MOCK-FIX: test_at_least_one_row_per_batch patched wrong path
(executor.estimate_token_budget -> _batch_sizer.estimate_token_budget).
Adds 131 orthogonal tests covering all 4 dialect detection code paths
across input formats, normalization, routing, quoting, encoding,
schema validation, and cross-component consistency.
72 lines
2.8 KiB
Python
72 lines
2.8 KiB
Python
# #region ServiceUtils [C:1] [TYPE Module] [SEMANTICS utils, dialect, response]
|
|
# @BRIEF Utility functions for the translate service layer.
|
|
|
|
|
|
from ...models.translate import TranslationJob
|
|
from ...schemas.translate import TranslateJobResponse
|
|
|
|
|
|
# #region _extract_dialect [C:2] [TYPE Function]
|
|
# @BRIEF Extract database dialect from Superset backend URI or engine name.
|
|
def _extract_dialect(backend: str) -> str:
|
|
"""Extract dialect name from a Superset database backend URI or engine name.
|
|
|
|
Handles both URI formats (e.g. 'postgresql://...', 'clickhousedb://...')
|
|
and plain engine names (e.g. 'postgresql', 'clickhousedb') returned by Superset.
|
|
Normalises known variants like 'clickhousedb' → 'clickhouse'.
|
|
"""
|
|
if not backend or not backend.strip():
|
|
return "unknown"
|
|
try:
|
|
# Extract scheme from URI or use plain name
|
|
scheme = backend.split("://")[0]
|
|
dialect = scheme.split("+")[0]
|
|
raw = dialect.strip().lower()
|
|
# Normalise known Superset backend variants
|
|
dialect_map = {
|
|
"clickhousedb": "clickhouse",
|
|
"greenplum": "postgresql",
|
|
}
|
|
return dialect_map.get(raw, raw)
|
|
except Exception:
|
|
return "unknown"
|
|
# #endregion _extract_dialect
|
|
|
|
|
|
# #region job_to_response [C:1] [TYPE Function]
|
|
# @BRIEF Convert a TranslationJob ORM model to a TranslateJobResponse schema with dictionary_ids.
|
|
def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> TranslateJobResponse:
|
|
return TranslateJobResponse(
|
|
id=job.id,
|
|
name=job.name,
|
|
description=job.description,
|
|
source_dialect=job.source_dialect,
|
|
target_dialect=job.target_dialect,
|
|
database_dialect=job.database_dialect,
|
|
source_datasource_id=job.source_datasource_id,
|
|
source_table=job.source_table,
|
|
target_schema=job.target_schema,
|
|
target_table=job.target_table,
|
|
source_key_cols=job.source_key_cols or [],
|
|
target_key_cols=job.target_key_cols or [],
|
|
translation_column=job.translation_column,
|
|
target_column=job.target_column,
|
|
target_language_column=job.target_language_column,
|
|
target_source_column=job.target_source_column,
|
|
target_source_language_column=job.target_source_language_column,
|
|
context_columns=job.context_columns or [],
|
|
target_languages=job.target_languages,
|
|
provider_id=job.provider_id,
|
|
batch_size=job.batch_size or 50,
|
|
upsert_strategy=job.upsert_strategy or "MERGE",
|
|
status=job.status,
|
|
created_by=job.created_by,
|
|
created_at=job.created_at,
|
|
updated_at=job.updated_at,
|
|
dictionary_ids=dict_ids or [],
|
|
environment_id=job.environment_id,
|
|
target_database_id=job.target_database_id,
|
|
)
|
|
# #endregion job_to_response
|
|
# #endregion ServiceUtils
|