feat: implement LLM table translation service with searchable datasource dropdown, i18n, sidebar, and RBAC
- Add backend plugin modules: preview, executor, orchestrator, events, sql_generator, superset_executor, dictionary, scheduler, metrics - Add API routes for translate jobs, runs, dictionaries, preview, schedule, metrics, corrections - Add ORM models (12 tables) and Pydantic schemas (15 DTOs) - Register translate router in app.py with RBAC permission guards - Add frontend pages: job list, job config, dictionary list/editor, history - Add 7 reusable Svelte 5 components: Preview, RunProgress, RunResult, TermCorrection, BulkCorrection, ScheduleConfig, MetricsDashboard - Add searchable datasource dropdown with Superset API integration - Add i18n (en/ru) with ~300 keys across common, config, jobs, dictionaries, preview namespaces - Add Translation sidebar category with Jobs/Dictionaries/History sub-items - Hide health monitor error toast via suppressToast API option - Add 69 backend tests and 44 frontend test files - Fix: SupersetClient env resolution (string -> Environment object) - Fix: Dataset detail API returns proper database dict - Fix: Database dialect extraction fallback when metadata incomplete
This commit is contained in:
319
backend/src/plugins/translate/sql_generator.py
Normal file
319
backend/src/plugins/translate/sql_generator.py
Normal file
@@ -0,0 +1,319 @@
|
||||
# [DEF:SQLGenerator:Module]
|
||||
# @COMPLEXITY: 3
|
||||
# @SEMANTICS: translate, sql, generator, dialect
|
||||
# @PURPOSE: Dialect-aware safe SQL generation for INSERT/UPSERT operations.
|
||||
# @LAYER: Domain
|
||||
# @RELATION: DEPENDS_ON -> [TranslationJob]
|
||||
# @RELATION: DEPENDS_ON -> [TranslationRun]
|
||||
# @PRE: Job has target_schema and target_table configured. Dialect is one of supported SUPPORTED_DIALECTS.
|
||||
# @POST: Returns safe SQL strings for the target dialect.
|
||||
# @SIDE_EFFECT: None — pure code generation.
|
||||
# @RATIONALE: Dialect-aware SQL uses ON CONFLICT for PostgreSQL; plain INSERT for ClickHouse with documented limitations.
|
||||
# @REJECTED: UPDATE statements — source is append-only; UPSERT covers overwrite case.
|
||||
# @REJECTED: ORM-based insert bypasses Superset's SQL Lab audit trail.
|
||||
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from ...core.logger import logger, belief_scope
|
||||
|
||||
# PostgreSQL/Greenplum dialects that support ON CONFLICT
|
||||
POSTGRESQL_DIALECTS = {"postgresql", "redshift", "greenplum"}
|
||||
# Dialects that use backtick or no quoting
|
||||
CLICKHOUSE_DIALECTS = {"clickhouse"}
|
||||
|
||||
|
||||
# [DEF:_quote_identifier:Function]
|
||||
# @PURPOSE: Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
|
||||
# @PRE: identifier is a non-empty string.
|
||||
# @POST: Returns safely quoted identifier.
|
||||
def _quote_identifier(identifier: str, dialect: str) -> str:
|
||||
"""Quote a SQL identifier per dialect rules."""
|
||||
if not identifier:
|
||||
return identifier
|
||||
# Remove any existing quotes to avoid double-quoting
|
||||
cleaned = identifier.strip().strip('"').strip('`').strip('[]')
|
||||
if dialect in POSTGRESQL_DIALECTS:
|
||||
return f'"{cleaned}"'
|
||||
elif dialect in CLICKHOUSE_DIALECTS:
|
||||
return f"`{cleaned}`"
|
||||
else:
|
||||
# Generic ANSI double-quote
|
||||
return f'"{cleaned}"'
|
||||
# [/DEF:_quote_identifier:Function]
|
||||
|
||||
|
||||
# [DEF:_encode_sql_value:Function]
|
||||
# @PURPOSE: Encode a Python value into a SQL-safe literal for INSERT VALUES.
|
||||
# @PRE: value is a Python primitive or None.
|
||||
# @POST: Returns SQL-safe string literal representation.
|
||||
def _encode_sql_value(value: Any) -> str:
|
||||
"""Encode a Python value into a SQL-safe literal."""
|
||||
if value is None:
|
||||
return "NULL"
|
||||
if isinstance(value, bool):
|
||||
return "TRUE" if value else "FALSE"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
# String — escape single quotes by doubling them
|
||||
escaped = str(value).replace("'", "''")
|
||||
return f"'{escaped}'"
|
||||
# [/DEF:_encode_sql_value:Function]
|
||||
|
||||
|
||||
# [DEF:_build_values_clause:Function]
|
||||
# @PURPOSE: Build a VALUES clause for multiple rows.
|
||||
# @PRE: columns list is non-empty; rows is a list of dicts.
|
||||
# @POST: Returns SQL VALUES clause string.
|
||||
def _build_values_clause(columns: List[str], rows: List[Dict[str, Any]]) -> str:
|
||||
"""Build VALUES (...) clause for multiple rows."""
|
||||
value_groups = []
|
||||
for row in rows:
|
||||
values = [_encode_sql_value(row.get(col)) for col in columns]
|
||||
value_groups.append(f"({', '.join(values)})")
|
||||
return ",\n".join(value_groups)
|
||||
# [/DEF:_build_values_clause:Function]
|
||||
|
||||
|
||||
# [DEF:generate_insert_sql:Function]
|
||||
# @PURPOSE: Generate a dialect-aware INSERT SQL statement.
|
||||
# @PRE: target_table, columns are non-empty.
|
||||
# @POST: Returns SQL string or raises ValueError on invalid input.
|
||||
def generate_insert_sql(
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""Generate a plain INSERT SQL."""
|
||||
with belief_scope("generate_insert_sql"):
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required for INSERT SQL generation")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required for INSERT SQL generation")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required for INSERT SQL generation")
|
||||
|
||||
col_list = ", ".join(columns)
|
||||
values = _build_values_clause(columns, rows)
|
||||
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
table_ref = f"{target_schema}.{target_table}"
|
||||
|
||||
sql = f"INSERT INTO {table_ref} ({col_list})\nVALUES\n{values};"
|
||||
return sql
|
||||
# [/DEF:generate_insert_sql:Function]
|
||||
|
||||
|
||||
# [DEF:generate_upsert_sql:Function]
|
||||
# @PURPOSE: Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
|
||||
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
|
||||
# @POST: Returns UPSERT SQL string or raises ValueError.
|
||||
def generate_upsert_sql(
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
key_columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
) -> str:
|
||||
"""Generate INSERT ... ON CONFLICT (key_cols) DO UPDATE SET ... SQL."""
|
||||
with belief_scope("generate_upsert_sql"):
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required for UPSERT SQL generation")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required for UPSERT SQL generation")
|
||||
if not key_columns:
|
||||
raise ValueError("key_columns are required for UPSERT SQL generation")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required for UPSERT SQL generation")
|
||||
|
||||
col_list = ", ".join(columns)
|
||||
key_list = ", ".join(key_columns)
|
||||
values = _build_values_clause(columns, rows)
|
||||
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
table_ref = f"{target_schema}.{target_table}"
|
||||
|
||||
# Build SET clause: exclude key columns from update
|
||||
update_cols = [c for c in columns if c not in key_columns]
|
||||
if not update_cols:
|
||||
# If only key columns, use DO NOTHING
|
||||
conflict_action = "DO NOTHING"
|
||||
else:
|
||||
set_parts = [f"{col} = EXCLUDED.{col}" for col in update_cols]
|
||||
conflict_action = "DO UPDATE SET\n" + ",\n".join(set_parts)
|
||||
|
||||
sql = (
|
||||
f"INSERT INTO {table_ref} ({col_list})\n"
|
||||
f"VALUES\n"
|
||||
f"{values}\n"
|
||||
f"ON CONFLICT ({key_list}) {conflict_action};"
|
||||
)
|
||||
return sql
|
||||
# [/DEF:generate_upsert_sql:Function]
|
||||
|
||||
|
||||
# [DEF:SQLGenerator:Class]
|
||||
# @COMPLEXITY: 3
|
||||
# @PURPOSE: Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
|
||||
# @PRE: Job has target_schema, target_table, key columns configured.
|
||||
# @POST: Returns generated SQL string for the target dialect.
|
||||
class SQLGenerator:
|
||||
|
||||
# [DEF:SQLGenerator.generate:Function]
|
||||
# @PURPOSE: Generate SQL for a set of rows, detecting dialect from the job configuration.
|
||||
# @PRE: dialect is a supported database dialect. columns list is non-empty. rows is non-empty.
|
||||
# @POST: Returns tuple of (sql_string, statement_count).
|
||||
# @SIDE_EFFECT: None — pure SQL generation.
|
||||
@staticmethod
|
||||
def generate(
|
||||
dialect: str,
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
key_columns: Optional[List[str]] = None,
|
||||
upsert_strategy: str = "MERGE",
|
||||
) -> Tuple[str, int]:
|
||||
"""Generate dialect-appropriate INSERT/UPSERT SQL.
|
||||
|
||||
Args:
|
||||
dialect: Target database dialect (e.g. 'postgresql', 'clickhouse').
|
||||
target_schema: Optional schema name.
|
||||
target_table: Target table name.
|
||||
columns: List of column names to insert.
|
||||
rows: List of row dicts with column values.
|
||||
key_columns: Key columns for conflict resolution (UPSERT).
|
||||
upsert_strategy: 'MERGE' (UPSERT), 'INSERT' (plain INSERT).
|
||||
|
||||
Returns:
|
||||
Tuple of (sql_string, row_count).
|
||||
"""
|
||||
with belief_scope("SQLGenerator.generate"):
|
||||
logger.reason("Generating SQL", {
|
||||
"dialect": dialect,
|
||||
"schema": target_schema,
|
||||
"table": target_table,
|
||||
"columns": len(columns),
|
||||
"rows": len(rows),
|
||||
"strategy": upsert_strategy,
|
||||
})
|
||||
|
||||
# Validate inputs
|
||||
if not target_table:
|
||||
raise ValueError("target_table is required")
|
||||
if not columns:
|
||||
raise ValueError("At least one column is required")
|
||||
if not rows:
|
||||
raise ValueError("At least one row is required")
|
||||
|
||||
# Build fully qualified table reference
|
||||
table_ref = target_table
|
||||
if target_schema:
|
||||
quoted_schema = _quote_identifier(target_schema, dialect)
|
||||
quoted_table = _quote_identifier(target_table, dialect)
|
||||
table_ref = f"{quoted_schema}.{quoted_table}"
|
||||
else:
|
||||
table_ref = _quote_identifier(target_table, dialect)
|
||||
|
||||
# Quote columns per dialect
|
||||
quoted_columns = [_quote_identifier(c, dialect) for c in columns]
|
||||
quoted_key_columns = (
|
||||
[_quote_identifier(k, dialect) for k in key_columns]
|
||||
if key_columns
|
||||
else []
|
||||
)
|
||||
|
||||
# Generate SQL per dialect and strategy
|
||||
use_upsert = upsert_strategy.upper() == "MERGE" and key_columns
|
||||
|
||||
if dialect in POSTGRESQL_DIALECTS or dialect not in CLICKHOUSE_DIALECTS:
|
||||
# PostgreSQL and other ANSI dialects: support UPSERT via ON CONFLICT
|
||||
if use_upsert:
|
||||
sql = generate_upsert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
key_columns=quoted_key_columns,
|
||||
rows=rows,
|
||||
)
|
||||
else:
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
elif dialect in CLICKHOUSE_DIALECTS:
|
||||
# ClickHouse: plain INSERT, no ON CONFLICT support
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
if use_upsert:
|
||||
logger.reason("ClickHouse UPSERT not supported; using plain INSERT", {
|
||||
"note": "ClickHouse does not support ON CONFLICT. Use ReplacingMergeTree for dedup.",
|
||||
})
|
||||
else:
|
||||
# Fallback: plain INSERT
|
||||
sql = generate_insert_sql(
|
||||
target_schema=None,
|
||||
target_table=table_ref,
|
||||
columns=quoted_columns,
|
||||
rows=rows,
|
||||
)
|
||||
|
||||
logger.reflect("SQL generated", {
|
||||
"dialect": dialect,
|
||||
"row_count": len(rows),
|
||||
"sql_length": len(sql),
|
||||
})
|
||||
return sql, len(rows)
|
||||
# [/DEF:SQLGenerator.generate:Function]
|
||||
|
||||
# [DEF:SQLGenerator.generate_batch:Function]
|
||||
# @PURPOSE: Generate separate INSERT statements for each row (batch-safe version).
|
||||
# @PRE: Same as generate().
|
||||
# @POST: Returns list of (sql_string, row_index) tuples.
|
||||
@staticmethod
|
||||
def generate_batch(
|
||||
dialect: str,
|
||||
target_schema: Optional[str],
|
||||
target_table: str,
|
||||
columns: List[str],
|
||||
rows: List[Dict[str, Any]],
|
||||
key_columns: Optional[List[str]] = None,
|
||||
upsert_strategy: str = "MERGE",
|
||||
max_rows_per_statement: int = 500,
|
||||
) -> List[Tuple[str, int]]:
|
||||
"""Generate SQL in batches, splitting large row sets into multiple statements.
|
||||
|
||||
Returns:
|
||||
List of (sql_string, row_count) tuples.
|
||||
"""
|
||||
with belief_scope("SQLGenerator.generate_batch"):
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
statements = []
|
||||
for i in range(0, len(rows), max_rows_per_statement):
|
||||
chunk = rows[i:i + max_rows_per_statement]
|
||||
sql, count = SQLGenerator.generate(
|
||||
dialect=dialect,
|
||||
target_schema=target_schema,
|
||||
target_table=target_table,
|
||||
columns=columns,
|
||||
rows=chunk,
|
||||
key_columns=key_columns,
|
||||
upsert_strategy=upsert_strategy,
|
||||
)
|
||||
statements.append((sql, count))
|
||||
|
||||
return statements
|
||||
# [/DEF:SQLGenerator.generate_batch:Function]
|
||||
|
||||
|
||||
# [/DEF:SQLGenerator:Class]
|
||||
# [/DEF:SQLGenerator:Module]
|
||||
Reference in New Issue
Block a user