refactor(connections): remove ConnectionConfig, migrate mapper to SQL Lab
Backend: - Remove ConnectionConfig model, CRUD routes (connections.py), and tests - Remove ensure_connection_configs_table from database.py - Migrate MapperPlugin from psycopg2 direct PostgreSQL to SupersetSqlLabExecutor - Add DatasetMapper.get_sqllab_mappings() with default information_schema query - Update MapColumnsRequest: connection_id → database_id + sql_query - Fix pydantic_settings v2 deprecation (Field(env=...) → validation_alias) - Clean up app.py, routes/__init__.py, database.py imports Frontend: - Remove DatabaseConnectionsTab, ConnectionForm, ConnectionList components - Remove /settings/connections route page and Navbar link - Remove connectionService.js, connections i18n files - Update MapperTool.svelte: postgres → sqllab source - Update Map Columns modal: database selector + SQL query instead of connection_id - Clean up settings page tabs, nav links, test mocks - Clean up i18n keys (settings, nav, mapper) Tests: 25/25 pass (14 030-feature + 11 route tests) Build: succeeds
This commit is contained in:
@@ -1,18 +1,16 @@
|
||||
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, excel, mapping]
|
||||
# @BRIEF Implements a plugin for mapping dataset columns using external database connections or Excel files.
|
||||
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, sqllab, excel, mapping]
|
||||
# @BRIEF Implements a plugin for mapping dataset columns using Superset SQL Lab or Excel files.
|
||||
# @LAYER: Plugins
|
||||
# @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool.
|
||||
# @RELATION Inherits from PluginBase. Uses DatasetMapper and SupersetSqlLabExecutor.
|
||||
# @RELATION USES -> TaskContext
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..core.database import SessionLocal
|
||||
from ..core.logger import belief_scope, logger
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.task_manager.context import TaskContext
|
||||
from ..core.utils.dataset_mapper import DatasetMapper
|
||||
from ..models.connection import ConnectionConfig
|
||||
|
||||
|
||||
# #region MapperPlugin [TYPE Class]
|
||||
@@ -52,7 +50,7 @@ class MapperPlugin(PluginBase):
|
||||
# @RETURN: str - Plugin description.
|
||||
def description(self) -> str:
|
||||
with belief_scope("description"):
|
||||
return "Map dataset column verbose names using PostgreSQL comments or Excel files."
|
||||
return "Map dataset column verbose names using Superset SQL Lab or Excel files."
|
||||
# endregion description
|
||||
|
||||
@property
|
||||
@@ -98,24 +96,19 @@ class MapperPlugin(PluginBase):
|
||||
"source": {
|
||||
"type": "string",
|
||||
"title": "Mapping Source",
|
||||
"enum": ["postgres", "excel"],
|
||||
"default": "postgres"
|
||||
"enum": ["sqllab", "excel"],
|
||||
"default": "sqllab"
|
||||
},
|
||||
"connection_id": {
|
||||
"type": "string",
|
||||
"title": "Saved Connection",
|
||||
"description": "The ID of a saved database connection (for postgres source)."
|
||||
"database_id": {
|
||||
"type": "integer",
|
||||
"title": "Superset Database ID",
|
||||
"description": "Target database ID in Superset (for sqllab source)."
|
||||
},
|
||||
"table_name": {
|
||||
"sql_query": {
|
||||
"type": "string",
|
||||
"title": "Table Name",
|
||||
"description": "Target table name in PostgreSQL."
|
||||
},
|
||||
"table_schema": {
|
||||
"type": "string",
|
||||
"title": "Table Schema",
|
||||
"description": "Target table schema in PostgreSQL.",
|
||||
"default": "public"
|
||||
"title": "SQL Query",
|
||||
"description": "Custom SQL query returning column_name and verbose_name columns. Default: information_schema.columns.",
|
||||
"default": ""
|
||||
},
|
||||
"excel_path": {
|
||||
"type": "string",
|
||||
@@ -145,7 +138,6 @@ class MapperPlugin(PluginBase):
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
db_log = log.with_source("postgres") if context else log
|
||||
|
||||
if not env_name or dataset_id is None or not source:
|
||||
log.error("Missing required parameters: env, dataset_id, source")
|
||||
@@ -162,46 +154,37 @@ class MapperPlugin(PluginBase):
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
|
||||
postgres_config = None
|
||||
if source == "postgres":
|
||||
connection_id = params.get("connection_id")
|
||||
if not connection_id:
|
||||
log.error("connection_id is required for postgres source.")
|
||||
raise ValueError("connection_id is required for postgres source.")
|
||||
|
||||
# Load connection from DB
|
||||
db = SessionLocal()
|
||||
try:
|
||||
conn_config = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first()
|
||||
if not conn_config:
|
||||
db_log.error(f"Connection {connection_id} not found.")
|
||||
raise ValueError(f"Connection {connection_id} not found.")
|
||||
|
||||
postgres_config = {
|
||||
'dbname': conn_config.database,
|
||||
'user': conn_config.username,
|
||||
'password': conn_config.password,
|
||||
'host': conn_config.host,
|
||||
'port': str(conn_config.port) if conn_config.port else '5432'
|
||||
}
|
||||
db_log.debug(f"Loaded connection config for {conn_config.host}:{conn_config.port}/{conn_config.database}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
log.info(f"Starting mapping for dataset {dataset_id} in {env_name}")
|
||||
|
||||
mapper = DatasetMapper()
|
||||
|
||||
try:
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source=source,
|
||||
postgres_config=postgres_config,
|
||||
excel_path=params.get("excel_path"),
|
||||
table_name=params.get("table_name"),
|
||||
table_schema=params.get("table_schema") or "public"
|
||||
)
|
||||
if source == "sqllab":
|
||||
# Use Superset SQL Lab to query database metadata
|
||||
from ..plugins.translate.superset_executor import SupersetSqlLabExecutor
|
||||
executor = SupersetSqlLabExecutor(config_manager, env_name)
|
||||
database_id = params.get("database_id")
|
||||
if not database_id:
|
||||
log.error("database_id is required for sqllab source.")
|
||||
raise ValueError("database_id is required for sqllab source.")
|
||||
executor.resolve_database_id(target_database_id=str(database_id))
|
||||
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source="sqllab",
|
||||
sqllab_executor=executor,
|
||||
database_id=database_id,
|
||||
sql_query=params.get("sql_query"),
|
||||
)
|
||||
else:
|
||||
# Excel source
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source="excel",
|
||||
excel_path=params.get("excel_path") or params.get("file_data")
|
||||
)
|
||||
superset_log.info(f"Mapping completed for dataset {dataset_id}")
|
||||
return {"status": "success", "dataset_id": dataset_id}
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user