refactor(connections): remove ConnectionConfig, migrate mapper to SQL Lab

Backend:
- Remove ConnectionConfig model, CRUD routes (connections.py), and tests
- Remove ensure_connection_configs_table from database.py
- Migrate MapperPlugin from psycopg2 direct PostgreSQL to SupersetSqlLabExecutor
- Add DatasetMapper.get_sqllab_mappings() with default information_schema query
- Update MapColumnsRequest: connection_id → database_id + sql_query
- Fix pydantic_settings v2 deprecation (Field(env=...) → validation_alias)
- Clean up app.py, routes/__init__.py, database.py imports

Frontend:
- Remove DatabaseConnectionsTab, ConnectionForm, ConnectionList components
- Remove /settings/connections route page and Navbar link
- Remove connectionService.js, connections i18n files
- Update MapperTool.svelte: postgres → sqllab source
- Update Map Columns modal: database selector + SQL query instead of connection_id
- Clean up settings page tabs, nav links, test mocks
- Clean up i18n keys (settings, nav, mapper)

Tests: 25/25 pass (14 030-feature + 11 route tests)
Build: succeeds
This commit is contained in:
2026-05-21 18:47:41 +03:00
parent b529e8082a
commit 36643024cf
35 changed files with 269 additions and 1315 deletions

View File

@@ -12,7 +12,6 @@
# @RELATION DEPENDS_ON -> [PluginsRouter]
# @RELATION DEPENDS_ON -> [TasksRouter]
# @RELATION DEPENDS_ON -> [SettingsRouter]
# @RELATION DEPENDS_ON -> [ConnectionsRouter]
# @RELATION DEPENDS_ON -> [ReportsRouter]
# @RELATION DEPENDS_ON -> [LlmRoutes]
# @SIDE_EFFECT: Registers route group imports via __getattr__
@@ -22,7 +21,6 @@ __all__ = [
"assistant",
"clean_release",
"clean_release_v2",
"connections",
"dashboards",
"dataset_review",
"datasets",

View File

@@ -1,5 +1,11 @@
# #region RoutesTestsConftest [TYPE Module] [C:1] [SEMANTICS test, fixture, mock, conftest]
# @BRIEF Shared low-fidelity test doubles for API route test modules.
# Set required env vars before any app imports
import os
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
os.environ.setdefault("DEV_MODE", "true")
class FakeQuery:
"""Shared chainable query stub for route tests.
WARNING: filter() is predicate-blind — all ownership and permission filters are

View File

@@ -1,69 +0,0 @@
# #region ConnectionsRoutesTests [TYPE Module] [C:3] [SEMANTICS test, connection, route, crud, bootstrap]
# @BRIEF Verifies connection routes bootstrap their table before CRUD access.
# @LAYER: API
# @RELATION DEPENDS_ON -> ConnectionsRouter
import asyncio
import os
from pathlib import Path
import pytest
import sys
from sqlalchemy import create_engine, inspect
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool
# Force SQLite in-memory for database module imports.
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["TASKS_DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["AUTH_DATABASE_URL"] = "sqlite:///:memory:"
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
os.environ["ENVIRONMENT"] = "testing"
backend_dir = str(Path(__file__).parent.parent.parent.parent.resolve())
if backend_dir not in sys.path:
sys.path.insert(0, backend_dir)
@pytest.fixture
def db_session():
engine = create_engine(
"sqlite:///:memory:",
connect_args={"check_same_thread": False},
poolclass=StaticPool,
)
session = sessionmaker(bind=engine)()
try:
yield session
finally:
session.close()
# #region test_list_connections_bootstraps_missing_table [TYPE Function]
# @RELATION BINDS_TO -> ConnectionsRoutesTests
# @BRIEF Ensure listing connections auto-creates missing table and returns empty payload.
def test_list_connections_bootstraps_missing_table(db_session):
from src.api.routes.connections import list_connections
result = asyncio.run(list_connections(db=db_session))
inspector = inspect(db_session.get_bind())
assert result == []
assert "connection_configs" in inspector.get_table_names()
# #endregion test_list_connections_bootstraps_missing_table
# #region test_create_connection_bootstraps_missing_table [TYPE Function]
# @RELATION BINDS_TO -> ConnectionsRoutesTests
# @BRIEF Ensure connection creation bootstraps table and persists returned connection fields.
def test_create_connection_bootstraps_missing_table(db_session):
from src.api.routes.connections import ConnectionCreate, create_connection
payload = ConnectionCreate(
name="Analytics Warehouse",
type="postgres",
host="warehouse.internal",
port=5432,
database="analytics",
username="reporter",
password="secret",
)
created = asyncio.run(create_connection(connection=payload, db=db_session))
inspector = inspect(db_session.get_bind())
assert created.name == "Analytics Warehouse"
assert created.host == "warehouse.internal"
assert "connection_configs" in inspector.get_table_names()
# #endregion test_create_connection_bootstraps_missing_table
# #endregion ConnectionsRoutesTests

View File

@@ -132,7 +132,7 @@ def test_get_datasets_invalid_pagination(mock_deps):
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
# @BRIEF Validate map-columns request creates an async mapping task and returns its identifier.
# @TEST: POST /api/datasets/map-columns creates mapping task
# @PRE: Valid env_id, dataset_ids, source_type
# @PRE: Valid env_id, dataset_ids, source_type (sqllab)
# @POST: Returns task_id
def test_map_columns_success(mock_deps):
# Mock environment
@@ -145,7 +145,7 @@ def test_map_columns_success(mock_deps):
mock_deps["task"].create_task = AsyncMock(return_value=mock_task)
response = client.post(
"/api/datasets/map-columns",
json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "postgresql"},
json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "sqllab", "database_id": 1},
)
assert response.status_code == 200
data = response.json()
@@ -157,7 +157,7 @@ def test_map_columns_success(mock_deps):
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
# @BRIEF Validate map-columns rejects unsupported source types with a 400 contract response.
# @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type
# @PRE: source_type is not 'postgresql' or 'xlsx'
# @PRE: source_type is not 'sqllab' or 'xlsx'
# @POST: Returns 400 error
def test_map_columns_invalid_source_type(mock_deps):
response = client.post(
@@ -165,7 +165,7 @@ def test_map_columns_invalid_source_type(mock_deps):
json={"env_id": "prod", "dataset_ids": [1], "source_type": "invalid"},
)
assert response.status_code == 400
assert "Source type must be 'postgresql' or 'xlsx'" in response.json()["detail"]
assert "Source type must be 'sqllab' or 'xlsx'" in response.json()["detail"]
# #endregion test_map_columns_invalid_source_type
# #region test_generate_docs_success [TYPE Function]
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
@@ -202,11 +202,24 @@ def test_map_columns_empty_ids(mock_deps):
"""@PRE: dataset_ids must be non-empty."""
response = client.post(
"/api/datasets/map-columns",
json={"env_id": "prod", "dataset_ids": [], "source_type": "postgresql"},
json={"env_id": "prod", "dataset_ids": [], "source_type": "sqllab", "database_id": 1},
)
assert response.status_code == 400
assert "At least one dataset ID must be provided" in response.json()["detail"]
# #endregion test_map_columns_empty_ids
# #region test_map_columns_missing_database_id [TYPE Function]
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
# @BRIEF Validate map-columns rejects sqllab source without database_id.
# @TEST: POST /api/datasets/map-columns returns 400 for sqllab without database_id
# @POST: Returns 400 error
def test_map_columns_missing_database_id(mock_deps):
response = client.post(
"/api/datasets/map-columns",
json={"env_id": "prod", "dataset_ids": [1], "source_type": "sqllab"},
)
assert response.status_code == 400
assert "database_id is required" in response.json()["detail"]
# #endregion test_map_columns_missing_database_id
# #region test_generate_docs_empty_ids [TYPE Function]
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
# @BRIEF Validate generate-docs rejects empty dataset identifier lists.

View File

@@ -1,141 +0,0 @@
# #region ConnectionsRouter [C:3] [TYPE Module] [SEMANTICS fastapi, connection, api, search]
# @BRIEF Defines the FastAPI router for managing external database connections.
# @LAYER: API
# @RELATION DEPENDS_ON -> [get_db]
# @RELATION DEPENDS_ON -> [ConnectionConfig]
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel
from sqlalchemy.orm import Session
from ...core.database import ensure_connection_configs_table, get_db
from ...core.logger import belief_scope, logger
from ...models.connection import ConnectionConfig
router = APIRouter()
# #region _ensure_connections_schema [C:3] [TYPE Function]
# @BRIEF Ensures the connection_configs table exists before CRUD access.
# @PRE: db is an active SQLAlchemy session.
# @POST: The current bind can safely query ConnectionConfig.
# @RELATION CALLS -> ensure_connection_configs_table
def _ensure_connections_schema(db: Session):
with belief_scope("ConnectionsRouter.ensure_schema"):
ensure_connection_configs_table(db.get_bind())
# #endregion _ensure_connections_schema
# #region ConnectionSchema [C:3] [TYPE Class]
# @BRIEF Pydantic model for connection response.
# @RELATION BINDS_TO -> ConnectionConfig
class ConnectionSchema(BaseModel):
id: str
name: str
type: str
host: str | None = None
port: int | None = None
database: str | None = None
username: str | None = None
created_at: datetime
class Config:
orm_mode = True
# #endregion ConnectionSchema
# #region ConnectionCreate [C:3] [TYPE Class]
# @BRIEF Pydantic model for creating a connection.
# @RELATION BINDS_TO -> ConnectionConfig
class ConnectionCreate(BaseModel):
name: str
type: str
host: str | None = None
port: int | None = None
database: str | None = None
username: str | None = None
password: str | None = None
# #endregion ConnectionCreate
# #region list_connections [C:3] [TYPE Function]
# @BRIEF Lists all saved connections.
# @PRE: Database session is active.
# @POST: Returns list of connection configs.
# @RELATION CALLS -> _ensure_connections_schema
# @RELATION DEPENDS_ON -> ConnectionConfig
@router.get("", response_model=list[ConnectionSchema])
async def list_connections(db: Session = Depends(get_db)):
with belief_scope("ConnectionsRouter.list_connections"):
_ensure_connections_schema(db)
connections = db.query(ConnectionConfig).all()
return connections
# #endregion list_connections
# #region create_connection [C:3] [TYPE Function]
# @BRIEF Creates a new connection configuration.
# @PRE: Connection name is unique.
# @POST: Connection is saved to DB.
# @RELATION CALLS -> _ensure_connections_schema
# @RELATION DEPENDS_ON -> ConnectionConfig
@router.post("", response_model=ConnectionSchema, status_code=status.HTTP_201_CREATED)
async def create_connection(
connection: ConnectionCreate, db: Session = Depends(get_db)
):
with belief_scope("ConnectionsRouter.create_connection", f"name={connection.name}"):
_ensure_connections_schema(db)
db_connection = ConnectionConfig(**connection.dict())
db.add(db_connection)
db.commit()
db.refresh(db_connection)
logger.info(
f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}"
)
return db_connection
# #endregion create_connection
# #region delete_connection [C:3] [TYPE Function]
# @BRIEF Deletes a connection configuration.
# @PRE: Connection ID exists.
# @POST: Connection is removed from DB.
# @RELATION CALLS -> _ensure_connections_schema
# @RELATION DEPENDS_ON -> ConnectionConfig
@router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT)
async def delete_connection(connection_id: str, db: Session = Depends(get_db)):
with belief_scope("ConnectionsRouter.delete_connection", f"id={connection_id}"):
_ensure_connections_schema(db)
db_connection = (
db.query(ConnectionConfig)
.filter(ConnectionConfig.id == connection_id)
.first()
)
if not db_connection:
logger.error(
f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found"
)
raise HTTPException(status_code=404, detail="Connection not found")
db.delete(db_connection)
db.commit()
logger.info(
f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}"
)
return
# #endregion delete_connection
# #endregion ConnectionsRouter

View File

@@ -316,8 +316,9 @@ async def get_datasets(
class MapColumnsRequest(BaseModel):
env_id: str = Field(..., description="Environment ID")
dataset_ids: list[int] = Field(..., description="List of dataset IDs to map")
source_type: str = Field(..., description="Source type: 'postgresql' or 'xlsx'")
connection_id: str | None = Field(None, description="Connection ID for PostgreSQL source")
source_type: str = Field(..., description="Source type: 'sqllab' or 'xlsx'")
database_id: int | None = Field(None, description="Superset database ID for SQL Lab source")
sql_query: str | None = Field(None, description="Custom SQL query for SQL Lab (optional, defaults to information_schema.columns query)")
file_data: str | None = Field(None, description="File path or data for XLSX source")
# #endregion MapColumnsRequest
@@ -344,9 +345,13 @@ async def map_columns(
raise HTTPException(status_code=400, detail="At least one dataset ID must be provided")
# Validate source type
if request.source_type not in ['postgresql', 'xlsx']:
if request.source_type not in ['sqllab', 'xlsx']:
logger.error(f"[map_columns][Coherence:Failed] Invalid source type: {request.source_type}")
raise HTTPException(status_code=400, detail="Source type must be 'postgresql' or 'xlsx'")
raise HTTPException(status_code=400, detail="Source type must be 'sqllab' or 'xlsx'")
# Validate database_id for sqllab source
if request.source_type == 'sqllab' and not request.database_id:
raise HTTPException(status_code=400, detail="database_id is required for 'sqllab' source type")
# Validate environment exists
environments = config_manager.get_environments()
@@ -361,8 +366,9 @@ async def map_columns(
task_params = {
'env': request.env_id,
'dataset_id': request.dataset_ids[0] if request.dataset_ids else None,
'source': request.source_type,
'connection_id': request.connection_id,
'source': 'sqllab' if request.source_type == 'sqllab' else 'excel',
'database_id': request.database_id,
'sql_query': request.sql_query,
'file_data': request.file_data
}

View File

@@ -28,7 +28,6 @@ from .api.routes import (
assistant,
clean_release,
clean_release_v2,
connections,
dashboards,
dataset_review,
datasets,
@@ -215,7 +214,6 @@ app.add_middleware(TraceContextMiddleware)
# @RELATION DEPENDS_ON -> [PluginsRouter]
# @RELATION DEPENDS_ON -> [TasksRouter]
# @RELATION DEPENDS_ON -> [SettingsRouter]
# @RELATION DEPENDS_ON -> [ConnectionsRouter]
# @RELATION DEPENDS_ON -> [ReportsRouter]
# @RELATION DEPENDS_ON -> [LlmRoutes]
# @RELATION DEPENDS_ON -> [CleanReleaseV2Api]
@@ -225,9 +223,6 @@ app.include_router(admin.router)
app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"])
app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"])
app.include_router(settings.router, prefix="/api/settings", tags=["Settings"])
app.include_router(
connections.router, prefix="/api/settings/connections", tags=["Connections"]
)
app.include_router(environments.router, tags=["Environments"])
app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"])
app.include_router(migration.router)

View File

@@ -14,7 +14,7 @@
import os
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings
from pydantic_settings import BaseSettings, SettingsConfigDict
# #region AuthConfig [TYPE Class]
@@ -23,19 +23,21 @@ from pydantic_settings import BaseSettings
# @POST: Returns a configuration object with validated settings.
# @RELATION INHERITS -> pydantic_settings.BaseSettings
class AuthConfig(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
# JWT Settings
SECRET_KEY: str = Field(default="", env="AUTH_SECRET_KEY")
SECRET_KEY: str = Field(default="", validation_alias="AUTH_SECRET_KEY")
ALGORITHM: str = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES: int = 480
REFRESH_TOKEN_EXPIRE_DAYS: int = 7
# Database Settings
AUTH_DATABASE_URL: str = Field(default="", env="AUTH_DATABASE_URL")
AUTH_DATABASE_URL: str = Field(default="", validation_alias="AUTH_DATABASE_URL")
# ADFS Settings
ADFS_CLIENT_ID: str = Field(default="", env="ADFS_CLIENT_ID")
ADFS_CLIENT_SECRET: str = Field(default="", env="ADFS_CLIENT_SECRET")
ADFS_METADATA_URL: str = Field(default="", env="ADFS_METADATA_URL")
ADFS_CLIENT_ID: str = Field(default="", validation_alias="ADFS_CLIENT_ID")
ADFS_CLIENT_SECRET: str = Field(default="", validation_alias="ADFS_CLIENT_SECRET")
ADFS_METADATA_URL: str = Field(default="", validation_alias="ADFS_METADATA_URL")
@field_validator("SECRET_KEY", mode="after")
@classmethod
@@ -60,9 +62,6 @@ class AuthConfig(BaseSettings):
"Set it in .env or export it before starting the server."
)
class Config:
env_file = ".env"
extra = "ignore"
# #endregion AuthConfig
# #region auth_config [TYPE Variable]

View File

@@ -4,7 +4,6 @@
# @LAYER: Infrastructure
# @RELATION DEPENDS_ON -> [MappingModels]
# @RELATION DEPENDS_ON -> [auth_config]
# @RELATION DEPENDS_ON -> [ConnectionConfig]
#
# @INVARIANT: A single engine instance is used for the entire application.
@@ -20,13 +19,11 @@ from ..models import (
auth as _auth_models, # noqa: F401
clean_release as _clean_release_models, # noqa: F401
config as _config_models, # noqa: F401
connection as _connection_models, # noqa: F401
dataset_review as _dataset_review_models, # noqa: F401
llm as _llm_models, # noqa: F401
profile as _profile_models, # noqa: F401
task as _task_models, # noqa: F401
)
from ..models.connection import ConnectionConfig
from ..models.mapping import Base
from .auth.config import auth_config
from .logger import belief_scope, logger
@@ -367,26 +364,6 @@ def _ensure_auth_users_columns(bind_engine):
# #endregion _ensure_auth_users_columns
# #region ensure_connection_configs_table [C:3] [TYPE Function]
# @BRIEF Ensures the external connection registry table exists in the main database.
# @PRE: bind_engine points to the application database.
# @POST: connection_configs table exists without dropping existing data.
# @RELATION DEPENDS_ON -> [ConnectionConfig]
def ensure_connection_configs_table(bind_engine):
with belief_scope("ensure_connection_configs_table"):
try:
ConnectionConfig.__table__.create(bind=bind_engine, checkfirst=True)
except Exception as migration_error:
logger.explore(
"ConnectionConfig table ensure failed",
extra={"src": "database", "error": str(migration_error)},
)
raise
# #endregion ensure_connection_configs_table
# #region _ensure_filter_source_enum_values [C:3] [TYPE Function]
# @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type.
# @PRE: bind_engine points to application database with imported_filters table.
@@ -686,7 +663,6 @@ def _ensure_dictionary_entries_columns(bind_engine):
# @PRE: engine, tasks_engine and auth_engine are initialized.
# @POST: Database tables created in all databases.
# @SIDE_EFFECT: Creates physical database files if they don't exist.
# @RELATION CALLS -> [ensure_connection_configs_table]
# @RELATION CALLS -> [_ensure_filter_source_enum_values]
# @RELATION CALLS -> [_ensure_dataset_review_session_columns]
# @RELATION CALLS -> [_ensure_dictionary_entries_columns]
@@ -700,7 +676,6 @@ def init_db():
_ensure_user_dashboard_preferences_health_columns(engine)
_ensure_git_server_configs_columns(engine)
_ensure_auth_users_columns(auth_engine)
ensure_connection_configs_table(engine)
_ensure_filter_source_enum_values(engine)
_ensure_dataset_review_session_columns(engine)
_ensure_translation_jobs_columns(engine)

View File

@@ -1,21 +1,20 @@
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, superset, dataset-mapper]
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, superset, dataset-mapper, sqllab]
#
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_name) в датасетах Superset,
# извлекая их из Superset SQL Lab (любая БД, подключённая к Superset) или XLSX-файлов.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> backend.core.superset_client
# @RELATION DEPENDS_ON -> pandas
# @RELATION DEPENDS_ON -> psycopg2
# @PUBLIC_API: DatasetMapper
from typing import Any
import pandas as pd # type: ignore
import psycopg2 # type: ignore
from ..logger import belief_scope, logger as app_logger
# #region DatasetMapper [TYPE Class]
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
# @BRIEF Класс для маппинга и обновления verbose_name в датасетах Superset.
class DatasetMapper:
# #region __init__ [TYPE Function]
# @PURPOSE: Initializes the mapper.
@@ -23,77 +22,78 @@ class DatasetMapper:
def __init__(self):
pass
# #endregion __init__
# #region get_postgres_comments [TYPE Function]
# @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.
# @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
# @PRE: table_name и table_schema должны быть строками.
# @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
# @THROW: Exception - При ошибках подключения или выполнения запроса к БД.
# @PARAM: db_config (Dict) - Конфигурация для подключения к БД.
# @PARAM: table_name (str) - Имя таблицы.
# @PARAM: table_schema (str) - Схема таблицы.
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
def get_postgres_comments(self, db_config: dict, table_name: str, table_schema: str) -> dict[str, str]:
with belief_scope("Fetch comments from PostgreSQL"):
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
query = f"""
SELECT
cols.column_name,
CASE
WHEN pg_catalog.col_description(
(SELECT c.oid
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = cols.table_name
AND n.nspname = cols.table_schema),
cols.ordinal_position::int
) LIKE '%|%' THEN
split_part(
pg_catalog.col_description(
(SELECT c.oid
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = cols.table_name
AND n.nspname = cols.table_schema),
cols.ordinal_position::int
),
'|',
1
)
ELSE
pg_catalog.col_description(
(SELECT c.oid
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE c.relname = cols.table_name
AND n.nspname = cols.table_schema),
cols.ordinal_position::int
)
END AS column_comment
FROM
information_schema.columns cols
WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';
"""
comments = {}
try:
with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:
cursor.execute(query)
for row in cursor.fetchall():
if row[1]:
comments[row[0]] = row[1]
app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments))
except Exception as e:
app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True)
raise
return comments
# #endregion get_postgres_comments
# #region get_sqllab_mappings [TYPE Function]
# @PURPOSE: Извлекает маппинги column_name -> verbose_name через SQL Lab Superset.
# @PRE: sqllab_executor должен быть инициализирован с database_id.
# @PRE: dataset_id должен существовать в Superset.
# @POST: Возвращается словарь column_name -> verbose_name из результатов SQL-запроса.
# Если sql_query не указан, строится запрос к information_schema.columns.
# @PARAM: client (SupersetClient) - Авторизованный клиент Superset.
# @PARAM: dataset_id (int) - ID датасета (для получения table_name/schema).
# @PARAM: sqllab_executor (SupersetSqlLabExecutor) - Исполнитель SQL Lab.
# @PARAM: database_id (int) - ID базы данных в Superset.
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (должен вернуть column_name + verbose_name).
# @RETURN: Dict[str, str] - Словарь column_name -> verbose_name.
def get_sqllab_mappings(
self,
client: Any,
dataset_id: int,
sqllab_executor: Any,
database_id: int,
sql_query: str | None = None,
) -> dict[str, str]:
with belief_scope("Fetch mappings via SQL Lab"):
# Получаем датасет из Superset, чтобы узнать table_name и schema
dataset_response = client.get_dataset(dataset_id)
dataset_data = dataset_response.get("result", dataset_response)
table_name = dataset_data.get("table_name")
table_schema = dataset_data.get("schema") or "public"
database_name = dataset_data.get("database", {}).get("database_name", "")
app_logger.info(
"[get_sqllab_mappings] Dataset %d: %s.%s (db: %s)",
dataset_id, table_schema, table_name, database_name
)
# Если SQL-запрос не указан, строим дефолтный для information_schema
if not sql_query:
sql_query = (
"SELECT column_name, pg_catalog.col_description(\n"
" (quote_ident(table_schema) || '.' || quote_ident(table_name))::regclass::oid,\n"
" ordinal_position\n"
") AS verbose_name\n"
f"FROM information_schema.columns\n"
f"WHERE table_schema = '{table_schema}' AND table_name = '{table_name}';\n"
)
app_logger.info("[get_sqllab_mappings] Using default information_schema query for %s.%s", table_schema, table_name)
else:
app_logger.info("[get_sqllab_mappings] Using custom SQL query")
# Выполняем запрос через SQL Lab
app_logger.info("[get_sqllab_mappings] Executing SQL Lab query on database %d...", database_id)
result = sqllab_executor.execute_and_poll(sql_query, database_id)
rows = result.get("results") or result.get("data") or result.get("result", [])
app_logger.info("[get_sqllab_mappings] SQL Lab returned %d rows", len(rows))
mappings: dict[str, str] = {}
for row in rows:
col_name = row.get("column_name")
verbose_name = row.get("verbose_name")
if col_name and verbose_name:
mappings[col_name] = str(verbose_name)
app_logger.info("[get_sqllab_mappings] Extracted %d mappings", len(mappings))
return mappings
# #endregion get_sqllab_mappings
# #region load_excel_mappings [TYPE Function]
# @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
# @PURPOSE: Загружает маппинги column_name -> verbose_name из XLSX файла.
# @PRE: file_path должен указывать на существующий XLSX файл.
# @POST: Возвращается словарь с меппингами из файла.
# @POST: Возвращается словарь с маппингами из файла.
# @THROW: Exception - При ошибках чтения файла или парсинга.
# @PARAM: file_path (str) - Путь к XLSX файлу.
# @RETURN: Dict[str, str] - Словарь с меппингами.
# @RETURN: Dict[str, str] - Словарь с маппингами.
def load_excel_mappings(self, file_path: str) -> dict[str, str]:
with belief_scope("Load mappings from Excel"):
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
@@ -106,37 +106,48 @@ class DatasetMapper:
app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True)
raise
# #endregion load_excel_mappings
# #region run_mapping [TYPE Function]
# @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
# @PURPOSE: Основная функция для выполнения маппинга и обновления verbose_name датасета в Superset.
# @PRE: superset_client должен быть авторизован.
# @PRE: dataset_id должен быть существующим ID в Superset.
# @POST: Если найдены изменения, датасет в Superset обновлен через API.
# @RELATION: CALLS -> self.get_postgres_comments
# @RELATION: CALLS -> self.get_sqllab_mappings
# @RELATION: CALLS -> self.load_excel_mappings
# @RELATION: CALLS -> superset_client.get_dataset
# @RELATION: CALLS -> superset_client.update_dataset
# @PARAM: superset_client (Any) - Клиент Superset.
# @PARAM: dataset_id (int) - ID датасета для обновления.
# @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').
# @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.
# @PARAM: source (str) - Источник данных ('sqllab' или 'excel').
# @PARAM: sqllab_executor (Optional[Any]) - Исполнитель SQL Lab (для sqllab source).
# @PARAM: database_id (Optional[int]) - ID базы данных Superset (для sqllab source).
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (для sqllab source).
# @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.
# @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: dict | None = None, excel_path: str | None = None, table_name: str | None = None, table_schema: str | None = None):
def run_mapping(
self,
superset_client: Any,
dataset_id: int,
source: str,
sqllab_executor: Any = None,
database_id: int | None = None,
sql_query: str | None = None,
excel_path: str | None = None,
):
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
mappings: dict[str, str] = {}
try:
if source in ['postgres', 'both']:
assert postgres_config and table_name and table_schema, "Postgres config is required."
mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))
if source in ['excel', 'both']:
assert excel_path, "Excel path is required."
if source == "sqllab":
assert sqllab_executor and database_id, "sqllab_executor and database_id are required for sqllab source."
mappings.update(self.get_sqllab_mappings(superset_client, dataset_id, sqllab_executor, database_id, sql_query))
elif source == "excel":
assert excel_path, "excel_path is required."
mappings.update(self.load_excel_mappings(excel_path))
if source not in ['postgres', 'excel', 'both']:
else:
app_logger.error("[run_mapping][Failure] Invalid source: %s.", source)
return
dataset_response = superset_client.get_dataset(dataset_id)
dataset_data = dataset_response['result']
@@ -171,6 +182,7 @@ class DatasetMapper:
changes_made = True
updated_columns.append(new_column)
updated_metrics = []
for metric in dataset_data.get("metrics", []):
new_metric = {
@@ -187,6 +199,7 @@ class DatasetMapper:
"uuid": metric.get("uuid"),
}
updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})
if changes_made:
payload_for_update = {
"database_id": dataset_data.get("database", {}).get("id"),
@@ -222,4 +235,4 @@ class DatasetMapper:
return
# #endregion run_mapping
# #endregion DatasetMapper
# #endregion DatasetMapperModule
# #endregion DatasetMapperModule

View File

@@ -1,34 +0,0 @@
# #region ConnectionModels [C:1] [TYPE Module] [SEMANTICS sqlalchemy, connection, model, schema, connection-config]
#
# @BRIEF Defines the database schema for external database connection configurations.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> sqlalchemy
#
# @INVARIANT: All primary keys are UUID strings.
import uuid
from sqlalchemy import Column, DateTime, Integer, String
from sqlalchemy.sql import func
from .mapping import Base
# #region ConnectionConfig [C:1] [TYPE Class]
# @BRIEF Stores credentials for external databases used for column mapping.
class ConnectionConfig(Base):
__tablename__ = "connection_configs"
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
name = Column(String, nullable=False)
type = Column(String, nullable=False) # e.g., "postgres"
host = Column(String, nullable=True)
port = Column(Integer, nullable=True)
database = Column(String, nullable=True)
username = Column(String, nullable=True)
password = Column(String, nullable=True) # Encrypted/Obfuscated password
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
# #endregion ConnectionConfig
# #endregion ConnectionModels

View File

@@ -1,18 +1,16 @@
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, excel, mapping]
# @BRIEF Implements a plugin for mapping dataset columns using external database connections or Excel files.
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, sqllab, excel, mapping]
# @BRIEF Implements a plugin for mapping dataset columns using Superset SQL Lab or Excel files.
# @LAYER: Plugins
# @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool.
# @RELATION Inherits from PluginBase. Uses DatasetMapper and SupersetSqlLabExecutor.
# @RELATION USES -> TaskContext
from typing import Any
from ..core.database import SessionLocal
from ..core.logger import belief_scope, logger
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.task_manager.context import TaskContext
from ..core.utils.dataset_mapper import DatasetMapper
from ..models.connection import ConnectionConfig
# #region MapperPlugin [TYPE Class]
@@ -52,7 +50,7 @@ class MapperPlugin(PluginBase):
# @RETURN: str - Plugin description.
def description(self) -> str:
with belief_scope("description"):
return "Map dataset column verbose names using PostgreSQL comments or Excel files."
return "Map dataset column verbose names using Superset SQL Lab or Excel files."
# endregion description
@property
@@ -98,24 +96,19 @@ class MapperPlugin(PluginBase):
"source": {
"type": "string",
"title": "Mapping Source",
"enum": ["postgres", "excel"],
"default": "postgres"
"enum": ["sqllab", "excel"],
"default": "sqllab"
},
"connection_id": {
"type": "string",
"title": "Saved Connection",
"description": "The ID of a saved database connection (for postgres source)."
"database_id": {
"type": "integer",
"title": "Superset Database ID",
"description": "Target database ID in Superset (for sqllab source)."
},
"table_name": {
"sql_query": {
"type": "string",
"title": "Table Name",
"description": "Target table name in PostgreSQL."
},
"table_schema": {
"type": "string",
"title": "Table Schema",
"description": "Target table schema in PostgreSQL.",
"default": "public"
"title": "SQL Query",
"description": "Custom SQL query returning column_name and verbose_name columns. Default: information_schema.columns.",
"default": ""
},
"excel_path": {
"type": "string",
@@ -145,7 +138,6 @@ class MapperPlugin(PluginBase):
# Create sub-loggers for different components
superset_log = log.with_source("superset_api") if context else log
db_log = log.with_source("postgres") if context else log
if not env_name or dataset_id is None or not source:
log.error("Missing required parameters: env, dataset_id, source")
@@ -162,46 +154,37 @@ class MapperPlugin(PluginBase):
client = SupersetClient(env_config)
client.authenticate()
postgres_config = None
if source == "postgres":
connection_id = params.get("connection_id")
if not connection_id:
log.error("connection_id is required for postgres source.")
raise ValueError("connection_id is required for postgres source.")
# Load connection from DB
db = SessionLocal()
try:
conn_config = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first()
if not conn_config:
db_log.error(f"Connection {connection_id} not found.")
raise ValueError(f"Connection {connection_id} not found.")
postgres_config = {
'dbname': conn_config.database,
'user': conn_config.username,
'password': conn_config.password,
'host': conn_config.host,
'port': str(conn_config.port) if conn_config.port else '5432'
}
db_log.debug(f"Loaded connection config for {conn_config.host}:{conn_config.port}/{conn_config.database}")
finally:
db.close()
log.info(f"Starting mapping for dataset {dataset_id} in {env_name}")
mapper = DatasetMapper()
try:
mapper.run_mapping(
superset_client=client,
dataset_id=dataset_id,
source=source,
postgres_config=postgres_config,
excel_path=params.get("excel_path"),
table_name=params.get("table_name"),
table_schema=params.get("table_schema") or "public"
)
if source == "sqllab":
# Use Superset SQL Lab to query database metadata
from ..plugins.translate.superset_executor import SupersetSqlLabExecutor
executor = SupersetSqlLabExecutor(config_manager, env_name)
database_id = params.get("database_id")
if not database_id:
log.error("database_id is required for sqllab source.")
raise ValueError("database_id is required for sqllab source.")
executor.resolve_database_id(target_database_id=str(database_id))
mapper.run_mapping(
superset_client=client,
dataset_id=dataset_id,
source="sqllab",
sqllab_executor=executor,
database_id=database_id,
sql_query=params.get("sql_query"),
)
else:
# Excel source
mapper.run_mapping(
superset_client=client,
dataset_id=dataset_id,
source="excel",
excel_path=params.get("excel_path") or params.get("file_data")
)
superset_log.info(f"Mapping completed for dataset {dataset_id}")
return {"status": "success", "dataset_id": dataset_id}
except Exception as e:

View File

@@ -54,7 +54,6 @@
</button>
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100 before:absolute before:-top-2 before:left-0 before:right-0 before:h-2 before:content-[''] right-0">
<a href="/settings" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_general}</a>
<a href="/settings/connections" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_connections}</a>
<a href="/settings/git" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_git}</a>
</div>
</div>

View File

@@ -1,95 +0,0 @@
<!-- #region ConnectionForm [C:3] [TYPE Component] [SEMANTICS connection, form, database, crud, create] -->
<!-- @BRIEF Component component: components/tools/ConnectionForm.svelte -->
<!-- @LAYER UI -->
<!--
@SEMANTICS: connection, form, settings
@PURPOSE: UI component for creating a new database connection configuration.
@LAYER: UI
@RELATION: USES -> frontend/src/services/connectionService.js
-->
<script>
// [SECTION: IMPORTS]
import { createConnection } from '../../services/connectionService.js';
import { addToast } from '../../lib/toasts.js';
import { t } from '../../lib/i18n';
import { Button, Input, Card } from '../../lib/ui';
// [/SECTION]
let { onsuccess = () => {} } = $props();
let name = $state('');
let type = $state('postgres');
let host = $state('');
let port = $state("5432");
let database = $state('');
let username = $state('');
let password = $state('');
let isSubmitting = $state(false);
// #region handleSubmit:Function [TYPE Function]
// @PURPOSE: Submits the connection form to the backend.
// @PRE: All required fields (name, host, database, username, password) must be filled.
// @POST: A new connection is created via the connection service and the parent success callback is invoked.
async function handleSubmit() {
if (!name || !host || !database || !username || !password) {
addToast($t.connections?.required_fields, 'warning');
return;
}
isSubmitting = true;
try {
const newConnection = await createConnection({
name, type, host, port: Number(port), database, username, password
});
addToast($t.connections?.created_success, 'success');
onsuccess(newConnection);
resetForm();
} catch (e) {
addToast(e.message, 'error');
} finally {
isSubmitting = false;
}
}
// #endregion handleSubmit:Function
// #region resetForm:Function [TYPE Function]
/* @PURPOSE: Resets the connection form fields to their default values.
@PRE: None.
@POST: All form input variables are reset.
*/
function resetForm() {
name = '';
host = '';
port = "5432";
database = '';
username = '';
password = '';
}
// #endregion resetForm:Function
</script>
<!-- [SECTION: TEMPLATE] -->
<Card title={$t.connections?.add_new}>
<form onsubmit={(event) => { event.preventDefault(); handleSubmit(); }} class="space-y-6">
<Input label={$t.connections?.name} bind:value={name} placeholder={$t.connections?.name_placeholder} />
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<Input label={$t.connections?.host} bind:value={host} placeholder={$t.connections?.host_placeholder} />
<Input label={$t.connections?.port} type="number" bind:value={port} />
</div>
<Input label={$t.connections?.db_name} bind:value={database} />
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<Input label={$t.connections?.user} bind:value={username} />
<Input label={$t.connections?.pass} type="password" bind:value={password} />
</div>
<div class="flex justify-end pt-2">
<Button type="submit" disabled={isSubmitting} isLoading={isSubmitting}>
{$t.connections?.create}
</Button>
</div>
</form>
</Card>
<!-- [/SECTION] -->
<!-- #endregion ConnectionForm -->

View File

@@ -1,88 +0,0 @@
<!-- #region ConnectionList [C:3] [TYPE Component] [SEMANTICS connection, list, crud, delete, manage] -->
<!-- @BRIEF Component component: components/tools/ConnectionList.svelte -->
<!-- @LAYER UI -->
<!--
@SEMANTICS: connection, list, settings
@PURPOSE: UI component for listing and deleting saved database connection configurations.
@LAYER: UI
@RELATION: USES -> frontend/src/services/connectionService.js
-->
<script>
// [SECTION: IMPORTS]
import { onMount } from 'svelte';
import { getConnections, deleteConnection } from '../../services/connectionService.js';
import { addToast } from '../../lib/toasts.js';
import { t } from '../../lib/i18n';
import { Button, Card } from '../../lib/ui';
// [/SECTION]
let connections = $state([]);
let isLoading = $state(true);
// #region fetchConnections:Function [TYPE Function]
// @PURPOSE: Fetches the list of connections from the backend.
// @PRE: None.
// @POST: connections array is populated.
async function fetchConnections() {
isLoading = true;
try {
connections = await getConnections();
} catch (e) {
addToast($t.connections?.fetch_failed, 'error');
} finally {
isLoading = false;
}
}
// #endregion fetchConnections:Function
// #region handleDelete:Function [TYPE Function]
// @PURPOSE: Deletes a connection configuration.
// @PRE: id is provided and user confirms deletion.
// @POST: Connection is deleted from backend and list is reloaded.
async function handleDelete(id) {
if (!confirm($t.connections?.delete_confirm)) return;
try {
await deleteConnection(id);
addToast($t.connections?.deleted_success, 'success');
await fetchConnections();
} catch (e) {
addToast(e.message, 'error');
}
}
// #endregion handleDelete:Function
onMount(fetchConnections);
// Expose fetchConnections to parent
export { fetchConnections };
</script>
<!-- [SECTION: TEMPLATE] -->
<Card title={$t.connections?.saved} padding="none">
<ul class="divide-y divide-gray-100">
{#if isLoading}
<li class="p-6 text-center text-gray-500">{$t.common.loading}</li>
{:else if connections.length === 0}
<li class="p-12 text-center text-gray-500 italic">{$t.connections?.no_saved}</li>
{:else}
{#each connections as conn}
<li class="p-6 flex items-center justify-between hover:bg-gray-50 transition-colors">
<div>
<div class="text-sm font-medium text-primary truncate">{conn.name}</div>
<div class="text-xs text-gray-400 mt-1 font-mono">{conn.type}://{conn.username}@{conn.host}:{conn.port}/{conn.database}</div>
</div>
<Button
variant="danger"
size="sm"
onclick={() => handleDelete(conn.id)}
>
{$t.connections?.delete}
</Button>
</li>
{/each}
{/if}
</ul>
</Card>
<!-- [/SECTION] -->
<!-- #endregion ConnectionList -->

View File

@@ -1,18 +1,16 @@
<!-- #region MapperTool [C:3] [TYPE Component] [SEMANTICS mapper, dataset, column, mapping, postgresql] -->
<!-- #region MapperTool [C:3] [TYPE Component] [SEMANTICS mapper, dataset, column, mapping, sqllab] -->
<!-- @BRIEF Component component: components/tools/MapperTool.svelte -->
<!-- @LAYER UI -->
<!--
@SEMANTICS: mapper, tool, dataset, postgresql, excel
@PURPOSE: UI component for mapping dataset column verbose names using the MapperPlugin.
@SEMANTICS: mapper, tool, dataset, sqllab, excel
@PURPOSE: UI component for mapping dataset column verbose names using Superset SQL Lab or Excel files.
@LAYER: UI
@RELATION: USES -> frontend/src/services/toolsService.js
@RELATION: USES -> frontend/src/services/connectionService.js
-->
<script>
// [SECTION: IMPORTS]
import { onMount } from 'svelte';
import { runTask } from '../../services/toolsService.js';
import { getConnections } from '../../services/connectionService.js';
import { api } from '../../lib/api';
import { selectedTask } from '../../lib/stores.js';
import { addToast } from '../../lib/toasts.js';
@@ -22,26 +20,23 @@
// [/SECTION]
let envs = [];
let connections = [];
let selectedEnv = '';
let datasetId = '';
let source = 'postgres';
let selectedConnection = '';
let tableName = '';
let tableSchema = 'public';
let source = 'sqllab';
let databaseId = '';
let sqlQuery = '';
let excelPath = '';
let isRunning = false;
let isGeneratingDocs = false;
let generatedDoc = null;
// #region fetchData:Function [TYPE Function]
// @PURPOSE: Fetches environments and saved connections.
// @PURPOSE: Fetches environments.
// @PRE: None.
// @POST: envs and connections arrays are populated.
// @POST: envs array is populated.
async function fetchData() {
try {
envs = await api.fetchApi('/environments');
connections = await getConnections();
} catch (e) {
addToast($t.mapper.errors.fetch_failed, 'error');
}
@@ -49,7 +44,7 @@
// #endregion fetchData:Function
// #region handleRunMapper:Function [TYPE Function]
// @PURPOSE: Triggers the MapperPlugin task.
// @PURPOSE: Triggers the MapperPlugin task via new sqllab/excel sources.
// @PRE: selectedEnv and datasetId are set; source-specific fields are valid.
// @POST: Mapper task is started and selectedTask is updated.
async function handleRunMapper() {
@@ -58,8 +53,8 @@
return;
}
if (source === 'postgres' && (!selectedConnection || !tableName)) {
addToast($t.mapper.errors.postgres_required, 'warning');
if (source === 'sqllab' && !databaseId) {
addToast($t.mapper.errors.sqllab_required, 'warning');
return;
}
@@ -75,9 +70,8 @@
env: env.name,
dataset_id: parseInt(datasetId),
source,
connection_id: selectedConnection,
table_name: tableName,
table_schema: tableSchema,
database_id: databaseId ? parseInt(databaseId) : undefined,
sql_query: sqlQuery || undefined,
excel_path: excelPath
});
@@ -169,8 +163,8 @@
<legend class="block text-sm font-medium text-gray-700 mb-2">{$t.mapper.source}</legend>
<div class="flex space-x-4">
<label class="inline-flex items-center">
<input type="radio" bind:group={source} value="postgres" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
<span class="ml-2 text-sm text-gray-700">{$t.mapper.source_postgres}</span>
<input type="radio" bind:group={source} value="sqllab" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
<span class="ml-2 text-sm text-gray-700">{$t.mapper.source_sqllab}</span>
</label>
<label class="inline-flex items-center">
<input type="radio" bind:group={source} value="excel" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
@@ -179,33 +173,23 @@
</div>
</fieldset>
{#if source === 'postgres'}
{#if source === 'sqllab'}
<div class="space-y-4 p-4 bg-gray-50 rounded-md border border-gray-100">
<div>
<Select
label={$t.mapper.connection}
bind:value={selectedConnection}
options={[
{ value: '', label: $t.mapper.select_connection },
...connections.map(c => ({ value: c.id, label: c.name }))
]}
<Input
label={$t.mapper.database_id}
type="number"
bind:value={databaseId}
placeholder={$t.mapper.database_id_placeholder}
/>
</div>
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<Input
label={$t.mapper.table_name}
type="text"
bind:value={tableName}
/>
</div>
<div>
<Input
label={$t.mapper.table_schema}
type="text"
bind:value={tableSchema}
/>
</div>
<div>
<Input
label={$t.mapper.sql_query_label}
type="text"
bind:value={sqlQuery}
placeholder={$t.mapper.sql_query_placeholder}
/>
</div>
</div>
{:else}
@@ -249,4 +233,4 @@
/>
</div>
<!-- [/SECTION] -->
<!-- #endregion MapperTool -->
<!-- #endregion MapperTool -->

View File

@@ -35,7 +35,6 @@ import enAdmin from './locales/en/admin.json';
import enTranslate from './locales/en/translate.json';
import enAssistant from './locales/en/assistant.json';
import enAuth from './locales/en/auth.json';
import enConnections from './locales/en/connections.json';
import enMapper from './locales/en/mapper.json';
import enMigration from './locales/en/migration.json';
import enStorage from './locales/en/storage.json';
@@ -57,7 +56,6 @@ import ruAdmin from './locales/ru/admin.json';
import ruTranslate from './locales/ru/translate.json';
import ruAssistant from './locales/ru/assistant.json';
import ruAuth from './locales/ru/auth.json';
import ruConnections from './locales/ru/connections.json';
import ruMapper from './locales/ru/mapper.json';
import ruMigration from './locales/ru/migration.json';
import ruStorage from './locales/ru/storage.json';
@@ -81,7 +79,6 @@ const en = {
translate: enTranslate,
assistant: enAssistant,
auth: enAuth,
connections: enConnections,
mapper: enMapper,
migration: enMigration,
storage: enStorage,
@@ -105,7 +102,6 @@ const ru = {
translate: ruTranslate,
assistant: ruAssistant,
auth: ruAuth,
connections: ruConnections,
mapper: ruMapper,
migration: ruMigration,
storage: ruStorage,

View File

@@ -1,21 +0,0 @@
{
"management": "Connection Management",
"add_new": "Add New Connection",
"name": "Connection Name",
"host": "Host",
"port": "Port",
"db_name": "Database Name",
"user": "Username",
"pass": "Password",
"create": "Create Connection",
"saved": "Saved Connections",
"no_saved": "No connections saved yet.",
"delete": "Delete",
"required_fields": "Please fill in all required fields",
"created_success": "Connection created successfully",
"fetch_failed": "Failed to fetch connections",
"delete_confirm": "Are you sure you want to delete this connection?",
"deleted_success": "Connection deleted",
"name_placeholder": "e.g. Production DWH",
"host_placeholder": "10.0.0.1"
}

View File

@@ -20,10 +20,12 @@
"bulk_map_columns": "Bulk Column Mapping",
"bulk_docs_generation": "Bulk Documentation Generation",
"source_type": "Source Type",
"source_postgresql_comments": "PostgreSQL Comments",
"source_sqllab": "Superset Database (SQL Lab)",
"source_xlsx": "XLSX File",
"connection_id": "Connection ID",
"connection_id_placeholder": "Enter connection ID...",
"source_database": "Database",
"select_database": "select a database",
"sql_query": "SQL Query",
"sql_query_hint": "leave empty for column information",
"xlsx_file": "XLSX File",
"selected_datasets": "Selected Datasets",
"start_mapping": "Start Mapping",

View File

@@ -4,12 +4,12 @@
"select_env": "-- Select Environment --",
"dataset_id": "Dataset ID",
"source": "Mapping Source",
"source_postgres": "PostgreSQL",
"source_sqllab": "Superset SQL Lab",
"source_excel": "Excel",
"connection": "Saved Connection",
"select_connection": "-- Select Connection --",
"table_name": "Table Name",
"table_schema": "Table Schema",
"database_id": "Superset Database ID",
"database_id_placeholder": "Enter Superset database ID...",
"sql_query_label": "SQL Query (optional)",
"sql_query_placeholder": "SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table'",
"excel_path": "Excel File Path",
"run": "Run Mapper",
"starting": "Starting...",
@@ -17,7 +17,7 @@
"errors": {
"fetch_failed": "Failed to fetch data",
"required_fields": "Please fill in required fields",
"postgres_required": "Connection and Table Name are required for postgres source",
"sqllab_required": "Database ID is required for SQL Lab source",
"excel_required": "Excel path is required for excel source",
"no_active_llm_provider": "No active LLM provider found",
"docs_start_failed": "Failed to start documentation generation",
@@ -30,4 +30,4 @@
},
"auto_document": "Auto-Document",
"excel_placeholder": "/path/to/mapping.xlsx"
}
}

View File

@@ -42,7 +42,6 @@
"settings": "Settings",
"settings_general": "General",
"settings_connections": "Connections",
"admin": "Admin",
"admin_users": "User Management",

View File

@@ -4,7 +4,6 @@
"appearance": "Appearance",
"llm": "LLM",
"storage": "Storage",
"connections": "Connections",
"environments": "Environments",
"global_title": "Global Settings",
"env_title": "Superset Environments",
@@ -23,7 +22,6 @@
"storage_preview": "Path Preview",
"env_description": "Configure Superset environments for dashboards and datasets.",
"env_actions": "Actions",
"connections_description": "Configure database connections for data mapping.",
"llm_description": "Configure LLM providers for dataset documentation.",
"llm_prompts_title": "LLM Prompt Templates",
"llm_prompts_description": "Edit reusable prompts used for documentation, dashboard validation, and git commit generation.",
@@ -97,7 +95,6 @@
"belief_state_hint": "Logs agent reasoning and internal state changes for debugging.",
"save_logging": "Save Logging Config",
"save_global_settings": "Save Global Settings",
"no_external_connections": "No external connections configured.",
"name": "Name",
"username": "Username",
"password": "Password",

View File

@@ -1,21 +0,0 @@
{
"management": "Управление подключениями",
"add_new": "Добавить новое подключение",
"name": "Название подключения",
"host": "Хост",
"port": "Порт",
"db_name": "Название БД",
"user": "Имя пользователя",
"pass": "Пароль",
"create": "Создать подключение",
"saved": "Сохраненные подключения",
"no_saved": "Нет сохраненных подключений.",
"delete": "Удалить",
"required_fields": "Заполните все обязательные поля",
"created_success": "Подключение успешно создано",
"fetch_failed": "Не удалось загрузить подключения",
"delete_confirm": "Вы уверены, что хотите удалить это подключение?",
"deleted_success": "Подключение удалено",
"name_placeholder": "например, Production DWH",
"host_placeholder": "10.0.0.1"
}

View File

@@ -20,10 +20,12 @@
"bulk_map_columns": "Массовый маппинг колонок",
"bulk_docs_generation": "Массовая генерация документации",
"source_type": "Тип источника",
"source_postgresql_comments": "Комментарии PostgreSQL",
"source_sqllab": "База данных Superset (SQL Lab)",
"source_xlsx": "XLSX-файл",
"connection_id": "ID подключения",
"connection_id_placeholder": "Введите ID подключения...",
"source_database": "База данных",
"select_database": "выберите базу данных",
"sql_query": "SQL-запрос",
"sql_query_hint": "оставьте пустым для информации о колонках",
"xlsx_file": "XLSX-файл",
"selected_datasets": "Выбранные датасеты",
"start_mapping": "Запустить маппинг",

View File

@@ -4,12 +4,12 @@
"select_env": "-- Выберите окружение --",
"dataset_id": "ID датасета",
"source": "Источник маппинга",
"source_postgres": "PostgreSQL",
"source_sqllab": "Superset SQL Lab",
"source_excel": "Excel",
"connection": "Сохраненное подключение",
"select_connection": "-- Выберите подключение --",
"table_name": "Имя таблицы",
"table_schema": "Схема таблицы",
"database_id": "ID базы данных Superset",
"database_id_placeholder": "Введите ID базы данных Superset...",
"sql_query_label": "SQL-запрос (опционально)",
"sql_query_placeholder": "SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table'",
"excel_path": "Путь к файлу Excel",
"run": "Запустить маппер",
"starting": "Запуск...",
@@ -17,7 +17,7 @@
"errors": {
"fetch_failed": "Не удалось загрузить данные",
"required_fields": "Пожалуйста, заполните обязательные поля",
"postgres_required": "Подключение и имя таблицы обязательны для источника PostgreSQL",
"sqllab_required": "ID базы данных обязателен для SQL Lab",
"excel_required": "Путь к Excel обязателен для источника Excel",
"no_active_llm_provider": "Не найден активный LLM-провайдер",
"docs_start_failed": "Не удалось запустить генерацию документации",
@@ -30,4 +30,4 @@
},
"auto_document": "Авто-документирование",
"excel_placeholder": "/path/to/mapping.xlsx"
}
}

View File

@@ -42,7 +42,6 @@
"settings": "Настройки",
"settings_general": "Общие",
"settings_connections": "Подключения",
"admin": "Админ",
"admin_users": "Пользователи",

View File

@@ -3,7 +3,6 @@
"language": "Язык",
"appearance": "Внешний вид",
"llm": "LLM",
"connections": "Подключения",
"environments": "Окружения",
"global_title": "Общие настройки",
"env_title": "Окружения Superset",
@@ -22,7 +21,6 @@
"storage_preview": "Предпросмотр пути",
"env_description": "Настройка окружений Superset для дашбордов и датасетов.",
"env_actions": "Действия",
"connections_description": "Настройка подключений к базам данных для маппинга.",
"llm_description": "Настройка LLM провайдеров для документирования датасетов.",
"llm_prompts_title": "Шаблоны промптов LLM",
"llm_prompts_description": "Редактируйте промпты для документации, проверки дашбордов и генерации git-коммитов.",
@@ -97,7 +95,6 @@
"belief_state_hint": "Логирует рассуждения агента и изменения внутреннего состояния для отладки.",
"save_logging": "Сохранить настройки логирования",
"save_global_settings": "Сохранить общие настройки",
"no_external_connections": "Внешние подключения не настроены.",
"name": "Название",
"username": "Имя пользователя",
"password": "Пароль",

View File

@@ -30,7 +30,7 @@
let selectedIds = $state(new SvelteSet());
let selectedDatasetId = $state(null), detailData = $state(null), detailLoading = $state(false), detailError = $state(null);
let showMapColumnsModal = $state(false), showGenerateDocsModal = $state(false);
let mapSourceType = $state("postgresql"), mapConnectionId = $state(""), mapFileData = $state(null);
let mapSourceType = $state("sqllab"), mapDatabases = $state([]), mapDatabaseId = $state(null), mapSqlQuery = $state(""), mapFileData = $state(null);
let llmProvider = $state(""), llmOptions = $state({});
let isMobile = $state(false), mobileSlideOpen = $state(false);
let ws = null;
@@ -118,9 +118,14 @@
if (action === "map_columns") {
selectedIds = new SvelteSet([dataset.id]);
showMapColumnsModal = true;
mapSourceType = "postgresql";
mapConnectionId = null;
mapSourceType = "sqllab";
mapDatabaseId = null;
mapSqlQuery = "";
mapFileData = null;
// Fetch databases on open
if (selectedEnv) {
api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {});
}
} else if (action === "generate_docs") {
selectedIds = new SvelteSet([dataset.id]);
showGenerateDocsModal = true;
@@ -147,14 +152,14 @@
// Bulk actions
async function handleBulkMapColumns() {
if (selectedIds.size === 0) return;
if (mapSourceType === "postgresql" && !mapConnectionId) return;
if (mapSourceType === "sqllab" && !mapDatabaseId) return;
if (mapSourceType === "xlsx" && (!mapFileData || mapFileData.length === 0)) return;
try {
let fileData = null;
if (mapSourceType === "xlsx" && mapFileData?.length) fileData = mapFileData[0].name;
const resp = await api.postApi("/datasets/map-columns", {
env_id: selectedEnv, dataset_ids: [...selectedIds],
source_type: mapSourceType, connection_id: mapConnectionId || undefined, file_data: fileData || undefined,
source_type: mapSourceType, database_id: mapDatabaseId || undefined, sql_query: mapSqlQuery || undefined, file_data: fileData || undefined,
});
showMapColumnsModal = false;
selectedIds = new SvelteSet();
@@ -326,7 +331,7 @@
<div class="max-w-screen-2xl mx-auto flex items-center justify-between">
<span class="font-medium text-sm">{selectedIds.size} {$t.datasets?.selected}</span>
<div class="flex gap-2">
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showMapColumnsModal = true; }}>
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showMapColumnsModal = true; mapSourceType = "sqllab"; mapDatabaseId = null; mapSqlQuery = ""; mapFileData = null; if (selectedEnv) api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {}); }}>
🗺 {$t.datasets?.action_map_columns}
</button>
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showGenerateDocsModal = true; }}>
@@ -343,15 +348,31 @@
<!-- Map Columns Modal -->
{#if showMapColumnsModal}
<div class="fixed inset-0 bg-black/50 z-50 flex items-center justify-center p-4" role="dialog" aria-modal="true" onclick={(e) => { if (e.target === e.currentTarget) showMapColumnsModal = false; }}>
<div class="bg-white rounded-lg shadow-xl p-6 max-w-md w-full" onclick={(e) => e.stopPropagation()}>
<div class="bg-white rounded-lg shadow-xl p-6 max-w-md w-full" style="max-height: 90vh; overflow-y: auto;" onclick={(e) => e.stopPropagation()}>
<h3 class="text-lg font-semibold mb-4">{$t.datasets?.bulk_map_columns}</h3>
<p class="text-sm text-gray-600 mb-3">{$t.datasets?.selected_count?.replace('{count}', String(selectedIds.size))}</p>
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapSourceType}>
<option value="postgresql">{$t.datasets?.source_postgresql_comments}</option>
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapSourceType} onchange={() => { if (mapSourceType === 'sqllab' && mapDatabases.length === 0 && selectedEnv) { api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {}); } }}>
<option value="sqllab">{$t.datasets?.source_sqllab ?? 'Superset SQL Lab'}</option>
<option value="xlsx">{$t.datasets?.source_xlsx}</option>
</select>
{#if mapSourceType === "postgresql"}
<input class="w-full border rounded px-2 py-1.5 mb-4" placeholder={$t.datasets?.connection_id_placeholder} bind:value={mapConnectionId} />
{#if mapSourceType === "sqllab"}
<label class="text-sm font-medium mb-1 block">{$t.datasets?.source_database ?? 'База данных'}</label>
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapDatabaseId}>
<option value="">-- {$t.datasets?.select_database ?? 'выберите базу данных'} --</option>
{#each mapDatabases as db}
<option value={db.id}>{db.database_name || db.name}</option>
{/each}
</select>
<label class="text-sm font-medium mb-1 block">
{$t.datasets?.sql_query ?? 'SQL-запрос'}
<span class="text-xs text-gray-400 font-normal ml-1">({$t.datasets?.sql_query_hint ?? 'оставьте пустым для information_schema'})</span>
</label>
<textarea
class="w-full border rounded px-2 py-1.5 mb-4 font-mono text-xs"
rows="4"
placeholder="SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table';"
bind:value={mapSqlQuery}
></textarea>
{:else}
<input type="file" accept=".xlsx" class="w-full mb-4" onchange={(e) => mapFileData = e.target.files} />
{/if}

View File

@@ -7,7 +7,6 @@
<!-- @REPLACED_BY N/A -->
<!-- @RELATION DEPENDS_ON -> [EnvironmentsTab] -->
<!-- @RELATION DEPENDS_ON -> [LoggingSettings] -->
<!-- @RELATION DEPENDS_ON -> [DatabaseConnectionsTab] -->
<!-- @RELATION DEPENDS_ON -> [LlmSettings] -->
<!-- @RELATION DEPENDS_ON -> [MigrationSettings] -->
<!-- @RELATION DEPENDS_ON -> [StorageSettings] -->
@@ -36,7 +35,6 @@
import EnvironmentsTab from "./EnvironmentsTab.svelte";
import LoggingSettings from "./LoggingSettings.svelte";
import DatabaseConnectionsTab from "./DatabaseConnectionsTab.svelte";
import LlmSettings from "./LlmSettings.svelte";
import MigrationSettings from "./MigrationSettings.svelte";
import StorageSettings from "./StorageSettings.svelte";
@@ -171,19 +169,6 @@
>
{$t.settings?.logging}
</button>
<button
class="px-4 py-2 text-sm font-medium transition-colors focus:outline-none border-b-2"
class:text-primary={activeTab === "connections"}
class:border-primary={activeTab === "connections"}
class:text-gray-600={activeTab !== "connections"}
class:border-transparent={activeTab !== "connections"}
class:hover:text-gray-800={activeTab !== "connections"}
class:hover:border-gray-300={activeTab !== "connections"}
aria-current={activeTab === "connections" ? "page" : undefined}
onclick={() => handleTabChange("connections")}
>
{$t.settings?.connections}
</button>
<button
class="px-4 py-2 text-sm font-medium transition-colors focus:outline-none border-b-2"
class:text-primary={activeTab === "git"}
@@ -270,8 +255,6 @@
<EnvironmentsTab bind:settings onSave={loadSettings} />
{:else if activeTab === "logging"}
<LoggingSettings bind:settings onSave={handleSave} />
{:else if activeTab === "connections"}
<DatabaseConnectionsTab />
{:else if activeTab === "git"}
<GitSettingsPage />
{:else if activeTab === "llm"}

View File

@@ -1,342 +0,0 @@
<!-- #region DatabaseConnectionsTab [C:3] [TYPE Component] [SEMANTICS settings, connections, database, crud] -->
<!-- @BRIEF Database connections management tab: list, create, delete external DB connections. -->
<!-- @LAYER UI -->
<!-- @RELATION CALLS -> [api_module] -->
<!-- @PRE: API client is initialized. -->
<!-- @POST: Connections list is displayed; user can create and delete connections. -->
<!-- @UX_STATE Loading -> Loading spinner. -->
<!-- @UX_STATE Loaded -> Table of connections with actions. -->
<!-- @UX_STATE FormVisible -> Connection creation form displayed. -->
<!-- @UX_FEEDBACK Toast on create/delete success/failure. -->
<script>
import { onMount } from "svelte";
import { t } from "$lib/i18n";
import { api } from "$lib/api.js";
import { addToast } from "$lib/toasts";
import { createEmptyConnectionDraft } from "./settings-utils.js";
let connections = $state([]);
let hasLoadedConnections = $state(false);
let isLoadingConnections = $state(false);
let isSubmittingConnection = $state(false);
let isConnectionFormVisible = $state(false);
let newConnection = $state(createEmptyConnectionDraft());
onMount(() => {
void loadConnections();
});
async function loadConnections(force = false) {
if (isLoadingConnections || (hasLoadedConnections && !force)) {
return;
}
isLoadingConnections = true;
try {
const response = await api.requestApi("/settings/connections");
connections = Array.isArray(response) ? response : [];
hasLoadedConnections = true;
} catch (err) {
console.error("[DatabaseConnectionsTab] Failed to load:", err);
addToast(err.message || $t.connections?.fetch_failed, "error");
connections = [];
} finally {
isLoadingConnections = false;
}
}
function resetConnectionForm() {
newConnection = createEmptyConnectionDraft();
}
function openConnectionForm() {
isConnectionFormVisible = true;
}
function closeConnectionForm() {
isConnectionFormVisible = false;
resetConnectionForm();
}
async function handleCreateConnection() {
if (
!newConnection.name ||
!newConnection.host ||
!newConnection.database ||
!newConnection.username ||
!newConnection.password
) {
addToast($t.connections?.required_fields, "warning");
return;
}
isSubmittingConnection = true;
try {
await api.requestApi("/settings/connections", "POST", {
...newConnection,
port: newConnection.port ? Number(newConnection.port) : null,
});
addToast($t.connections?.created_success, "success");
closeConnectionForm();
await loadConnections(true);
} catch (err) {
console.error("[DatabaseConnectionsTab] Failed to create:", err);
addToast(err.message || $t.settings?.save_failed, "error");
} finally {
isSubmittingConnection = false;
}
}
async function handleDeleteConnection(connectionId) {
if (!confirm($t.connections?.delete_confirm)) {
return;
}
try {
await api.requestApi(`/settings/connections/${connectionId}`, "DELETE");
addToast($t.connections?.deleted_success, "success");
await loadConnections(true);
} catch (err) {
console.error("[DatabaseConnectionsTab] Failed to delete:", err);
addToast(err.message || $t.settings?.save_failed, "error");
}
}
</script>
<div class="text-lg font-medium mb-4">
<div class="mb-6 flex items-start justify-between gap-4">
<div>
<h2 class="text-xl font-bold mb-4">
{$t.settings?.connections}
</h2>
<p class="text-gray-600">
{$t.settings?.connections_description}
</p>
</div>
{#if connections.length > 0 || isConnectionFormVisible}
<button
class="rounded-lg bg-primary px-4 py-2 text-sm font-medium text-white hover:bg-primary-hover"
onclick={openConnectionForm}
>
{$t.connections?.add_new}
</button>
{/if}
</div>
{#if isConnectionFormVisible}
<div class="mb-6 rounded-lg border border-gray-200 bg-gray-50 p-6">
<form
class="space-y-4"
onsubmit={(event) => {
event.preventDefault();
handleCreateConnection();
}}
>
<div>
<label
for="connection_name"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.name}</label
>
<input
id="connection_name"
type="text"
bind:value={newConnection.name}
placeholder={$t.connections?.name_placeholder}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
<div class="grid grid-cols-1 gap-4 md:grid-cols-3">
<div>
<label
for="connection_type"
class="block text-sm font-medium text-gray-700"
>Type</label
>
<select
id="connection_type"
bind:value={newConnection.type}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
>
<option value="postgres">PostgreSQL</option>
</select>
</div>
<div class="md:col-span-2">
<label
for="connection_host"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.host}</label
>
<input
id="connection_host"
type="text"
bind:value={newConnection.host}
placeholder={$t.connections?.host_placeholder}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
</div>
<div class="grid grid-cols-1 gap-4 md:grid-cols-2">
<div>
<label
for="connection_port"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.port}</label
>
<input
id="connection_port"
type="number"
bind:value={newConnection.port}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
<div>
<label
for="connection_database"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.db_name}</label
>
<input
id="connection_database"
type="text"
bind:value={newConnection.database}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
</div>
<div class="grid grid-cols-1 gap-4 md:grid-cols-2">
<div>
<label
for="connection_user"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.user}</label
>
<input
id="connection_user"
type="text"
bind:value={newConnection.username}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
<div>
<label
for="connection_password"
class="block text-sm font-medium text-gray-700"
>{$t.connections?.pass}</label
>
<input
id="connection_password"
type="password"
bind:value={newConnection.password}
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
/>
</div>
</div>
<div class="flex justify-end gap-2">
<button
type="button"
class="rounded-lg bg-gray-200 px-4 py-2 text-sm font-medium text-gray-700 hover:bg-gray-300"
onclick={closeConnectionForm}
>
{$t.common?.cancel}
</button>
<button
type="submit"
class="rounded-lg bg-primary px-4 py-2 text-sm font-medium text-white hover:bg-primary-hover disabled:cursor-not-allowed disabled:opacity-60"
disabled={isSubmittingConnection}
>
{isSubmittingConnection
? ($t.settings?.saving || $t.common?.loading || "Saving...")
: $t.connections?.create}
</button>
</div>
</form>
</div>
{/if}
{#if isLoadingConnections}
<div
class="rounded-lg border border-gray-200 bg-gray-50 p-8 text-center text-gray-500"
>
{$t.common?.loading || "Loading..."}
</div>
{:else if connections.length > 0}
<div class="overflow-x-auto rounded-lg border border-gray-200">
<table class="min-w-full divide-y divide-gray-200">
<thead class="bg-gray-50">
<tr>
<th
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
>{$t.connections?.name}</th
>
<th
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
>{$t.connections?.host}</th
>
<th
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
>{$t.connections?.user}</th
>
<th
class="px-6 py-3 text-right text-xs font-medium uppercase tracking-wider text-gray-500"
>{$t.settings?.env_actions || "Actions"}</th
>
</tr>
</thead>
<tbody class="divide-y divide-gray-200 bg-white">
{#each connections as connection}
<tr>
<td class="px-6 py-4 whitespace-nowrap">
{connection.name}
</td>
<td class="px-6 py-4 text-sm text-gray-600">
<span class="font-mono"
>{connection.type}://{connection.host}:{connection.port}/{connection.database}</span
>
</td>
<td class="px-6 py-4 whitespace-nowrap">
{connection.username}
</td>
<td class="px-6 py-4 text-right text-sm font-medium">
<button
class="text-red-600 hover:text-red-900"
onclick={() => handleDeleteConnection(connection.id)}
>
{$t.connections?.delete}
</button>
</td>
</tr>
{/each}
</tbody>
</table>
</div>
{:else if !isConnectionFormVisible}
<div
class="rounded-lg border border-dashed border-gray-300 bg-gray-50 py-8 text-center"
>
<p class="text-gray-500">
{$t.settings?.no_external_connections}
</p>
<button
class="mt-4 rounded border border-primary px-4 py-2 text-primary hover:bg-primary-light"
onclick={() => {
openConnectionForm();
void loadConnections();
}}
>
{$t.connections?.add_new}
</button>
</div>
{:else}
<div
class="rounded-lg border border-dashed border-gray-300 bg-white p-6 text-center text-sm text-gray-500"
>
{$t.connections?.no_saved}
</div>
{/if}
</div>
<!-- #endregion DatabaseConnectionsTab -->

View File

@@ -33,11 +33,8 @@ vi.mock('$lib/i18n', () => ({
saving: 'Saving...',
environments: 'Environments',
logging: 'Logging',
connections: 'Connections',
connections_description: 'Configure database connections for data mapping.',
llm: 'LLM',
storage: 'Storage',
no_external_connections: 'No external connections configured.',
save_success: 'Settings saved',
save_failed: 'Failed',
env_description: 'Configure environments',
@@ -47,22 +44,7 @@ vi.mock('$lib/i18n', () => ({
common: { refresh: 'Refresh', save: 'Save', cancel: 'Cancel', loading: 'Loading...' },
tasks: { cron_label: 'Cron Expression', cron_hint: 'Standard cron format' },
nav: { settings_git: 'Git' },
connections: {
add_new: 'Add New Connection',
create: 'Create Connection',
name: 'Connection Name',
host: 'Host',
port: 'Port',
db_name: 'Database Name',
user: 'Username',
pass: 'Password',
created_success: 'Connection created successfully',
fetch_failed: 'Failed to fetch connections',
delete_confirm: 'Are you sure you want to delete this connection?',
deleted_success: 'Connection deleted',
required_fields: 'Please fill in all required fields'
}
});
common: { refresh: 'Refresh', save: 'Save', cancel: 'Cancel', loading: 'Loading...' },
return () => { };
}
},
@@ -98,14 +80,6 @@ describe('SettingsPage.integration.test.js', () => {
api.requestApi.mockImplementation((url, method = 'GET', body) => {
if (url === '/migration/settings') return Promise.resolve(mockMigrationSettings);
if (url.includes('/migration/mappings-data')) return Promise.resolve({ items: [], total: 0 });
if (url === '/settings/connections' && method === 'POST') {
return Promise.resolve({
id: 'conn-1',
created_at: '2026-03-15T00:00:00Z',
...body
});
}
if (url === '/settings/connections') return Promise.resolve([]);
return Promise.resolve({});
});
});
@@ -160,57 +134,4 @@ describe('SettingsPage.integration.test.js', () => {
expect(addToast).toHaveBeenCalledWith('Synced 1 environment(s)', 'success');
});
});
it('opens the connection form and creates a connection from the consolidated settings tab', async () => {
render(SettingsPage);
await waitFor(() => expect(api.getConsolidatedSettings).toHaveBeenCalled());
// Click connections tab
await fireEvent.click(screen.getAllByText('Connections')[0]);
await waitFor(() => {
expect(api.requestApi).toHaveBeenCalledWith('/settings/connections');
});
await fireEvent.click(screen.getByText('Add New Connection'));
expect(screen.getByLabelText('Connection Name')).toBeTruthy();
// Verify form fields and submit button exist
const nameInput = screen.getByLabelText('Connection Name');
const submitBtn = screen.getByText('Create Connection');
expect(nameInput).toBeTruthy();
expect(submitBtn).toBeTruthy();
// Set input values using native value setter + input event for Svelte 5
nameInput.value = 'Analytics Warehouse';
await fireEvent.input(nameInput);
screen.getByLabelText('Host').value = 'warehouse.internal';
await fireEvent.input(screen.getByLabelText('Host'));
screen.getByLabelText('Database Name').value = 'analytics';
await fireEvent.input(screen.getByLabelText('Database Name'));
screen.getByLabelText('Username').value = 'reporter';
await fireEvent.input(screen.getByLabelText('Username'));
screen.getByLabelText('Password').value = 'secret';
await fireEvent.input(screen.getByLabelText('Password'));
// Click Create Connection button
await fireEvent.click(submitBtn);
await waitFor(() => {
expect(api.requestApi).toHaveBeenCalledWith('/settings/connections', 'POST', {
name: 'Analytics Warehouse',
type: 'postgres',
host: 'warehouse.internal',
port: 5432,
database: 'analytics',
username: 'reporter',
password: 'secret'
});
});
await waitFor(() => {
expect(addToast).toHaveBeenCalledWith('Connection created successfully', 'success');
});
});
});

View File

@@ -44,7 +44,6 @@ vi.mock('$lib/i18n', () => ({
log_level: 'Log Level',
task_log_level: 'Task Log Level',
enable_belief_state: 'Enable Belief State',
connections: 'Connections',
llm: 'LLM',
storage: 'Storage',
save_success: 'Settings saved',
@@ -57,7 +56,6 @@ vi.mock('$lib/i18n', () => ({
common: { refresh: 'Refresh', retry: 'Retry', save: 'Save' },
tasks: { cron_label: 'Cron Expression', cron_hint: 'Standard cron format' },
nav: { settings_git: 'Git' },
connections: { name: 'Name', user: 'User' }
});
return () => { };
}
@@ -78,7 +76,6 @@ describe('SettingsPage UX Contracts', () => {
const mockSettings = {
environments: [],
logging: { level: 'INFO', task_log_level: 'INFO', enable_belief_state: false },
connections: [],
llm: {}
};

View File

@@ -1,51 +0,0 @@
<!-- #region ConnectionsPage [C:3] [TYPE Page] [SEMANTICS sveltekit, connection, database, management, config] -->
<!-- @BRIEF Page component: routes/settings/connections/+page.svelte -->
<!-- @LAYER Page -->
<!--
@SEMANTICS: settings, connections, database, management
@PURPOSE: Page for managing database connection configurations.
@LAYER: UI
@RELATION: IMPLEMENTS -> [RoutePages]
@RELATION: IMPLEMENTS -> [PageContracts]
@RELATION: BINDS_TO -> [NavigationContracts]
@RELATION: DEPENDS_ON -> ConnectionForm
@RELATION: DEPENDS_ON -> ConnectionList
@UX_STATE: Ready -> Connection form and saved connection list render side by side.
@UX_STATE: Saving -> Successful form submission refreshes the list without leaving the page.
@UX_STATE: Error -> Child components surface validation or request errors inline.
-->
<script>
import ConnectionForm from '../../../components/tools/ConnectionForm.svelte';
import ConnectionList from '../../../components/tools/ConnectionList.svelte';
import { t } from '$lib/i18n';
let listComponent;
// #region handleSuccess:Function [TYPE Function]
/* @PURPOSE: Refreshes the connection list after a successful creation.
@PRE: listComponent must be bound.
@POST: Triggers the fetchConnections method on the list component.
*/
function handleSuccess() {
if (listComponent) {
listComponent.fetchConnections();
}
}
// #endregion handleSuccess:Function
</script>
<div class="max-w-7xl mx-auto py-6 sm:px-6 lg:px-8">
<div class="px-4 py-6 sm:px-0">
<h1 class="text-2xl font-semibold text-gray-900 mb-6">{$t.connections?.management }</h1>
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
<div>
<ConnectionForm onsuccess={handleSuccess} />
</div>
<div>
<ConnectionList bind:this={listComponent} />
</div>
</div>
</div>
</div>
<!-- #endregion ConnectionsPage -->

View File

@@ -11,7 +11,6 @@
export const SETTINGS_TABS = [
"environments",
"logging",
"connections",
"git",
"llm",
"migration",
@@ -158,17 +157,3 @@ export function resolveEnvStage(env) {
// ---------------------------------------------------------------------------
// Connection draft helper
// ---------------------------------------------------------------------------
/**
* Create a blank connection object suitable for the "new connection" form.
*/
export function createEmptyConnectionDraft() {
return {
name: "",
host: "",
port: "",
database: "",
username: "",
password: "",
};
}

View File

@@ -1,54 +0,0 @@
// #region ConnectionService [C:2] [TYPE Service] [SEMANTICS connection, service, api, crud, database]
// @BRIEF Service for Connection Management API — list, create, and delete connection configurations.
/**
* Service for interacting with the Connection Management API.
*/
import { requestApi } from '../lib/api';
const API_BASE = '/settings/connections';
// #region getConnections:Function [TYPE Function]
/* @PURPOSE: Fetch a list of saved connections.
@PRE: None.
@POST: Returns a promise resolving to an array of connections.
*/
/**
* Fetch a list of saved connections.
* @returns {Promise<Array>} List of connections.
*/
export async function getConnections() {
return requestApi(API_BASE);
}
// #endregion getConnections:Function
// #region createConnection:Function [TYPE Function]
/* @PURPOSE: Create a new connection configuration.
@PRE: connectionData must be a valid object.
@POST: Returns a promise resolving to the created connection.
*/
/**
* Create a new connection configuration.
* @param {Object} connectionData - The connection data.
* @returns {Promise<Object>} The created connection instance.
*/
export async function createConnection(connectionData) {
return requestApi(API_BASE, 'POST', connectionData);
}
// #endregion createConnection:Function
// #region deleteConnection:Function [TYPE Function]
/* @PURPOSE: Delete a connection configuration.
@PRE: connectionId must be a valid string.
@POST: Returns a promise that resolves when deletion is complete.
*/
/**
* Delete a connection configuration.
* @param {string} connectionId - The ID of the connection to delete.
*/
export async function deleteConnection(connectionId) {
return requestApi(`${API_BASE}/${connectionId}`, 'DELETE');
}
// #endregion deleteConnection:Function
// #endregion ConnectionService