refactor(connections): remove ConnectionConfig, migrate mapper to SQL Lab
Backend: - Remove ConnectionConfig model, CRUD routes (connections.py), and tests - Remove ensure_connection_configs_table from database.py - Migrate MapperPlugin from psycopg2 direct PostgreSQL to SupersetSqlLabExecutor - Add DatasetMapper.get_sqllab_mappings() with default information_schema query - Update MapColumnsRequest: connection_id → database_id + sql_query - Fix pydantic_settings v2 deprecation (Field(env=...) → validation_alias) - Clean up app.py, routes/__init__.py, database.py imports Frontend: - Remove DatabaseConnectionsTab, ConnectionForm, ConnectionList components - Remove /settings/connections route page and Navbar link - Remove connectionService.js, connections i18n files - Update MapperTool.svelte: postgres → sqllab source - Update Map Columns modal: database selector + SQL query instead of connection_id - Clean up settings page tabs, nav links, test mocks - Clean up i18n keys (settings, nav, mapper) Tests: 25/25 pass (14 030-feature + 11 route tests) Build: succeeds
This commit is contained in:
@@ -12,7 +12,6 @@
|
||||
# @RELATION DEPENDS_ON -> [PluginsRouter]
|
||||
# @RELATION DEPENDS_ON -> [TasksRouter]
|
||||
# @RELATION DEPENDS_ON -> [SettingsRouter]
|
||||
# @RELATION DEPENDS_ON -> [ConnectionsRouter]
|
||||
# @RELATION DEPENDS_ON -> [ReportsRouter]
|
||||
# @RELATION DEPENDS_ON -> [LlmRoutes]
|
||||
# @SIDE_EFFECT: Registers route group imports via __getattr__
|
||||
@@ -22,7 +21,6 @@ __all__ = [
|
||||
"assistant",
|
||||
"clean_release",
|
||||
"clean_release_v2",
|
||||
"connections",
|
||||
"dashboards",
|
||||
"dataset_review",
|
||||
"datasets",
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# #region RoutesTestsConftest [TYPE Module] [C:1] [SEMANTICS test, fixture, mock, conftest]
|
||||
# @BRIEF Shared low-fidelity test doubles for API route test modules.
|
||||
|
||||
# Set required env vars before any app imports
|
||||
import os
|
||||
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
||||
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:test_auth")
|
||||
os.environ.setdefault("DEV_MODE", "true")
|
||||
class FakeQuery:
|
||||
"""Shared chainable query stub for route tests.
|
||||
WARNING: filter() is predicate-blind — all ownership and permission filters are
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
# #region ConnectionsRoutesTests [TYPE Module] [C:3] [SEMANTICS test, connection, route, crud, bootstrap]
|
||||
# @BRIEF Verifies connection routes bootstrap their table before CRUD access.
|
||||
# @LAYER: API
|
||||
# @RELATION DEPENDS_ON -> ConnectionsRouter
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
from sqlalchemy import create_engine, inspect
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
# Force SQLite in-memory for database module imports.
|
||||
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
|
||||
os.environ["DATABASE_URL"] = "sqlite:///:memory:"
|
||||
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
|
||||
os.environ["TASKS_DATABASE_URL"] = "sqlite:///:memory:"
|
||||
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
|
||||
os.environ["AUTH_DATABASE_URL"] = "sqlite:///:memory:"
|
||||
# @SIDE_EFFECT_WARNING: os.environ mutation at module import time — no teardown. This bleeds into all subsequently collected tests. Migrate to pytest.fixture(autouse=True) with monkeypatch.setenv.
|
||||
os.environ["ENVIRONMENT"] = "testing"
|
||||
backend_dir = str(Path(__file__).parent.parent.parent.parent.resolve())
|
||||
if backend_dir not in sys.path:
|
||||
sys.path.insert(0, backend_dir)
|
||||
@pytest.fixture
|
||||
def db_session():
|
||||
engine = create_engine(
|
||||
"sqlite:///:memory:",
|
||||
connect_args={"check_same_thread": False},
|
||||
poolclass=StaticPool,
|
||||
)
|
||||
session = sessionmaker(bind=engine)()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
# #region test_list_connections_bootstraps_missing_table [TYPE Function]
|
||||
# @RELATION BINDS_TO -> ConnectionsRoutesTests
|
||||
# @BRIEF Ensure listing connections auto-creates missing table and returns empty payload.
|
||||
def test_list_connections_bootstraps_missing_table(db_session):
|
||||
from src.api.routes.connections import list_connections
|
||||
result = asyncio.run(list_connections(db=db_session))
|
||||
inspector = inspect(db_session.get_bind())
|
||||
assert result == []
|
||||
assert "connection_configs" in inspector.get_table_names()
|
||||
# #endregion test_list_connections_bootstraps_missing_table
|
||||
# #region test_create_connection_bootstraps_missing_table [TYPE Function]
|
||||
# @RELATION BINDS_TO -> ConnectionsRoutesTests
|
||||
# @BRIEF Ensure connection creation bootstraps table and persists returned connection fields.
|
||||
def test_create_connection_bootstraps_missing_table(db_session):
|
||||
from src.api.routes.connections import ConnectionCreate, create_connection
|
||||
payload = ConnectionCreate(
|
||||
name="Analytics Warehouse",
|
||||
type="postgres",
|
||||
host="warehouse.internal",
|
||||
port=5432,
|
||||
database="analytics",
|
||||
username="reporter",
|
||||
password="secret",
|
||||
)
|
||||
created = asyncio.run(create_connection(connection=payload, db=db_session))
|
||||
inspector = inspect(db_session.get_bind())
|
||||
assert created.name == "Analytics Warehouse"
|
||||
assert created.host == "warehouse.internal"
|
||||
assert "connection_configs" in inspector.get_table_names()
|
||||
# #endregion test_create_connection_bootstraps_missing_table
|
||||
# #endregion ConnectionsRoutesTests
|
||||
@@ -132,7 +132,7 @@ def test_get_datasets_invalid_pagination(mock_deps):
|
||||
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
|
||||
# @BRIEF Validate map-columns request creates an async mapping task and returns its identifier.
|
||||
# @TEST: POST /api/datasets/map-columns creates mapping task
|
||||
# @PRE: Valid env_id, dataset_ids, source_type
|
||||
# @PRE: Valid env_id, dataset_ids, source_type (sqllab)
|
||||
# @POST: Returns task_id
|
||||
def test_map_columns_success(mock_deps):
|
||||
# Mock environment
|
||||
@@ -145,7 +145,7 @@ def test_map_columns_success(mock_deps):
|
||||
mock_deps["task"].create_task = AsyncMock(return_value=mock_task)
|
||||
response = client.post(
|
||||
"/api/datasets/map-columns",
|
||||
json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "postgresql"},
|
||||
json={"env_id": "prod", "dataset_ids": [1, 2, 3], "source_type": "sqllab", "database_id": 1},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
@@ -157,7 +157,7 @@ def test_map_columns_success(mock_deps):
|
||||
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
|
||||
# @BRIEF Validate map-columns rejects unsupported source types with a 400 contract response.
|
||||
# @TEST: POST /api/datasets/map-columns returns 400 for invalid source_type
|
||||
# @PRE: source_type is not 'postgresql' or 'xlsx'
|
||||
# @PRE: source_type is not 'sqllab' or 'xlsx'
|
||||
# @POST: Returns 400 error
|
||||
def test_map_columns_invalid_source_type(mock_deps):
|
||||
response = client.post(
|
||||
@@ -165,7 +165,7 @@ def test_map_columns_invalid_source_type(mock_deps):
|
||||
json={"env_id": "prod", "dataset_ids": [1], "source_type": "invalid"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "Source type must be 'postgresql' or 'xlsx'" in response.json()["detail"]
|
||||
assert "Source type must be 'sqllab' or 'xlsx'" in response.json()["detail"]
|
||||
# #endregion test_map_columns_invalid_source_type
|
||||
# #region test_generate_docs_success [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
|
||||
@@ -202,11 +202,24 @@ def test_map_columns_empty_ids(mock_deps):
|
||||
"""@PRE: dataset_ids must be non-empty."""
|
||||
response = client.post(
|
||||
"/api/datasets/map-columns",
|
||||
json={"env_id": "prod", "dataset_ids": [], "source_type": "postgresql"},
|
||||
json={"env_id": "prod", "dataset_ids": [], "source_type": "sqllab", "database_id": 1},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "At least one dataset ID must be provided" in response.json()["detail"]
|
||||
# #endregion test_map_columns_empty_ids
|
||||
# #region test_map_columns_missing_database_id [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
|
||||
# @BRIEF Validate map-columns rejects sqllab source without database_id.
|
||||
# @TEST: POST /api/datasets/map-columns returns 400 for sqllab without database_id
|
||||
# @POST: Returns 400 error
|
||||
def test_map_columns_missing_database_id(mock_deps):
|
||||
response = client.post(
|
||||
"/api/datasets/map-columns",
|
||||
json={"env_id": "prod", "dataset_ids": [1], "source_type": "sqllab"},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "database_id is required" in response.json()["detail"]
|
||||
# #endregion test_map_columns_missing_database_id
|
||||
# #region test_generate_docs_empty_ids [TYPE Function]
|
||||
# @RELATION BINDS_TO -> [DatasetsApiTests:Module]
|
||||
# @BRIEF Validate generate-docs rejects empty dataset identifier lists.
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
# #region ConnectionsRouter [C:3] [TYPE Module] [SEMANTICS fastapi, connection, api, search]
|
||||
# @BRIEF Defines the FastAPI router for managing external database connections.
|
||||
# @LAYER: API
|
||||
# @RELATION DEPENDS_ON -> [get_db]
|
||||
# @RELATION DEPENDS_ON -> [ConnectionConfig]
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...core.database import ensure_connection_configs_table, get_db
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.connection import ConnectionConfig
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# #region _ensure_connections_schema [C:3] [TYPE Function]
|
||||
# @BRIEF Ensures the connection_configs table exists before CRUD access.
|
||||
# @PRE: db is an active SQLAlchemy session.
|
||||
# @POST: The current bind can safely query ConnectionConfig.
|
||||
# @RELATION CALLS -> ensure_connection_configs_table
|
||||
def _ensure_connections_schema(db: Session):
|
||||
with belief_scope("ConnectionsRouter.ensure_schema"):
|
||||
ensure_connection_configs_table(db.get_bind())
|
||||
|
||||
|
||||
# #endregion _ensure_connections_schema
|
||||
|
||||
|
||||
# #region ConnectionSchema [C:3] [TYPE Class]
|
||||
# @BRIEF Pydantic model for connection response.
|
||||
# @RELATION BINDS_TO -> ConnectionConfig
|
||||
class ConnectionSchema(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
type: str
|
||||
host: str | None = None
|
||||
port: int | None = None
|
||||
database: str | None = None
|
||||
username: str | None = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
orm_mode = True
|
||||
|
||||
|
||||
# #endregion ConnectionSchema
|
||||
|
||||
|
||||
# #region ConnectionCreate [C:3] [TYPE Class]
|
||||
# @BRIEF Pydantic model for creating a connection.
|
||||
# @RELATION BINDS_TO -> ConnectionConfig
|
||||
class ConnectionCreate(BaseModel):
|
||||
name: str
|
||||
type: str
|
||||
host: str | None = None
|
||||
port: int | None = None
|
||||
database: str | None = None
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
|
||||
|
||||
# #endregion ConnectionCreate
|
||||
|
||||
|
||||
# #region list_connections [C:3] [TYPE Function]
|
||||
# @BRIEF Lists all saved connections.
|
||||
# @PRE: Database session is active.
|
||||
# @POST: Returns list of connection configs.
|
||||
# @RELATION CALLS -> _ensure_connections_schema
|
||||
# @RELATION DEPENDS_ON -> ConnectionConfig
|
||||
@router.get("", response_model=list[ConnectionSchema])
|
||||
async def list_connections(db: Session = Depends(get_db)):
|
||||
with belief_scope("ConnectionsRouter.list_connections"):
|
||||
_ensure_connections_schema(db)
|
||||
connections = db.query(ConnectionConfig).all()
|
||||
return connections
|
||||
|
||||
|
||||
# #endregion list_connections
|
||||
|
||||
|
||||
# #region create_connection [C:3] [TYPE Function]
|
||||
# @BRIEF Creates a new connection configuration.
|
||||
# @PRE: Connection name is unique.
|
||||
# @POST: Connection is saved to DB.
|
||||
# @RELATION CALLS -> _ensure_connections_schema
|
||||
# @RELATION DEPENDS_ON -> ConnectionConfig
|
||||
@router.post("", response_model=ConnectionSchema, status_code=status.HTTP_201_CREATED)
|
||||
async def create_connection(
|
||||
connection: ConnectionCreate, db: Session = Depends(get_db)
|
||||
):
|
||||
with belief_scope("ConnectionsRouter.create_connection", f"name={connection.name}"):
|
||||
_ensure_connections_schema(db)
|
||||
db_connection = ConnectionConfig(**connection.dict())
|
||||
db.add(db_connection)
|
||||
db.commit()
|
||||
db.refresh(db_connection)
|
||||
logger.info(
|
||||
f"[ConnectionsRouter.create_connection][Success] Created connection {db_connection.id}"
|
||||
)
|
||||
return db_connection
|
||||
|
||||
|
||||
# #endregion create_connection
|
||||
|
||||
|
||||
# #region delete_connection [C:3] [TYPE Function]
|
||||
# @BRIEF Deletes a connection configuration.
|
||||
# @PRE: Connection ID exists.
|
||||
# @POST: Connection is removed from DB.
|
||||
# @RELATION CALLS -> _ensure_connections_schema
|
||||
# @RELATION DEPENDS_ON -> ConnectionConfig
|
||||
@router.delete("/{connection_id}", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_connection(connection_id: str, db: Session = Depends(get_db)):
|
||||
with belief_scope("ConnectionsRouter.delete_connection", f"id={connection_id}"):
|
||||
_ensure_connections_schema(db)
|
||||
db_connection = (
|
||||
db.query(ConnectionConfig)
|
||||
.filter(ConnectionConfig.id == connection_id)
|
||||
.first()
|
||||
)
|
||||
if not db_connection:
|
||||
logger.error(
|
||||
f"[ConnectionsRouter.delete_connection][State] Connection {connection_id} not found"
|
||||
)
|
||||
raise HTTPException(status_code=404, detail="Connection not found")
|
||||
db.delete(db_connection)
|
||||
db.commit()
|
||||
logger.info(
|
||||
f"[ConnectionsRouter.delete_connection][Success] Deleted connection {connection_id}"
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
# #endregion delete_connection
|
||||
|
||||
# #endregion ConnectionsRouter
|
||||
@@ -316,8 +316,9 @@ async def get_datasets(
|
||||
class MapColumnsRequest(BaseModel):
|
||||
env_id: str = Field(..., description="Environment ID")
|
||||
dataset_ids: list[int] = Field(..., description="List of dataset IDs to map")
|
||||
source_type: str = Field(..., description="Source type: 'postgresql' or 'xlsx'")
|
||||
connection_id: str | None = Field(None, description="Connection ID for PostgreSQL source")
|
||||
source_type: str = Field(..., description="Source type: 'sqllab' or 'xlsx'")
|
||||
database_id: int | None = Field(None, description="Superset database ID for SQL Lab source")
|
||||
sql_query: str | None = Field(None, description="Custom SQL query for SQL Lab (optional, defaults to information_schema.columns query)")
|
||||
file_data: str | None = Field(None, description="File path or data for XLSX source")
|
||||
# #endregion MapColumnsRequest
|
||||
|
||||
@@ -344,9 +345,13 @@ async def map_columns(
|
||||
raise HTTPException(status_code=400, detail="At least one dataset ID must be provided")
|
||||
|
||||
# Validate source type
|
||||
if request.source_type not in ['postgresql', 'xlsx']:
|
||||
if request.source_type not in ['sqllab', 'xlsx']:
|
||||
logger.error(f"[map_columns][Coherence:Failed] Invalid source type: {request.source_type}")
|
||||
raise HTTPException(status_code=400, detail="Source type must be 'postgresql' or 'xlsx'")
|
||||
raise HTTPException(status_code=400, detail="Source type must be 'sqllab' or 'xlsx'")
|
||||
|
||||
# Validate database_id for sqllab source
|
||||
if request.source_type == 'sqllab' and not request.database_id:
|
||||
raise HTTPException(status_code=400, detail="database_id is required for 'sqllab' source type")
|
||||
|
||||
# Validate environment exists
|
||||
environments = config_manager.get_environments()
|
||||
@@ -361,8 +366,9 @@ async def map_columns(
|
||||
task_params = {
|
||||
'env': request.env_id,
|
||||
'dataset_id': request.dataset_ids[0] if request.dataset_ids else None,
|
||||
'source': request.source_type,
|
||||
'connection_id': request.connection_id,
|
||||
'source': 'sqllab' if request.source_type == 'sqllab' else 'excel',
|
||||
'database_id': request.database_id,
|
||||
'sql_query': request.sql_query,
|
||||
'file_data': request.file_data
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,6 @@ from .api.routes import (
|
||||
assistant,
|
||||
clean_release,
|
||||
clean_release_v2,
|
||||
connections,
|
||||
dashboards,
|
||||
dataset_review,
|
||||
datasets,
|
||||
@@ -215,7 +214,6 @@ app.add_middleware(TraceContextMiddleware)
|
||||
# @RELATION DEPENDS_ON -> [PluginsRouter]
|
||||
# @RELATION DEPENDS_ON -> [TasksRouter]
|
||||
# @RELATION DEPENDS_ON -> [SettingsRouter]
|
||||
# @RELATION DEPENDS_ON -> [ConnectionsRouter]
|
||||
# @RELATION DEPENDS_ON -> [ReportsRouter]
|
||||
# @RELATION DEPENDS_ON -> [LlmRoutes]
|
||||
# @RELATION DEPENDS_ON -> [CleanReleaseV2Api]
|
||||
@@ -225,9 +223,6 @@ app.include_router(admin.router)
|
||||
app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"])
|
||||
app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"])
|
||||
app.include_router(settings.router, prefix="/api/settings", tags=["Settings"])
|
||||
app.include_router(
|
||||
connections.router, prefix="/api/settings/connections", tags=["Connections"]
|
||||
)
|
||||
app.include_router(environments.router, tags=["Environments"])
|
||||
app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"])
|
||||
app.include_router(migration.router)
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
import os
|
||||
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
# #region AuthConfig [TYPE Class]
|
||||
@@ -23,19 +23,21 @@ from pydantic_settings import BaseSettings
|
||||
# @POST: Returns a configuration object with validated settings.
|
||||
# @RELATION INHERITS -> pydantic_settings.BaseSettings
|
||||
class AuthConfig(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
# JWT Settings
|
||||
SECRET_KEY: str = Field(default="", env="AUTH_SECRET_KEY")
|
||||
SECRET_KEY: str = Field(default="", validation_alias="AUTH_SECRET_KEY")
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 480
|
||||
REFRESH_TOKEN_EXPIRE_DAYS: int = 7
|
||||
|
||||
# Database Settings
|
||||
AUTH_DATABASE_URL: str = Field(default="", env="AUTH_DATABASE_URL")
|
||||
AUTH_DATABASE_URL: str = Field(default="", validation_alias="AUTH_DATABASE_URL")
|
||||
|
||||
# ADFS Settings
|
||||
ADFS_CLIENT_ID: str = Field(default="", env="ADFS_CLIENT_ID")
|
||||
ADFS_CLIENT_SECRET: str = Field(default="", env="ADFS_CLIENT_SECRET")
|
||||
ADFS_METADATA_URL: str = Field(default="", env="ADFS_METADATA_URL")
|
||||
ADFS_CLIENT_ID: str = Field(default="", validation_alias="ADFS_CLIENT_ID")
|
||||
ADFS_CLIENT_SECRET: str = Field(default="", validation_alias="ADFS_CLIENT_SECRET")
|
||||
ADFS_METADATA_URL: str = Field(default="", validation_alias="ADFS_METADATA_URL")
|
||||
|
||||
@field_validator("SECRET_KEY", mode="after")
|
||||
@classmethod
|
||||
@@ -60,9 +62,6 @@ class AuthConfig(BaseSettings):
|
||||
"Set it in .env or export it before starting the server."
|
||||
)
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
extra = "ignore"
|
||||
# #endregion AuthConfig
|
||||
|
||||
# #region auth_config [TYPE Variable]
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
# @LAYER: Infrastructure
|
||||
# @RELATION DEPENDS_ON -> [MappingModels]
|
||||
# @RELATION DEPENDS_ON -> [auth_config]
|
||||
# @RELATION DEPENDS_ON -> [ConnectionConfig]
|
||||
#
|
||||
# @INVARIANT: A single engine instance is used for the entire application.
|
||||
|
||||
@@ -20,13 +19,11 @@ from ..models import (
|
||||
auth as _auth_models, # noqa: F401
|
||||
clean_release as _clean_release_models, # noqa: F401
|
||||
config as _config_models, # noqa: F401
|
||||
connection as _connection_models, # noqa: F401
|
||||
dataset_review as _dataset_review_models, # noqa: F401
|
||||
llm as _llm_models, # noqa: F401
|
||||
profile as _profile_models, # noqa: F401
|
||||
task as _task_models, # noqa: F401
|
||||
)
|
||||
from ..models.connection import ConnectionConfig
|
||||
from ..models.mapping import Base
|
||||
from .auth.config import auth_config
|
||||
from .logger import belief_scope, logger
|
||||
@@ -367,26 +364,6 @@ def _ensure_auth_users_columns(bind_engine):
|
||||
# #endregion _ensure_auth_users_columns
|
||||
|
||||
|
||||
# #region ensure_connection_configs_table [C:3] [TYPE Function]
|
||||
# @BRIEF Ensures the external connection registry table exists in the main database.
|
||||
# @PRE: bind_engine points to the application database.
|
||||
# @POST: connection_configs table exists without dropping existing data.
|
||||
# @RELATION DEPENDS_ON -> [ConnectionConfig]
|
||||
def ensure_connection_configs_table(bind_engine):
|
||||
with belief_scope("ensure_connection_configs_table"):
|
||||
try:
|
||||
ConnectionConfig.__table__.create(bind=bind_engine, checkfirst=True)
|
||||
except Exception as migration_error:
|
||||
logger.explore(
|
||||
"ConnectionConfig table ensure failed",
|
||||
extra={"src": "database", "error": str(migration_error)},
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
# #endregion ensure_connection_configs_table
|
||||
|
||||
|
||||
# #region _ensure_filter_source_enum_values [C:3] [TYPE Function]
|
||||
# @BRIEF Adds missing FilterSource enum values to the PostgreSQL native filtersource type.
|
||||
# @PRE: bind_engine points to application database with imported_filters table.
|
||||
@@ -686,7 +663,6 @@ def _ensure_dictionary_entries_columns(bind_engine):
|
||||
# @PRE: engine, tasks_engine and auth_engine are initialized.
|
||||
# @POST: Database tables created in all databases.
|
||||
# @SIDE_EFFECT: Creates physical database files if they don't exist.
|
||||
# @RELATION CALLS -> [ensure_connection_configs_table]
|
||||
# @RELATION CALLS -> [_ensure_filter_source_enum_values]
|
||||
# @RELATION CALLS -> [_ensure_dataset_review_session_columns]
|
||||
# @RELATION CALLS -> [_ensure_dictionary_entries_columns]
|
||||
@@ -700,7 +676,6 @@ def init_db():
|
||||
_ensure_user_dashboard_preferences_health_columns(engine)
|
||||
_ensure_git_server_configs_columns(engine)
|
||||
_ensure_auth_users_columns(auth_engine)
|
||||
ensure_connection_configs_table(engine)
|
||||
_ensure_filter_source_enum_values(engine)
|
||||
_ensure_dataset_review_session_columns(engine)
|
||||
_ensure_translation_jobs_columns(engine)
|
||||
|
||||
@@ -1,21 +1,20 @@
|
||||
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, superset, dataset-mapper]
|
||||
# #region DatasetMapperModule [TYPE Module] [SEMANTICS dataset, superset, dataset-mapper, sqllab]
|
||||
#
|
||||
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_map) в датасетах Superset, извлекая их из PostgreSQL или XLSX-файлов.
|
||||
# @BRIEF Этот модуль отвечает за обновление метаданных (verbose_name) в датасетах Superset,
|
||||
# извлекая их из Superset SQL Lab (любая БД, подключённая к Superset) или XLSX-файлов.
|
||||
# @LAYER: Domain
|
||||
# @RELATION DEPENDS_ON -> backend.core.superset_client
|
||||
# @RELATION DEPENDS_ON -> pandas
|
||||
# @RELATION DEPENDS_ON -> psycopg2
|
||||
# @PUBLIC_API: DatasetMapper
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd # type: ignore
|
||||
import psycopg2 # type: ignore
|
||||
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
|
||||
# #region DatasetMapper [TYPE Class]
|
||||
# @BRIEF Класс для меппинга и обновления verbose_map в датасетах Superset.
|
||||
# @BRIEF Класс для маппинга и обновления verbose_name в датасетах Superset.
|
||||
class DatasetMapper:
|
||||
# #region __init__ [TYPE Function]
|
||||
# @PURPOSE: Initializes the mapper.
|
||||
@@ -23,77 +22,78 @@ class DatasetMapper:
|
||||
def __init__(self):
|
||||
pass
|
||||
# #endregion __init__
|
||||
# #region get_postgres_comments [TYPE Function]
|
||||
# @PURPOSE: Извлекает комментарии к колонкам из системного каталога PostgreSQL.
|
||||
# @PRE: db_config должен содержать валидные параметры подключения (host, port, user, password, dbname).
|
||||
# @PRE: table_name и table_schema должны быть строками.
|
||||
# @POST: Возвращается словарь, где ключи - имена колонок, значения - комментарии из БД.
|
||||
# @THROW: Exception - При ошибках подключения или выполнения запроса к БД.
|
||||
# @PARAM: db_config (Dict) - Конфигурация для подключения к БД.
|
||||
# @PARAM: table_name (str) - Имя таблицы.
|
||||
# @PARAM: table_schema (str) - Схема таблицы.
|
||||
# @RETURN: Dict[str, str] - Словарь с комментариями к колонкам.
|
||||
def get_postgres_comments(self, db_config: dict, table_name: str, table_schema: str) -> dict[str, str]:
|
||||
with belief_scope("Fetch comments from PostgreSQL"):
|
||||
app_logger.info("[get_postgres_comments][Enter] Fetching comments from PostgreSQL for %s.%s.", table_schema, table_name)
|
||||
query = f"""
|
||||
SELECT
|
||||
cols.column_name,
|
||||
CASE
|
||||
WHEN pg_catalog.col_description(
|
||||
(SELECT c.oid
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relname = cols.table_name
|
||||
AND n.nspname = cols.table_schema),
|
||||
cols.ordinal_position::int
|
||||
) LIKE '%|%' THEN
|
||||
split_part(
|
||||
pg_catalog.col_description(
|
||||
(SELECT c.oid
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relname = cols.table_name
|
||||
AND n.nspname = cols.table_schema),
|
||||
cols.ordinal_position::int
|
||||
),
|
||||
'|',
|
||||
1
|
||||
)
|
||||
ELSE
|
||||
pg_catalog.col_description(
|
||||
(SELECT c.oid
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE c.relname = cols.table_name
|
||||
AND n.nspname = cols.table_schema),
|
||||
cols.ordinal_position::int
|
||||
)
|
||||
END AS column_comment
|
||||
FROM
|
||||
information_schema.columns cols
|
||||
WHERE cols.table_catalog = '{db_config.get('dbname')}' AND cols.table_name = '{table_name}' AND cols.table_schema = '{table_schema}';
|
||||
"""
|
||||
comments = {}
|
||||
try:
|
||||
with psycopg2.connect(**db_config) as conn, conn.cursor() as cursor:
|
||||
cursor.execute(query)
|
||||
for row in cursor.fetchall():
|
||||
if row[1]:
|
||||
comments[row[0]] = row[1]
|
||||
app_logger.info("[get_postgres_comments][Success] Fetched %d comments.", len(comments))
|
||||
except Exception as e:
|
||||
app_logger.error("[get_postgres_comments][Failure] %s", e, exc_info=True)
|
||||
raise
|
||||
return comments
|
||||
# #endregion get_postgres_comments
|
||||
|
||||
# #region get_sqllab_mappings [TYPE Function]
|
||||
# @PURPOSE: Извлекает маппинги column_name -> verbose_name через SQL Lab Superset.
|
||||
# @PRE: sqllab_executor должен быть инициализирован с database_id.
|
||||
# @PRE: dataset_id должен существовать в Superset.
|
||||
# @POST: Возвращается словарь column_name -> verbose_name из результатов SQL-запроса.
|
||||
# Если sql_query не указан, строится запрос к information_schema.columns.
|
||||
# @PARAM: client (SupersetClient) - Авторизованный клиент Superset.
|
||||
# @PARAM: dataset_id (int) - ID датасета (для получения table_name/schema).
|
||||
# @PARAM: sqllab_executor (SupersetSqlLabExecutor) - Исполнитель SQL Lab.
|
||||
# @PARAM: database_id (int) - ID базы данных в Superset.
|
||||
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (должен вернуть column_name + verbose_name).
|
||||
# @RETURN: Dict[str, str] - Словарь column_name -> verbose_name.
|
||||
def get_sqllab_mappings(
|
||||
self,
|
||||
client: Any,
|
||||
dataset_id: int,
|
||||
sqllab_executor: Any,
|
||||
database_id: int,
|
||||
sql_query: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
with belief_scope("Fetch mappings via SQL Lab"):
|
||||
# Получаем датасет из Superset, чтобы узнать table_name и schema
|
||||
dataset_response = client.get_dataset(dataset_id)
|
||||
dataset_data = dataset_response.get("result", dataset_response)
|
||||
table_name = dataset_data.get("table_name")
|
||||
table_schema = dataset_data.get("schema") or "public"
|
||||
database_name = dataset_data.get("database", {}).get("database_name", "")
|
||||
|
||||
app_logger.info(
|
||||
"[get_sqllab_mappings] Dataset %d: %s.%s (db: %s)",
|
||||
dataset_id, table_schema, table_name, database_name
|
||||
)
|
||||
|
||||
# Если SQL-запрос не указан, строим дефолтный для information_schema
|
||||
if not sql_query:
|
||||
sql_query = (
|
||||
"SELECT column_name, pg_catalog.col_description(\n"
|
||||
" (quote_ident(table_schema) || '.' || quote_ident(table_name))::regclass::oid,\n"
|
||||
" ordinal_position\n"
|
||||
") AS verbose_name\n"
|
||||
f"FROM information_schema.columns\n"
|
||||
f"WHERE table_schema = '{table_schema}' AND table_name = '{table_name}';\n"
|
||||
)
|
||||
app_logger.info("[get_sqllab_mappings] Using default information_schema query for %s.%s", table_schema, table_name)
|
||||
else:
|
||||
app_logger.info("[get_sqllab_mappings] Using custom SQL query")
|
||||
|
||||
# Выполняем запрос через SQL Lab
|
||||
app_logger.info("[get_sqllab_mappings] Executing SQL Lab query on database %d...", database_id)
|
||||
result = sqllab_executor.execute_and_poll(sql_query, database_id)
|
||||
rows = result.get("results") or result.get("data") or result.get("result", [])
|
||||
app_logger.info("[get_sqllab_mappings] SQL Lab returned %d rows", len(rows))
|
||||
|
||||
mappings: dict[str, str] = {}
|
||||
for row in rows:
|
||||
col_name = row.get("column_name")
|
||||
verbose_name = row.get("verbose_name")
|
||||
if col_name and verbose_name:
|
||||
mappings[col_name] = str(verbose_name)
|
||||
|
||||
app_logger.info("[get_sqllab_mappings] Extracted %d mappings", len(mappings))
|
||||
return mappings
|
||||
# #endregion get_sqllab_mappings
|
||||
|
||||
# #region load_excel_mappings [TYPE Function]
|
||||
# @PURPOSE: Загружает меппинги 'column_name' -> 'column_comment' из XLSX файла.
|
||||
# @PURPOSE: Загружает маппинги column_name -> verbose_name из XLSX файла.
|
||||
# @PRE: file_path должен указывать на существующий XLSX файл.
|
||||
# @POST: Возвращается словарь с меппингами из файла.
|
||||
# @POST: Возвращается словарь с маппингами из файла.
|
||||
# @THROW: Exception - При ошибках чтения файла или парсинга.
|
||||
# @PARAM: file_path (str) - Путь к XLSX файлу.
|
||||
# @RETURN: Dict[str, str] - Словарь с меппингами.
|
||||
# @RETURN: Dict[str, str] - Словарь с маппингами.
|
||||
def load_excel_mappings(self, file_path: str) -> dict[str, str]:
|
||||
with belief_scope("Load mappings from Excel"):
|
||||
app_logger.info("[load_excel_mappings][Enter] Loading mappings from %s.", file_path)
|
||||
@@ -106,37 +106,48 @@ class DatasetMapper:
|
||||
app_logger.error("[load_excel_mappings][Failure] %s", e, exc_info=True)
|
||||
raise
|
||||
# #endregion load_excel_mappings
|
||||
|
||||
# #region run_mapping [TYPE Function]
|
||||
# @PURPOSE: Основная функция для выполнения меппинга и обновления verbose_map датасета в Superset.
|
||||
# @PURPOSE: Основная функция для выполнения маппинга и обновления verbose_name датасета в Superset.
|
||||
# @PRE: superset_client должен быть авторизован.
|
||||
# @PRE: dataset_id должен быть существующим ID в Superset.
|
||||
# @POST: Если найдены изменения, датасет в Superset обновлен через API.
|
||||
# @RELATION: CALLS -> self.get_postgres_comments
|
||||
# @RELATION: CALLS -> self.get_sqllab_mappings
|
||||
# @RELATION: CALLS -> self.load_excel_mappings
|
||||
# @RELATION: CALLS -> superset_client.get_dataset
|
||||
# @RELATION: CALLS -> superset_client.update_dataset
|
||||
# @PARAM: superset_client (Any) - Клиент Superset.
|
||||
# @PARAM: dataset_id (int) - ID датасета для обновления.
|
||||
# @PARAM: source (str) - Источник данных ('postgres', 'excel', 'both').
|
||||
# @PARAM: postgres_config (Optional[Dict]) - Конфигурация для подключения к PostgreSQL.
|
||||
# @PARAM: source (str) - Источник данных ('sqllab' или 'excel').
|
||||
# @PARAM: sqllab_executor (Optional[Any]) - Исполнитель SQL Lab (для sqllab source).
|
||||
# @PARAM: database_id (Optional[int]) - ID базы данных Superset (для sqllab source).
|
||||
# @PARAM: sql_query (Optional[str]) - Произвольный SQL-запрос (для sqllab source).
|
||||
# @PARAM: excel_path (Optional[str]) - Путь к XLSX файлу.
|
||||
# @PARAM: table_name (Optional[str]) - Имя таблицы в PostgreSQL.
|
||||
# @PARAM: table_schema (Optional[str]) - Схема таблицы в PostgreSQL.
|
||||
def run_mapping(self, superset_client: Any, dataset_id: int, source: str, postgres_config: dict | None = None, excel_path: str | None = None, table_name: str | None = None, table_schema: str | None = None):
|
||||
def run_mapping(
|
||||
self,
|
||||
superset_client: Any,
|
||||
dataset_id: int,
|
||||
source: str,
|
||||
sqllab_executor: Any = None,
|
||||
database_id: int | None = None,
|
||||
sql_query: str | None = None,
|
||||
excel_path: str | None = None,
|
||||
):
|
||||
with belief_scope(f"Run dataset mapping for ID {dataset_id}"):
|
||||
app_logger.info("[run_mapping][Enter] Starting dataset mapping for ID %d from source '%s'.", dataset_id, source)
|
||||
mappings: dict[str, str] = {}
|
||||
|
||||
try:
|
||||
if source in ['postgres', 'both']:
|
||||
assert postgres_config and table_name and table_schema, "Postgres config is required."
|
||||
mappings.update(self.get_postgres_comments(postgres_config, table_name, table_schema))
|
||||
if source in ['excel', 'both']:
|
||||
assert excel_path, "Excel path is required."
|
||||
if source == "sqllab":
|
||||
assert sqllab_executor and database_id, "sqllab_executor and database_id are required for sqllab source."
|
||||
mappings.update(self.get_sqllab_mappings(superset_client, dataset_id, sqllab_executor, database_id, sql_query))
|
||||
elif source == "excel":
|
||||
assert excel_path, "excel_path is required."
|
||||
mappings.update(self.load_excel_mappings(excel_path))
|
||||
if source not in ['postgres', 'excel', 'both']:
|
||||
else:
|
||||
app_logger.error("[run_mapping][Failure] Invalid source: %s.", source)
|
||||
return
|
||||
|
||||
dataset_response = superset_client.get_dataset(dataset_id)
|
||||
dataset_data = dataset_response['result']
|
||||
|
||||
@@ -171,6 +182,7 @@ class DatasetMapper:
|
||||
changes_made = True
|
||||
|
||||
updated_columns.append(new_column)
|
||||
|
||||
updated_metrics = []
|
||||
for metric in dataset_data.get("metrics", []):
|
||||
new_metric = {
|
||||
@@ -187,6 +199,7 @@ class DatasetMapper:
|
||||
"uuid": metric.get("uuid"),
|
||||
}
|
||||
updated_metrics.append({k: v for k, v in new_metric.items() if v is not None})
|
||||
|
||||
if changes_made:
|
||||
payload_for_update = {
|
||||
"database_id": dataset_data.get("database", {}).get("id"),
|
||||
@@ -222,4 +235,4 @@ class DatasetMapper:
|
||||
return
|
||||
# #endregion run_mapping
|
||||
# #endregion DatasetMapper
|
||||
# #endregion DatasetMapperModule
|
||||
# #endregion DatasetMapperModule
|
||||
@@ -1,34 +0,0 @@
|
||||
# #region ConnectionModels [C:1] [TYPE Module] [SEMANTICS sqlalchemy, connection, model, schema, connection-config]
|
||||
#
|
||||
# @BRIEF Defines the database schema for external database connection configurations.
|
||||
# @LAYER: Domain
|
||||
# @RELATION DEPENDS_ON -> sqlalchemy
|
||||
#
|
||||
# @INVARIANT: All primary keys are UUID strings.
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Column, DateTime, Integer, String
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from .mapping import Base
|
||||
|
||||
|
||||
# #region ConnectionConfig [C:1] [TYPE Class]
|
||||
# @BRIEF Stores credentials for external databases used for column mapping.
|
||||
class ConnectionConfig(Base):
|
||||
__tablename__ = "connection_configs"
|
||||
|
||||
id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
name = Column(String, nullable=False)
|
||||
type = Column(String, nullable=False) # e.g., "postgres"
|
||||
host = Column(String, nullable=True)
|
||||
port = Column(Integer, nullable=True)
|
||||
database = Column(String, nullable=True)
|
||||
username = Column(String, nullable=True)
|
||||
password = Column(String, nullable=True) # Encrypted/Obfuscated password
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
# #endregion ConnectionConfig
|
||||
|
||||
# #endregion ConnectionModels
|
||||
@@ -1,18 +1,16 @@
|
||||
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, excel, mapping]
|
||||
# @BRIEF Implements a plugin for mapping dataset columns using external database connections or Excel files.
|
||||
# #region MapperPluginModule [TYPE Module] [SEMANTICS mapper, dataset, column, sqllab, excel, mapping]
|
||||
# @BRIEF Implements a plugin for mapping dataset columns using Superset SQL Lab or Excel files.
|
||||
# @LAYER: Plugins
|
||||
# @RELATION Inherits from PluginBase. Uses DatasetMapper from superset_tool.
|
||||
# @RELATION Inherits from PluginBase. Uses DatasetMapper and SupersetSqlLabExecutor.
|
||||
# @RELATION USES -> TaskContext
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ..core.database import SessionLocal
|
||||
from ..core.logger import belief_scope, logger
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.task_manager.context import TaskContext
|
||||
from ..core.utils.dataset_mapper import DatasetMapper
|
||||
from ..models.connection import ConnectionConfig
|
||||
|
||||
|
||||
# #region MapperPlugin [TYPE Class]
|
||||
@@ -52,7 +50,7 @@ class MapperPlugin(PluginBase):
|
||||
# @RETURN: str - Plugin description.
|
||||
def description(self) -> str:
|
||||
with belief_scope("description"):
|
||||
return "Map dataset column verbose names using PostgreSQL comments or Excel files."
|
||||
return "Map dataset column verbose names using Superset SQL Lab or Excel files."
|
||||
# endregion description
|
||||
|
||||
@property
|
||||
@@ -98,24 +96,19 @@ class MapperPlugin(PluginBase):
|
||||
"source": {
|
||||
"type": "string",
|
||||
"title": "Mapping Source",
|
||||
"enum": ["postgres", "excel"],
|
||||
"default": "postgres"
|
||||
"enum": ["sqllab", "excel"],
|
||||
"default": "sqllab"
|
||||
},
|
||||
"connection_id": {
|
||||
"type": "string",
|
||||
"title": "Saved Connection",
|
||||
"description": "The ID of a saved database connection (for postgres source)."
|
||||
"database_id": {
|
||||
"type": "integer",
|
||||
"title": "Superset Database ID",
|
||||
"description": "Target database ID in Superset (for sqllab source)."
|
||||
},
|
||||
"table_name": {
|
||||
"sql_query": {
|
||||
"type": "string",
|
||||
"title": "Table Name",
|
||||
"description": "Target table name in PostgreSQL."
|
||||
},
|
||||
"table_schema": {
|
||||
"type": "string",
|
||||
"title": "Table Schema",
|
||||
"description": "Target table schema in PostgreSQL.",
|
||||
"default": "public"
|
||||
"title": "SQL Query",
|
||||
"description": "Custom SQL query returning column_name and verbose_name columns. Default: information_schema.columns.",
|
||||
"default": ""
|
||||
},
|
||||
"excel_path": {
|
||||
"type": "string",
|
||||
@@ -145,7 +138,6 @@ class MapperPlugin(PluginBase):
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
db_log = log.with_source("postgres") if context else log
|
||||
|
||||
if not env_name or dataset_id is None or not source:
|
||||
log.error("Missing required parameters: env, dataset_id, source")
|
||||
@@ -162,46 +154,37 @@ class MapperPlugin(PluginBase):
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
|
||||
postgres_config = None
|
||||
if source == "postgres":
|
||||
connection_id = params.get("connection_id")
|
||||
if not connection_id:
|
||||
log.error("connection_id is required for postgres source.")
|
||||
raise ValueError("connection_id is required for postgres source.")
|
||||
|
||||
# Load connection from DB
|
||||
db = SessionLocal()
|
||||
try:
|
||||
conn_config = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first()
|
||||
if not conn_config:
|
||||
db_log.error(f"Connection {connection_id} not found.")
|
||||
raise ValueError(f"Connection {connection_id} not found.")
|
||||
|
||||
postgres_config = {
|
||||
'dbname': conn_config.database,
|
||||
'user': conn_config.username,
|
||||
'password': conn_config.password,
|
||||
'host': conn_config.host,
|
||||
'port': str(conn_config.port) if conn_config.port else '5432'
|
||||
}
|
||||
db_log.debug(f"Loaded connection config for {conn_config.host}:{conn_config.port}/{conn_config.database}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
log.info(f"Starting mapping for dataset {dataset_id} in {env_name}")
|
||||
|
||||
mapper = DatasetMapper()
|
||||
|
||||
try:
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source=source,
|
||||
postgres_config=postgres_config,
|
||||
excel_path=params.get("excel_path"),
|
||||
table_name=params.get("table_name"),
|
||||
table_schema=params.get("table_schema") or "public"
|
||||
)
|
||||
if source == "sqllab":
|
||||
# Use Superset SQL Lab to query database metadata
|
||||
from ..plugins.translate.superset_executor import SupersetSqlLabExecutor
|
||||
executor = SupersetSqlLabExecutor(config_manager, env_name)
|
||||
database_id = params.get("database_id")
|
||||
if not database_id:
|
||||
log.error("database_id is required for sqllab source.")
|
||||
raise ValueError("database_id is required for sqllab source.")
|
||||
executor.resolve_database_id(target_database_id=str(database_id))
|
||||
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source="sqllab",
|
||||
sqllab_executor=executor,
|
||||
database_id=database_id,
|
||||
sql_query=params.get("sql_query"),
|
||||
)
|
||||
else:
|
||||
# Excel source
|
||||
mapper.run_mapping(
|
||||
superset_client=client,
|
||||
dataset_id=dataset_id,
|
||||
source="excel",
|
||||
excel_path=params.get("excel_path") or params.get("file_data")
|
||||
)
|
||||
superset_log.info(f"Mapping completed for dataset {dataset_id}")
|
||||
return {"status": "success", "dataset_id": dataset_id}
|
||||
except Exception as e:
|
||||
|
||||
@@ -54,7 +54,6 @@
|
||||
</button>
|
||||
<div class="absolute hidden group-hover:block bg-white shadow-lg rounded-md mt-1 py-2 w-48 z-10 border border-gray-100 before:absolute before:-top-2 before:left-0 before:right-0 before:h-2 before:content-[''] right-0">
|
||||
<a href="/settings" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_general}</a>
|
||||
<a href="/settings/connections" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_connections}</a>
|
||||
<a href="/settings/git" class="block px-4 py-2 text-sm text-gray-700 hover:bg-blue-50 hover:text-primary">{$t.nav.settings_git}</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
<!-- #region ConnectionForm [C:3] [TYPE Component] [SEMANTICS connection, form, database, crud, create] -->
|
||||
<!-- @BRIEF Component component: components/tools/ConnectionForm.svelte -->
|
||||
<!-- @LAYER UI -->
|
||||
<!--
|
||||
@SEMANTICS: connection, form, settings
|
||||
@PURPOSE: UI component for creating a new database connection configuration.
|
||||
@LAYER: UI
|
||||
@RELATION: USES -> frontend/src/services/connectionService.js
|
||||
-->
|
||||
<script>
|
||||
// [SECTION: IMPORTS]
|
||||
import { createConnection } from '../../services/connectionService.js';
|
||||
import { addToast } from '../../lib/toasts.js';
|
||||
import { t } from '../../lib/i18n';
|
||||
import { Button, Input, Card } from '../../lib/ui';
|
||||
// [/SECTION]
|
||||
let { onsuccess = () => {} } = $props();
|
||||
|
||||
let name = $state('');
|
||||
let type = $state('postgres');
|
||||
let host = $state('');
|
||||
let port = $state("5432");
|
||||
let database = $state('');
|
||||
let username = $state('');
|
||||
let password = $state('');
|
||||
let isSubmitting = $state(false);
|
||||
|
||||
// #region handleSubmit:Function [TYPE Function]
|
||||
// @PURPOSE: Submits the connection form to the backend.
|
||||
// @PRE: All required fields (name, host, database, username, password) must be filled.
|
||||
// @POST: A new connection is created via the connection service and the parent success callback is invoked.
|
||||
async function handleSubmit() {
|
||||
if (!name || !host || !database || !username || !password) {
|
||||
addToast($t.connections?.required_fields, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
isSubmitting = true;
|
||||
try {
|
||||
const newConnection = await createConnection({
|
||||
name, type, host, port: Number(port), database, username, password
|
||||
});
|
||||
addToast($t.connections?.created_success, 'success');
|
||||
onsuccess(newConnection);
|
||||
resetForm();
|
||||
} catch (e) {
|
||||
addToast(e.message, 'error');
|
||||
} finally {
|
||||
isSubmitting = false;
|
||||
}
|
||||
}
|
||||
// #endregion handleSubmit:Function
|
||||
|
||||
// #region resetForm:Function [TYPE Function]
|
||||
/* @PURPOSE: Resets the connection form fields to their default values.
|
||||
@PRE: None.
|
||||
@POST: All form input variables are reset.
|
||||
*/
|
||||
function resetForm() {
|
||||
name = '';
|
||||
host = '';
|
||||
port = "5432";
|
||||
database = '';
|
||||
username = '';
|
||||
password = '';
|
||||
}
|
||||
// #endregion resetForm:Function
|
||||
</script>
|
||||
|
||||
<!-- [SECTION: TEMPLATE] -->
|
||||
<Card title={$t.connections?.add_new}>
|
||||
<form onsubmit={(event) => { event.preventDefault(); handleSubmit(); }} class="space-y-6">
|
||||
<Input label={$t.connections?.name} bind:value={name} placeholder={$t.connections?.name_placeholder} />
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
<Input label={$t.connections?.host} bind:value={host} placeholder={$t.connections?.host_placeholder} />
|
||||
<Input label={$t.connections?.port} type="number" bind:value={port} />
|
||||
</div>
|
||||
|
||||
<Input label={$t.connections?.db_name} bind:value={database} />
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||
<Input label={$t.connections?.user} bind:value={username} />
|
||||
<Input label={$t.connections?.pass} type="password" bind:value={password} />
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end pt-2">
|
||||
<Button type="submit" disabled={isSubmitting} isLoading={isSubmitting}>
|
||||
{$t.connections?.create}
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
</Card>
|
||||
<!-- [/SECTION] -->
|
||||
<!-- #endregion ConnectionForm -->
|
||||
@@ -1,88 +0,0 @@
|
||||
<!-- #region ConnectionList [C:3] [TYPE Component] [SEMANTICS connection, list, crud, delete, manage] -->
|
||||
<!-- @BRIEF Component component: components/tools/ConnectionList.svelte -->
|
||||
<!-- @LAYER UI -->
|
||||
<!--
|
||||
@SEMANTICS: connection, list, settings
|
||||
@PURPOSE: UI component for listing and deleting saved database connection configurations.
|
||||
@LAYER: UI
|
||||
@RELATION: USES -> frontend/src/services/connectionService.js
|
||||
-->
|
||||
<script>
|
||||
// [SECTION: IMPORTS]
|
||||
import { onMount } from 'svelte';
|
||||
import { getConnections, deleteConnection } from '../../services/connectionService.js';
|
||||
import { addToast } from '../../lib/toasts.js';
|
||||
import { t } from '../../lib/i18n';
|
||||
import { Button, Card } from '../../lib/ui';
|
||||
// [/SECTION]
|
||||
|
||||
let connections = $state([]);
|
||||
let isLoading = $state(true);
|
||||
|
||||
// #region fetchConnections:Function [TYPE Function]
|
||||
// @PURPOSE: Fetches the list of connections from the backend.
|
||||
// @PRE: None.
|
||||
// @POST: connections array is populated.
|
||||
async function fetchConnections() {
|
||||
isLoading = true;
|
||||
try {
|
||||
connections = await getConnections();
|
||||
} catch (e) {
|
||||
addToast($t.connections?.fetch_failed, 'error');
|
||||
} finally {
|
||||
isLoading = false;
|
||||
}
|
||||
}
|
||||
// #endregion fetchConnections:Function
|
||||
|
||||
// #region handleDelete:Function [TYPE Function]
|
||||
// @PURPOSE: Deletes a connection configuration.
|
||||
// @PRE: id is provided and user confirms deletion.
|
||||
// @POST: Connection is deleted from backend and list is reloaded.
|
||||
async function handleDelete(id) {
|
||||
if (!confirm($t.connections?.delete_confirm)) return;
|
||||
|
||||
try {
|
||||
await deleteConnection(id);
|
||||
addToast($t.connections?.deleted_success, 'success');
|
||||
await fetchConnections();
|
||||
} catch (e) {
|
||||
addToast(e.message, 'error');
|
||||
}
|
||||
}
|
||||
// #endregion handleDelete:Function
|
||||
|
||||
onMount(fetchConnections);
|
||||
|
||||
// Expose fetchConnections to parent
|
||||
export { fetchConnections };
|
||||
</script>
|
||||
|
||||
<!-- [SECTION: TEMPLATE] -->
|
||||
<Card title={$t.connections?.saved} padding="none">
|
||||
<ul class="divide-y divide-gray-100">
|
||||
{#if isLoading}
|
||||
<li class="p-6 text-center text-gray-500">{$t.common.loading}</li>
|
||||
{:else if connections.length === 0}
|
||||
<li class="p-12 text-center text-gray-500 italic">{$t.connections?.no_saved}</li>
|
||||
{:else}
|
||||
{#each connections as conn}
|
||||
<li class="p-6 flex items-center justify-between hover:bg-gray-50 transition-colors">
|
||||
<div>
|
||||
<div class="text-sm font-medium text-primary truncate">{conn.name}</div>
|
||||
<div class="text-xs text-gray-400 mt-1 font-mono">{conn.type}://{conn.username}@{conn.host}:{conn.port}/{conn.database}</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="danger"
|
||||
size="sm"
|
||||
onclick={() => handleDelete(conn.id)}
|
||||
>
|
||||
{$t.connections?.delete}
|
||||
</Button>
|
||||
</li>
|
||||
{/each}
|
||||
{/if}
|
||||
</ul>
|
||||
</Card>
|
||||
<!-- [/SECTION] -->
|
||||
<!-- #endregion ConnectionList -->
|
||||
@@ -1,18 +1,16 @@
|
||||
<!-- #region MapperTool [C:3] [TYPE Component] [SEMANTICS mapper, dataset, column, mapping, postgresql] -->
|
||||
<!-- #region MapperTool [C:3] [TYPE Component] [SEMANTICS mapper, dataset, column, mapping, sqllab] -->
|
||||
<!-- @BRIEF Component component: components/tools/MapperTool.svelte -->
|
||||
<!-- @LAYER UI -->
|
||||
<!--
|
||||
@SEMANTICS: mapper, tool, dataset, postgresql, excel
|
||||
@PURPOSE: UI component for mapping dataset column verbose names using the MapperPlugin.
|
||||
@SEMANTICS: mapper, tool, dataset, sqllab, excel
|
||||
@PURPOSE: UI component for mapping dataset column verbose names using Superset SQL Lab or Excel files.
|
||||
@LAYER: UI
|
||||
@RELATION: USES -> frontend/src/services/toolsService.js
|
||||
@RELATION: USES -> frontend/src/services/connectionService.js
|
||||
-->
|
||||
<script>
|
||||
// [SECTION: IMPORTS]
|
||||
import { onMount } from 'svelte';
|
||||
import { runTask } from '../../services/toolsService.js';
|
||||
import { getConnections } from '../../services/connectionService.js';
|
||||
import { api } from '../../lib/api';
|
||||
import { selectedTask } from '../../lib/stores.js';
|
||||
import { addToast } from '../../lib/toasts.js';
|
||||
@@ -22,26 +20,23 @@
|
||||
// [/SECTION]
|
||||
|
||||
let envs = [];
|
||||
let connections = [];
|
||||
let selectedEnv = '';
|
||||
let datasetId = '';
|
||||
let source = 'postgres';
|
||||
let selectedConnection = '';
|
||||
let tableName = '';
|
||||
let tableSchema = 'public';
|
||||
let source = 'sqllab';
|
||||
let databaseId = '';
|
||||
let sqlQuery = '';
|
||||
let excelPath = '';
|
||||
let isRunning = false;
|
||||
let isGeneratingDocs = false;
|
||||
let generatedDoc = null;
|
||||
|
||||
// #region fetchData:Function [TYPE Function]
|
||||
// @PURPOSE: Fetches environments and saved connections.
|
||||
// @PURPOSE: Fetches environments.
|
||||
// @PRE: None.
|
||||
// @POST: envs and connections arrays are populated.
|
||||
// @POST: envs array is populated.
|
||||
async function fetchData() {
|
||||
try {
|
||||
envs = await api.fetchApi('/environments');
|
||||
connections = await getConnections();
|
||||
} catch (e) {
|
||||
addToast($t.mapper.errors.fetch_failed, 'error');
|
||||
}
|
||||
@@ -49,7 +44,7 @@
|
||||
// #endregion fetchData:Function
|
||||
|
||||
// #region handleRunMapper:Function [TYPE Function]
|
||||
// @PURPOSE: Triggers the MapperPlugin task.
|
||||
// @PURPOSE: Triggers the MapperPlugin task via new sqllab/excel sources.
|
||||
// @PRE: selectedEnv and datasetId are set; source-specific fields are valid.
|
||||
// @POST: Mapper task is started and selectedTask is updated.
|
||||
async function handleRunMapper() {
|
||||
@@ -58,8 +53,8 @@
|
||||
return;
|
||||
}
|
||||
|
||||
if (source === 'postgres' && (!selectedConnection || !tableName)) {
|
||||
addToast($t.mapper.errors.postgres_required, 'warning');
|
||||
if (source === 'sqllab' && !databaseId) {
|
||||
addToast($t.mapper.errors.sqllab_required, 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -75,9 +70,8 @@
|
||||
env: env.name,
|
||||
dataset_id: parseInt(datasetId),
|
||||
source,
|
||||
connection_id: selectedConnection,
|
||||
table_name: tableName,
|
||||
table_schema: tableSchema,
|
||||
database_id: databaseId ? parseInt(databaseId) : undefined,
|
||||
sql_query: sqlQuery || undefined,
|
||||
excel_path: excelPath
|
||||
});
|
||||
|
||||
@@ -169,8 +163,8 @@
|
||||
<legend class="block text-sm font-medium text-gray-700 mb-2">{$t.mapper.source}</legend>
|
||||
<div class="flex space-x-4">
|
||||
<label class="inline-flex items-center">
|
||||
<input type="radio" bind:group={source} value="postgres" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
|
||||
<span class="ml-2 text-sm text-gray-700">{$t.mapper.source_postgres}</span>
|
||||
<input type="radio" bind:group={source} value="sqllab" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
|
||||
<span class="ml-2 text-sm text-gray-700">{$t.mapper.source_sqllab}</span>
|
||||
</label>
|
||||
<label class="inline-flex items-center">
|
||||
<input type="radio" bind:group={source} value="excel" class="focus-visible:ring-primary-ring h-4 w-4 text-primary border-gray-300" />
|
||||
@@ -179,33 +173,23 @@
|
||||
</div>
|
||||
</fieldset>
|
||||
|
||||
{#if source === 'postgres'}
|
||||
{#if source === 'sqllab'}
|
||||
<div class="space-y-4 p-4 bg-gray-50 rounded-md border border-gray-100">
|
||||
<div>
|
||||
<Select
|
||||
label={$t.mapper.connection}
|
||||
bind:value={selectedConnection}
|
||||
options={[
|
||||
{ value: '', label: $t.mapper.select_connection },
|
||||
...connections.map(c => ({ value: c.id, label: c.name }))
|
||||
]}
|
||||
<Input
|
||||
label={$t.mapper.database_id}
|
||||
type="number"
|
||||
bind:value={databaseId}
|
||||
placeholder={$t.mapper.database_id_placeholder}
|
||||
/>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<Input
|
||||
label={$t.mapper.table_name}
|
||||
type="text"
|
||||
bind:value={tableName}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Input
|
||||
label={$t.mapper.table_schema}
|
||||
type="text"
|
||||
bind:value={tableSchema}
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<Input
|
||||
label={$t.mapper.sql_query_label}
|
||||
type="text"
|
||||
bind:value={sqlQuery}
|
||||
placeholder={$t.mapper.sql_query_placeholder}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{:else}
|
||||
@@ -249,4 +233,4 @@
|
||||
/>
|
||||
</div>
|
||||
<!-- [/SECTION] -->
|
||||
<!-- #endregion MapperTool -->
|
||||
<!-- #endregion MapperTool -->
|
||||
@@ -35,7 +35,6 @@ import enAdmin from './locales/en/admin.json';
|
||||
import enTranslate from './locales/en/translate.json';
|
||||
import enAssistant from './locales/en/assistant.json';
|
||||
import enAuth from './locales/en/auth.json';
|
||||
import enConnections from './locales/en/connections.json';
|
||||
import enMapper from './locales/en/mapper.json';
|
||||
import enMigration from './locales/en/migration.json';
|
||||
import enStorage from './locales/en/storage.json';
|
||||
@@ -57,7 +56,6 @@ import ruAdmin from './locales/ru/admin.json';
|
||||
import ruTranslate from './locales/ru/translate.json';
|
||||
import ruAssistant from './locales/ru/assistant.json';
|
||||
import ruAuth from './locales/ru/auth.json';
|
||||
import ruConnections from './locales/ru/connections.json';
|
||||
import ruMapper from './locales/ru/mapper.json';
|
||||
import ruMigration from './locales/ru/migration.json';
|
||||
import ruStorage from './locales/ru/storage.json';
|
||||
@@ -81,7 +79,6 @@ const en = {
|
||||
translate: enTranslate,
|
||||
assistant: enAssistant,
|
||||
auth: enAuth,
|
||||
connections: enConnections,
|
||||
mapper: enMapper,
|
||||
migration: enMigration,
|
||||
storage: enStorage,
|
||||
@@ -105,7 +102,6 @@ const ru = {
|
||||
translate: ruTranslate,
|
||||
assistant: ruAssistant,
|
||||
auth: ruAuth,
|
||||
connections: ruConnections,
|
||||
mapper: ruMapper,
|
||||
migration: ruMigration,
|
||||
storage: ruStorage,
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"management": "Connection Management",
|
||||
"add_new": "Add New Connection",
|
||||
"name": "Connection Name",
|
||||
"host": "Host",
|
||||
"port": "Port",
|
||||
"db_name": "Database Name",
|
||||
"user": "Username",
|
||||
"pass": "Password",
|
||||
"create": "Create Connection",
|
||||
"saved": "Saved Connections",
|
||||
"no_saved": "No connections saved yet.",
|
||||
"delete": "Delete",
|
||||
"required_fields": "Please fill in all required fields",
|
||||
"created_success": "Connection created successfully",
|
||||
"fetch_failed": "Failed to fetch connections",
|
||||
"delete_confirm": "Are you sure you want to delete this connection?",
|
||||
"deleted_success": "Connection deleted",
|
||||
"name_placeholder": "e.g. Production DWH",
|
||||
"host_placeholder": "10.0.0.1"
|
||||
}
|
||||
@@ -20,10 +20,12 @@
|
||||
"bulk_map_columns": "Bulk Column Mapping",
|
||||
"bulk_docs_generation": "Bulk Documentation Generation",
|
||||
"source_type": "Source Type",
|
||||
"source_postgresql_comments": "PostgreSQL Comments",
|
||||
"source_sqllab": "Superset Database (SQL Lab)",
|
||||
"source_xlsx": "XLSX File",
|
||||
"connection_id": "Connection ID",
|
||||
"connection_id_placeholder": "Enter connection ID...",
|
||||
"source_database": "Database",
|
||||
"select_database": "select a database",
|
||||
"sql_query": "SQL Query",
|
||||
"sql_query_hint": "leave empty for column information",
|
||||
"xlsx_file": "XLSX File",
|
||||
"selected_datasets": "Selected Datasets",
|
||||
"start_mapping": "Start Mapping",
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
"select_env": "-- Select Environment --",
|
||||
"dataset_id": "Dataset ID",
|
||||
"source": "Mapping Source",
|
||||
"source_postgres": "PostgreSQL",
|
||||
"source_sqllab": "Superset SQL Lab",
|
||||
"source_excel": "Excel",
|
||||
"connection": "Saved Connection",
|
||||
"select_connection": "-- Select Connection --",
|
||||
"table_name": "Table Name",
|
||||
"table_schema": "Table Schema",
|
||||
"database_id": "Superset Database ID",
|
||||
"database_id_placeholder": "Enter Superset database ID...",
|
||||
"sql_query_label": "SQL Query (optional)",
|
||||
"sql_query_placeholder": "SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table'",
|
||||
"excel_path": "Excel File Path",
|
||||
"run": "Run Mapper",
|
||||
"starting": "Starting...",
|
||||
@@ -17,7 +17,7 @@
|
||||
"errors": {
|
||||
"fetch_failed": "Failed to fetch data",
|
||||
"required_fields": "Please fill in required fields",
|
||||
"postgres_required": "Connection and Table Name are required for postgres source",
|
||||
"sqllab_required": "Database ID is required for SQL Lab source",
|
||||
"excel_required": "Excel path is required for excel source",
|
||||
"no_active_llm_provider": "No active LLM provider found",
|
||||
"docs_start_failed": "Failed to start documentation generation",
|
||||
@@ -30,4 +30,4 @@
|
||||
},
|
||||
"auto_document": "Auto-Document",
|
||||
"excel_placeholder": "/path/to/mapping.xlsx"
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,6 @@
|
||||
|
||||
"settings": "Settings",
|
||||
"settings_general": "General",
|
||||
"settings_connections": "Connections",
|
||||
|
||||
"admin": "Admin",
|
||||
"admin_users": "User Management",
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
"appearance": "Appearance",
|
||||
"llm": "LLM",
|
||||
"storage": "Storage",
|
||||
"connections": "Connections",
|
||||
"environments": "Environments",
|
||||
"global_title": "Global Settings",
|
||||
"env_title": "Superset Environments",
|
||||
@@ -23,7 +22,6 @@
|
||||
"storage_preview": "Path Preview",
|
||||
"env_description": "Configure Superset environments for dashboards and datasets.",
|
||||
"env_actions": "Actions",
|
||||
"connections_description": "Configure database connections for data mapping.",
|
||||
"llm_description": "Configure LLM providers for dataset documentation.",
|
||||
"llm_prompts_title": "LLM Prompt Templates",
|
||||
"llm_prompts_description": "Edit reusable prompts used for documentation, dashboard validation, and git commit generation.",
|
||||
@@ -97,7 +95,6 @@
|
||||
"belief_state_hint": "Logs agent reasoning and internal state changes for debugging.",
|
||||
"save_logging": "Save Logging Config",
|
||||
"save_global_settings": "Save Global Settings",
|
||||
"no_external_connections": "No external connections configured.",
|
||||
"name": "Name",
|
||||
"username": "Username",
|
||||
"password": "Password",
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"management": "Управление подключениями",
|
||||
"add_new": "Добавить новое подключение",
|
||||
"name": "Название подключения",
|
||||
"host": "Хост",
|
||||
"port": "Порт",
|
||||
"db_name": "Название БД",
|
||||
"user": "Имя пользователя",
|
||||
"pass": "Пароль",
|
||||
"create": "Создать подключение",
|
||||
"saved": "Сохраненные подключения",
|
||||
"no_saved": "Нет сохраненных подключений.",
|
||||
"delete": "Удалить",
|
||||
"required_fields": "Заполните все обязательные поля",
|
||||
"created_success": "Подключение успешно создано",
|
||||
"fetch_failed": "Не удалось загрузить подключения",
|
||||
"delete_confirm": "Вы уверены, что хотите удалить это подключение?",
|
||||
"deleted_success": "Подключение удалено",
|
||||
"name_placeholder": "например, Production DWH",
|
||||
"host_placeholder": "10.0.0.1"
|
||||
}
|
||||
@@ -20,10 +20,12 @@
|
||||
"bulk_map_columns": "Массовый маппинг колонок",
|
||||
"bulk_docs_generation": "Массовая генерация документации",
|
||||
"source_type": "Тип источника",
|
||||
"source_postgresql_comments": "Комментарии PostgreSQL",
|
||||
"source_sqllab": "База данных Superset (SQL Lab)",
|
||||
"source_xlsx": "XLSX-файл",
|
||||
"connection_id": "ID подключения",
|
||||
"connection_id_placeholder": "Введите ID подключения...",
|
||||
"source_database": "База данных",
|
||||
"select_database": "выберите базу данных",
|
||||
"sql_query": "SQL-запрос",
|
||||
"sql_query_hint": "оставьте пустым для информации о колонках",
|
||||
"xlsx_file": "XLSX-файл",
|
||||
"selected_datasets": "Выбранные датасеты",
|
||||
"start_mapping": "Запустить маппинг",
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
"select_env": "-- Выберите окружение --",
|
||||
"dataset_id": "ID датасета",
|
||||
"source": "Источник маппинга",
|
||||
"source_postgres": "PostgreSQL",
|
||||
"source_sqllab": "Superset SQL Lab",
|
||||
"source_excel": "Excel",
|
||||
"connection": "Сохраненное подключение",
|
||||
"select_connection": "-- Выберите подключение --",
|
||||
"table_name": "Имя таблицы",
|
||||
"table_schema": "Схема таблицы",
|
||||
"database_id": "ID базы данных Superset",
|
||||
"database_id_placeholder": "Введите ID базы данных Superset...",
|
||||
"sql_query_label": "SQL-запрос (опционально)",
|
||||
"sql_query_placeholder": "SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table'",
|
||||
"excel_path": "Путь к файлу Excel",
|
||||
"run": "Запустить маппер",
|
||||
"starting": "Запуск...",
|
||||
@@ -17,7 +17,7 @@
|
||||
"errors": {
|
||||
"fetch_failed": "Не удалось загрузить данные",
|
||||
"required_fields": "Пожалуйста, заполните обязательные поля",
|
||||
"postgres_required": "Подключение и имя таблицы обязательны для источника PostgreSQL",
|
||||
"sqllab_required": "ID базы данных обязателен для SQL Lab",
|
||||
"excel_required": "Путь к Excel обязателен для источника Excel",
|
||||
"no_active_llm_provider": "Не найден активный LLM-провайдер",
|
||||
"docs_start_failed": "Не удалось запустить генерацию документации",
|
||||
@@ -30,4 +30,4 @@
|
||||
},
|
||||
"auto_document": "Авто-документирование",
|
||||
"excel_placeholder": "/path/to/mapping.xlsx"
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,6 @@
|
||||
|
||||
"settings": "Настройки",
|
||||
"settings_general": "Общие",
|
||||
"settings_connections": "Подключения",
|
||||
|
||||
"admin": "Админ",
|
||||
"admin_users": "Пользователи",
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
"language": "Язык",
|
||||
"appearance": "Внешний вид",
|
||||
"llm": "LLM",
|
||||
"connections": "Подключения",
|
||||
"environments": "Окружения",
|
||||
"global_title": "Общие настройки",
|
||||
"env_title": "Окружения Superset",
|
||||
@@ -22,7 +21,6 @@
|
||||
"storage_preview": "Предпросмотр пути",
|
||||
"env_description": "Настройка окружений Superset для дашбордов и датасетов.",
|
||||
"env_actions": "Действия",
|
||||
"connections_description": "Настройка подключений к базам данных для маппинга.",
|
||||
"llm_description": "Настройка LLM провайдеров для документирования датасетов.",
|
||||
"llm_prompts_title": "Шаблоны промптов LLM",
|
||||
"llm_prompts_description": "Редактируйте промпты для документации, проверки дашбордов и генерации git-коммитов.",
|
||||
@@ -97,7 +95,6 @@
|
||||
"belief_state_hint": "Логирует рассуждения агента и изменения внутреннего состояния для отладки.",
|
||||
"save_logging": "Сохранить настройки логирования",
|
||||
"save_global_settings": "Сохранить общие настройки",
|
||||
"no_external_connections": "Внешние подключения не настроены.",
|
||||
"name": "Название",
|
||||
"username": "Имя пользователя",
|
||||
"password": "Пароль",
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
let selectedIds = $state(new SvelteSet());
|
||||
let selectedDatasetId = $state(null), detailData = $state(null), detailLoading = $state(false), detailError = $state(null);
|
||||
let showMapColumnsModal = $state(false), showGenerateDocsModal = $state(false);
|
||||
let mapSourceType = $state("postgresql"), mapConnectionId = $state(""), mapFileData = $state(null);
|
||||
let mapSourceType = $state("sqllab"), mapDatabases = $state([]), mapDatabaseId = $state(null), mapSqlQuery = $state(""), mapFileData = $state(null);
|
||||
let llmProvider = $state(""), llmOptions = $state({});
|
||||
let isMobile = $state(false), mobileSlideOpen = $state(false);
|
||||
let ws = null;
|
||||
@@ -118,9 +118,14 @@
|
||||
if (action === "map_columns") {
|
||||
selectedIds = new SvelteSet([dataset.id]);
|
||||
showMapColumnsModal = true;
|
||||
mapSourceType = "postgresql";
|
||||
mapConnectionId = null;
|
||||
mapSourceType = "sqllab";
|
||||
mapDatabaseId = null;
|
||||
mapSqlQuery = "";
|
||||
mapFileData = null;
|
||||
// Fetch databases on open
|
||||
if (selectedEnv) {
|
||||
api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {});
|
||||
}
|
||||
} else if (action === "generate_docs") {
|
||||
selectedIds = new SvelteSet([dataset.id]);
|
||||
showGenerateDocsModal = true;
|
||||
@@ -147,14 +152,14 @@
|
||||
// Bulk actions
|
||||
async function handleBulkMapColumns() {
|
||||
if (selectedIds.size === 0) return;
|
||||
if (mapSourceType === "postgresql" && !mapConnectionId) return;
|
||||
if (mapSourceType === "sqllab" && !mapDatabaseId) return;
|
||||
if (mapSourceType === "xlsx" && (!mapFileData || mapFileData.length === 0)) return;
|
||||
try {
|
||||
let fileData = null;
|
||||
if (mapSourceType === "xlsx" && mapFileData?.length) fileData = mapFileData[0].name;
|
||||
const resp = await api.postApi("/datasets/map-columns", {
|
||||
env_id: selectedEnv, dataset_ids: [...selectedIds],
|
||||
source_type: mapSourceType, connection_id: mapConnectionId || undefined, file_data: fileData || undefined,
|
||||
source_type: mapSourceType, database_id: mapDatabaseId || undefined, sql_query: mapSqlQuery || undefined, file_data: fileData || undefined,
|
||||
});
|
||||
showMapColumnsModal = false;
|
||||
selectedIds = new SvelteSet();
|
||||
@@ -326,7 +331,7 @@
|
||||
<div class="max-w-screen-2xl mx-auto flex items-center justify-between">
|
||||
<span class="font-medium text-sm">✓ {selectedIds.size} {$t.datasets?.selected}</span>
|
||||
<div class="flex gap-2">
|
||||
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showMapColumnsModal = true; }}>
|
||||
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showMapColumnsModal = true; mapSourceType = "sqllab"; mapDatabaseId = null; mapSqlQuery = ""; mapFileData = null; if (selectedEnv) api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {}); }}>
|
||||
🗺 {$t.datasets?.action_map_columns}
|
||||
</button>
|
||||
<button class="px-3 py-1.5 text-sm bg-primary text-white rounded hover:bg-primary-hover" onclick={() => { showGenerateDocsModal = true; }}>
|
||||
@@ -343,15 +348,31 @@
|
||||
<!-- Map Columns Modal -->
|
||||
{#if showMapColumnsModal}
|
||||
<div class="fixed inset-0 bg-black/50 z-50 flex items-center justify-center p-4" role="dialog" aria-modal="true" onclick={(e) => { if (e.target === e.currentTarget) showMapColumnsModal = false; }}>
|
||||
<div class="bg-white rounded-lg shadow-xl p-6 max-w-md w-full" onclick={(e) => e.stopPropagation()}>
|
||||
<div class="bg-white rounded-lg shadow-xl p-6 max-w-md w-full" style="max-height: 90vh; overflow-y: auto;" onclick={(e) => e.stopPropagation()}>
|
||||
<h3 class="text-lg font-semibold mb-4">{$t.datasets?.bulk_map_columns}</h3>
|
||||
<p class="text-sm text-gray-600 mb-3">{$t.datasets?.selected_count?.replace('{count}', String(selectedIds.size))}</p>
|
||||
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapSourceType}>
|
||||
<option value="postgresql">{$t.datasets?.source_postgresql_comments}</option>
|
||||
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapSourceType} onchange={() => { if (mapSourceType === 'sqllab' && mapDatabases.length === 0 && selectedEnv) { api.getEnvironmentDatabases(selectedEnv).then(dbs => mapDatabases = dbs ?? []).catch(() => {}); } }}>
|
||||
<option value="sqllab">{$t.datasets?.source_sqllab ?? 'Superset SQL Lab'}</option>
|
||||
<option value="xlsx">{$t.datasets?.source_xlsx}</option>
|
||||
</select>
|
||||
{#if mapSourceType === "postgresql"}
|
||||
<input class="w-full border rounded px-2 py-1.5 mb-4" placeholder={$t.datasets?.connection_id_placeholder} bind:value={mapConnectionId} />
|
||||
{#if mapSourceType === "sqllab"}
|
||||
<label class="text-sm font-medium mb-1 block">{$t.datasets?.source_database ?? 'База данных'}</label>
|
||||
<select class="w-full border rounded px-2 py-1.5 mb-3" bind:value={mapDatabaseId}>
|
||||
<option value="">-- {$t.datasets?.select_database ?? 'выберите базу данных'} --</option>
|
||||
{#each mapDatabases as db}
|
||||
<option value={db.id}>{db.database_name || db.name}</option>
|
||||
{/each}
|
||||
</select>
|
||||
<label class="text-sm font-medium mb-1 block">
|
||||
{$t.datasets?.sql_query ?? 'SQL-запрос'}
|
||||
<span class="text-xs text-gray-400 font-normal ml-1">({$t.datasets?.sql_query_hint ?? 'оставьте пустым для information_schema'})</span>
|
||||
</label>
|
||||
<textarea
|
||||
class="w-full border rounded px-2 py-1.5 mb-4 font-mono text-xs"
|
||||
rows="4"
|
||||
placeholder="SELECT column_name, description AS verbose_name FROM my_mappings WHERE table_name = 'my_table';"
|
||||
bind:value={mapSqlQuery}
|
||||
></textarea>
|
||||
{:else}
|
||||
<input type="file" accept=".xlsx" class="w-full mb-4" onchange={(e) => mapFileData = e.target.files} />
|
||||
{/if}
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
<!-- @REPLACED_BY N/A -->
|
||||
<!-- @RELATION DEPENDS_ON -> [EnvironmentsTab] -->
|
||||
<!-- @RELATION DEPENDS_ON -> [LoggingSettings] -->
|
||||
<!-- @RELATION DEPENDS_ON -> [DatabaseConnectionsTab] -->
|
||||
<!-- @RELATION DEPENDS_ON -> [LlmSettings] -->
|
||||
<!-- @RELATION DEPENDS_ON -> [MigrationSettings] -->
|
||||
<!-- @RELATION DEPENDS_ON -> [StorageSettings] -->
|
||||
@@ -36,7 +35,6 @@
|
||||
|
||||
import EnvironmentsTab from "./EnvironmentsTab.svelte";
|
||||
import LoggingSettings from "./LoggingSettings.svelte";
|
||||
import DatabaseConnectionsTab from "./DatabaseConnectionsTab.svelte";
|
||||
import LlmSettings from "./LlmSettings.svelte";
|
||||
import MigrationSettings from "./MigrationSettings.svelte";
|
||||
import StorageSettings from "./StorageSettings.svelte";
|
||||
@@ -171,19 +169,6 @@
|
||||
>
|
||||
{$t.settings?.logging}
|
||||
</button>
|
||||
<button
|
||||
class="px-4 py-2 text-sm font-medium transition-colors focus:outline-none border-b-2"
|
||||
class:text-primary={activeTab === "connections"}
|
||||
class:border-primary={activeTab === "connections"}
|
||||
class:text-gray-600={activeTab !== "connections"}
|
||||
class:border-transparent={activeTab !== "connections"}
|
||||
class:hover:text-gray-800={activeTab !== "connections"}
|
||||
class:hover:border-gray-300={activeTab !== "connections"}
|
||||
aria-current={activeTab === "connections" ? "page" : undefined}
|
||||
onclick={() => handleTabChange("connections")}
|
||||
>
|
||||
{$t.settings?.connections}
|
||||
</button>
|
||||
<button
|
||||
class="px-4 py-2 text-sm font-medium transition-colors focus:outline-none border-b-2"
|
||||
class:text-primary={activeTab === "git"}
|
||||
@@ -270,8 +255,6 @@
|
||||
<EnvironmentsTab bind:settings onSave={loadSettings} />
|
||||
{:else if activeTab === "logging"}
|
||||
<LoggingSettings bind:settings onSave={handleSave} />
|
||||
{:else if activeTab === "connections"}
|
||||
<DatabaseConnectionsTab />
|
||||
{:else if activeTab === "git"}
|
||||
<GitSettingsPage />
|
||||
{:else if activeTab === "llm"}
|
||||
|
||||
@@ -1,342 +0,0 @@
|
||||
<!-- #region DatabaseConnectionsTab [C:3] [TYPE Component] [SEMANTICS settings, connections, database, crud] -->
|
||||
<!-- @BRIEF Database connections management tab: list, create, delete external DB connections. -->
|
||||
<!-- @LAYER UI -->
|
||||
<!-- @RELATION CALLS -> [api_module] -->
|
||||
<!-- @PRE: API client is initialized. -->
|
||||
<!-- @POST: Connections list is displayed; user can create and delete connections. -->
|
||||
<!-- @UX_STATE Loading -> Loading spinner. -->
|
||||
<!-- @UX_STATE Loaded -> Table of connections with actions. -->
|
||||
<!-- @UX_STATE FormVisible -> Connection creation form displayed. -->
|
||||
<!-- @UX_FEEDBACK Toast on create/delete success/failure. -->
|
||||
<script>
|
||||
import { onMount } from "svelte";
|
||||
import { t } from "$lib/i18n";
|
||||
import { api } from "$lib/api.js";
|
||||
import { addToast } from "$lib/toasts";
|
||||
import { createEmptyConnectionDraft } from "./settings-utils.js";
|
||||
|
||||
let connections = $state([]);
|
||||
let hasLoadedConnections = $state(false);
|
||||
let isLoadingConnections = $state(false);
|
||||
let isSubmittingConnection = $state(false);
|
||||
let isConnectionFormVisible = $state(false);
|
||||
let newConnection = $state(createEmptyConnectionDraft());
|
||||
|
||||
onMount(() => {
|
||||
void loadConnections();
|
||||
});
|
||||
|
||||
async function loadConnections(force = false) {
|
||||
if (isLoadingConnections || (hasLoadedConnections && !force)) {
|
||||
return;
|
||||
}
|
||||
isLoadingConnections = true;
|
||||
try {
|
||||
const response = await api.requestApi("/settings/connections");
|
||||
connections = Array.isArray(response) ? response : [];
|
||||
hasLoadedConnections = true;
|
||||
} catch (err) {
|
||||
console.error("[DatabaseConnectionsTab] Failed to load:", err);
|
||||
addToast(err.message || $t.connections?.fetch_failed, "error");
|
||||
connections = [];
|
||||
} finally {
|
||||
isLoadingConnections = false;
|
||||
}
|
||||
}
|
||||
|
||||
function resetConnectionForm() {
|
||||
newConnection = createEmptyConnectionDraft();
|
||||
}
|
||||
|
||||
function openConnectionForm() {
|
||||
isConnectionFormVisible = true;
|
||||
}
|
||||
|
||||
function closeConnectionForm() {
|
||||
isConnectionFormVisible = false;
|
||||
resetConnectionForm();
|
||||
}
|
||||
|
||||
async function handleCreateConnection() {
|
||||
if (
|
||||
!newConnection.name ||
|
||||
!newConnection.host ||
|
||||
!newConnection.database ||
|
||||
!newConnection.username ||
|
||||
!newConnection.password
|
||||
) {
|
||||
addToast($t.connections?.required_fields, "warning");
|
||||
return;
|
||||
}
|
||||
|
||||
isSubmittingConnection = true;
|
||||
try {
|
||||
await api.requestApi("/settings/connections", "POST", {
|
||||
...newConnection,
|
||||
port: newConnection.port ? Number(newConnection.port) : null,
|
||||
});
|
||||
addToast($t.connections?.created_success, "success");
|
||||
closeConnectionForm();
|
||||
await loadConnections(true);
|
||||
} catch (err) {
|
||||
console.error("[DatabaseConnectionsTab] Failed to create:", err);
|
||||
addToast(err.message || $t.settings?.save_failed, "error");
|
||||
} finally {
|
||||
isSubmittingConnection = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDeleteConnection(connectionId) {
|
||||
if (!confirm($t.connections?.delete_confirm)) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await api.requestApi(`/settings/connections/${connectionId}`, "DELETE");
|
||||
addToast($t.connections?.deleted_success, "success");
|
||||
await loadConnections(true);
|
||||
} catch (err) {
|
||||
console.error("[DatabaseConnectionsTab] Failed to delete:", err);
|
||||
addToast(err.message || $t.settings?.save_failed, "error");
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="text-lg font-medium mb-4">
|
||||
<div class="mb-6 flex items-start justify-between gap-4">
|
||||
<div>
|
||||
<h2 class="text-xl font-bold mb-4">
|
||||
{$t.settings?.connections}
|
||||
</h2>
|
||||
<p class="text-gray-600">
|
||||
{$t.settings?.connections_description}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{#if connections.length > 0 || isConnectionFormVisible}
|
||||
<button
|
||||
class="rounded-lg bg-primary px-4 py-2 text-sm font-medium text-white hover:bg-primary-hover"
|
||||
onclick={openConnectionForm}
|
||||
>
|
||||
{$t.connections?.add_new}
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if isConnectionFormVisible}
|
||||
<div class="mb-6 rounded-lg border border-gray-200 bg-gray-50 p-6">
|
||||
<form
|
||||
class="space-y-4"
|
||||
onsubmit={(event) => {
|
||||
event.preventDefault();
|
||||
handleCreateConnection();
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<label
|
||||
for="connection_name"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.name}</label
|
||||
>
|
||||
<input
|
||||
id="connection_name"
|
||||
type="text"
|
||||
bind:value={newConnection.name}
|
||||
placeholder={$t.connections?.name_placeholder}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 gap-4 md:grid-cols-3">
|
||||
<div>
|
||||
<label
|
||||
for="connection_type"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>Type</label
|
||||
>
|
||||
<select
|
||||
id="connection_type"
|
||||
bind:value={newConnection.type}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
>
|
||||
<option value="postgres">PostgreSQL</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="md:col-span-2">
|
||||
<label
|
||||
for="connection_host"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.host}</label
|
||||
>
|
||||
<input
|
||||
id="connection_host"
|
||||
type="text"
|
||||
bind:value={newConnection.host}
|
||||
placeholder={$t.connections?.host_placeholder}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 gap-4 md:grid-cols-2">
|
||||
<div>
|
||||
<label
|
||||
for="connection_port"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.port}</label
|
||||
>
|
||||
<input
|
||||
id="connection_port"
|
||||
type="number"
|
||||
bind:value={newConnection.port}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label
|
||||
for="connection_database"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.db_name}</label
|
||||
>
|
||||
<input
|
||||
id="connection_database"
|
||||
type="text"
|
||||
bind:value={newConnection.database}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 gap-4 md:grid-cols-2">
|
||||
<div>
|
||||
<label
|
||||
for="connection_user"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.user}</label
|
||||
>
|
||||
<input
|
||||
id="connection_user"
|
||||
type="text"
|
||||
bind:value={newConnection.username}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label
|
||||
for="connection_password"
|
||||
class="block text-sm font-medium text-gray-700"
|
||||
>{$t.connections?.pass}</label
|
||||
>
|
||||
<input
|
||||
id="connection_password"
|
||||
type="password"
|
||||
bind:value={newConnection.password}
|
||||
class="mt-1 block w-full rounded-md border border-gray-300 p-2 shadow-sm"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end gap-2">
|
||||
<button
|
||||
type="button"
|
||||
class="rounded-lg bg-gray-200 px-4 py-2 text-sm font-medium text-gray-700 hover:bg-gray-300"
|
||||
onclick={closeConnectionForm}
|
||||
>
|
||||
{$t.common?.cancel}
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
class="rounded-lg bg-primary px-4 py-2 text-sm font-medium text-white hover:bg-primary-hover disabled:cursor-not-allowed disabled:opacity-60"
|
||||
disabled={isSubmittingConnection}
|
||||
>
|
||||
{isSubmittingConnection
|
||||
? ($t.settings?.saving || $t.common?.loading || "Saving...")
|
||||
: $t.connections?.create}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if isLoadingConnections}
|
||||
<div
|
||||
class="rounded-lg border border-gray-200 bg-gray-50 p-8 text-center text-gray-500"
|
||||
>
|
||||
{$t.common?.loading || "Loading..."}
|
||||
</div>
|
||||
{:else if connections.length > 0}
|
||||
<div class="overflow-x-auto rounded-lg border border-gray-200">
|
||||
<table class="min-w-full divide-y divide-gray-200">
|
||||
<thead class="bg-gray-50">
|
||||
<tr>
|
||||
<th
|
||||
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
|
||||
>{$t.connections?.name}</th
|
||||
>
|
||||
<th
|
||||
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
|
||||
>{$t.connections?.host}</th
|
||||
>
|
||||
<th
|
||||
class="px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500"
|
||||
>{$t.connections?.user}</th
|
||||
>
|
||||
<th
|
||||
class="px-6 py-3 text-right text-xs font-medium uppercase tracking-wider text-gray-500"
|
||||
>{$t.settings?.env_actions || "Actions"}</th
|
||||
>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-gray-200 bg-white">
|
||||
{#each connections as connection}
|
||||
<tr>
|
||||
<td class="px-6 py-4 whitespace-nowrap">
|
||||
{connection.name}
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm text-gray-600">
|
||||
<span class="font-mono"
|
||||
>{connection.type}://{connection.host}:{connection.port}/{connection.database}</span
|
||||
>
|
||||
</td>
|
||||
<td class="px-6 py-4 whitespace-nowrap">
|
||||
{connection.username}
|
||||
</td>
|
||||
<td class="px-6 py-4 text-right text-sm font-medium">
|
||||
<button
|
||||
class="text-red-600 hover:text-red-900"
|
||||
onclick={() => handleDeleteConnection(connection.id)}
|
||||
>
|
||||
{$t.connections?.delete}
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{:else if !isConnectionFormVisible}
|
||||
<div
|
||||
class="rounded-lg border border-dashed border-gray-300 bg-gray-50 py-8 text-center"
|
||||
>
|
||||
<p class="text-gray-500">
|
||||
{$t.settings?.no_external_connections}
|
||||
</p>
|
||||
<button
|
||||
class="mt-4 rounded border border-primary px-4 py-2 text-primary hover:bg-primary-light"
|
||||
onclick={() => {
|
||||
openConnectionForm();
|
||||
void loadConnections();
|
||||
}}
|
||||
>
|
||||
{$t.connections?.add_new}
|
||||
</button>
|
||||
</div>
|
||||
{:else}
|
||||
<div
|
||||
class="rounded-lg border border-dashed border-gray-300 bg-white p-6 text-center text-sm text-gray-500"
|
||||
>
|
||||
{$t.connections?.no_saved}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<!-- #endregion DatabaseConnectionsTab -->
|
||||
@@ -33,11 +33,8 @@ vi.mock('$lib/i18n', () => ({
|
||||
saving: 'Saving...',
|
||||
environments: 'Environments',
|
||||
logging: 'Logging',
|
||||
connections: 'Connections',
|
||||
connections_description: 'Configure database connections for data mapping.',
|
||||
llm: 'LLM',
|
||||
storage: 'Storage',
|
||||
no_external_connections: 'No external connections configured.',
|
||||
save_success: 'Settings saved',
|
||||
save_failed: 'Failed',
|
||||
env_description: 'Configure environments',
|
||||
@@ -47,22 +44,7 @@ vi.mock('$lib/i18n', () => ({
|
||||
common: { refresh: 'Refresh', save: 'Save', cancel: 'Cancel', loading: 'Loading...' },
|
||||
tasks: { cron_label: 'Cron Expression', cron_hint: 'Standard cron format' },
|
||||
nav: { settings_git: 'Git' },
|
||||
connections: {
|
||||
add_new: 'Add New Connection',
|
||||
create: 'Create Connection',
|
||||
name: 'Connection Name',
|
||||
host: 'Host',
|
||||
port: 'Port',
|
||||
db_name: 'Database Name',
|
||||
user: 'Username',
|
||||
pass: 'Password',
|
||||
created_success: 'Connection created successfully',
|
||||
fetch_failed: 'Failed to fetch connections',
|
||||
delete_confirm: 'Are you sure you want to delete this connection?',
|
||||
deleted_success: 'Connection deleted',
|
||||
required_fields: 'Please fill in all required fields'
|
||||
}
|
||||
});
|
||||
common: { refresh: 'Refresh', save: 'Save', cancel: 'Cancel', loading: 'Loading...' },
|
||||
return () => { };
|
||||
}
|
||||
},
|
||||
@@ -98,14 +80,6 @@ describe('SettingsPage.integration.test.js', () => {
|
||||
api.requestApi.mockImplementation((url, method = 'GET', body) => {
|
||||
if (url === '/migration/settings') return Promise.resolve(mockMigrationSettings);
|
||||
if (url.includes('/migration/mappings-data')) return Promise.resolve({ items: [], total: 0 });
|
||||
if (url === '/settings/connections' && method === 'POST') {
|
||||
return Promise.resolve({
|
||||
id: 'conn-1',
|
||||
created_at: '2026-03-15T00:00:00Z',
|
||||
...body
|
||||
});
|
||||
}
|
||||
if (url === '/settings/connections') return Promise.resolve([]);
|
||||
return Promise.resolve({});
|
||||
});
|
||||
});
|
||||
@@ -160,57 +134,4 @@ describe('SettingsPage.integration.test.js', () => {
|
||||
expect(addToast).toHaveBeenCalledWith('Synced 1 environment(s)', 'success');
|
||||
});
|
||||
});
|
||||
|
||||
it('opens the connection form and creates a connection from the consolidated settings tab', async () => {
|
||||
render(SettingsPage);
|
||||
await waitFor(() => expect(api.getConsolidatedSettings).toHaveBeenCalled());
|
||||
|
||||
// Click connections tab
|
||||
await fireEvent.click(screen.getAllByText('Connections')[0]);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(api.requestApi).toHaveBeenCalledWith('/settings/connections');
|
||||
});
|
||||
|
||||
await fireEvent.click(screen.getByText('Add New Connection'));
|
||||
|
||||
expect(screen.getByLabelText('Connection Name')).toBeTruthy();
|
||||
|
||||
// Verify form fields and submit button exist
|
||||
const nameInput = screen.getByLabelText('Connection Name');
|
||||
const submitBtn = screen.getByText('Create Connection');
|
||||
expect(nameInput).toBeTruthy();
|
||||
expect(submitBtn).toBeTruthy();
|
||||
|
||||
// Set input values using native value setter + input event for Svelte 5
|
||||
nameInput.value = 'Analytics Warehouse';
|
||||
await fireEvent.input(nameInput);
|
||||
screen.getByLabelText('Host').value = 'warehouse.internal';
|
||||
await fireEvent.input(screen.getByLabelText('Host'));
|
||||
screen.getByLabelText('Database Name').value = 'analytics';
|
||||
await fireEvent.input(screen.getByLabelText('Database Name'));
|
||||
screen.getByLabelText('Username').value = 'reporter';
|
||||
await fireEvent.input(screen.getByLabelText('Username'));
|
||||
screen.getByLabelText('Password').value = 'secret';
|
||||
await fireEvent.input(screen.getByLabelText('Password'));
|
||||
|
||||
// Click Create Connection button
|
||||
await fireEvent.click(submitBtn);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(api.requestApi).toHaveBeenCalledWith('/settings/connections', 'POST', {
|
||||
name: 'Analytics Warehouse',
|
||||
type: 'postgres',
|
||||
host: 'warehouse.internal',
|
||||
port: 5432,
|
||||
database: 'analytics',
|
||||
username: 'reporter',
|
||||
password: 'secret'
|
||||
});
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(addToast).toHaveBeenCalledWith('Connection created successfully', 'success');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -44,7 +44,6 @@ vi.mock('$lib/i18n', () => ({
|
||||
log_level: 'Log Level',
|
||||
task_log_level: 'Task Log Level',
|
||||
enable_belief_state: 'Enable Belief State',
|
||||
connections: 'Connections',
|
||||
llm: 'LLM',
|
||||
storage: 'Storage',
|
||||
save_success: 'Settings saved',
|
||||
@@ -57,7 +56,6 @@ vi.mock('$lib/i18n', () => ({
|
||||
common: { refresh: 'Refresh', retry: 'Retry', save: 'Save' },
|
||||
tasks: { cron_label: 'Cron Expression', cron_hint: 'Standard cron format' },
|
||||
nav: { settings_git: 'Git' },
|
||||
connections: { name: 'Name', user: 'User' }
|
||||
});
|
||||
return () => { };
|
||||
}
|
||||
@@ -78,7 +76,6 @@ describe('SettingsPage UX Contracts', () => {
|
||||
const mockSettings = {
|
||||
environments: [],
|
||||
logging: { level: 'INFO', task_log_level: 'INFO', enable_belief_state: false },
|
||||
connections: [],
|
||||
llm: {}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
<!-- #region ConnectionsPage [C:3] [TYPE Page] [SEMANTICS sveltekit, connection, database, management, config] -->
|
||||
<!-- @BRIEF Page component: routes/settings/connections/+page.svelte -->
|
||||
<!-- @LAYER Page -->
|
||||
<!--
|
||||
@SEMANTICS: settings, connections, database, management
|
||||
@PURPOSE: Page for managing database connection configurations.
|
||||
@LAYER: UI
|
||||
@RELATION: IMPLEMENTS -> [RoutePages]
|
||||
@RELATION: IMPLEMENTS -> [PageContracts]
|
||||
@RELATION: BINDS_TO -> [NavigationContracts]
|
||||
@RELATION: DEPENDS_ON -> ConnectionForm
|
||||
@RELATION: DEPENDS_ON -> ConnectionList
|
||||
@UX_STATE: Ready -> Connection form and saved connection list render side by side.
|
||||
@UX_STATE: Saving -> Successful form submission refreshes the list without leaving the page.
|
||||
@UX_STATE: Error -> Child components surface validation or request errors inline.
|
||||
-->
|
||||
<script>
|
||||
import ConnectionForm from '../../../components/tools/ConnectionForm.svelte';
|
||||
import ConnectionList from '../../../components/tools/ConnectionList.svelte';
|
||||
import { t } from '$lib/i18n';
|
||||
|
||||
let listComponent;
|
||||
|
||||
// #region handleSuccess:Function [TYPE Function]
|
||||
/* @PURPOSE: Refreshes the connection list after a successful creation.
|
||||
@PRE: listComponent must be bound.
|
||||
@POST: Triggers the fetchConnections method on the list component.
|
||||
*/
|
||||
function handleSuccess() {
|
||||
if (listComponent) {
|
||||
listComponent.fetchConnections();
|
||||
}
|
||||
}
|
||||
// #endregion handleSuccess:Function
|
||||
</script>
|
||||
|
||||
<div class="max-w-7xl mx-auto py-6 sm:px-6 lg:px-8">
|
||||
<div class="px-4 py-6 sm:px-0">
|
||||
<h1 class="text-2xl font-semibold text-gray-900 mb-6">{$t.connections?.management }</h1>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
|
||||
<div>
|
||||
<ConnectionForm onsuccess={handleSuccess} />
|
||||
</div>
|
||||
<div>
|
||||
<ConnectionList bind:this={listComponent} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- #endregion ConnectionsPage -->
|
||||
@@ -11,7 +11,6 @@
|
||||
export const SETTINGS_TABS = [
|
||||
"environments",
|
||||
"logging",
|
||||
"connections",
|
||||
"git",
|
||||
"llm",
|
||||
"migration",
|
||||
@@ -158,17 +157,3 @@ export function resolveEnvStage(env) {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Connection draft helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Create a blank connection object suitable for the "new connection" form.
|
||||
*/
|
||||
export function createEmptyConnectionDraft() {
|
||||
return {
|
||||
name: "",
|
||||
host: "",
|
||||
port: "",
|
||||
database: "",
|
||||
username: "",
|
||||
password: "",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
// #region ConnectionService [C:2] [TYPE Service] [SEMANTICS connection, service, api, crud, database]
|
||||
// @BRIEF Service for Connection Management API — list, create, and delete connection configurations.
|
||||
|
||||
/**
|
||||
* Service for interacting with the Connection Management API.
|
||||
*/
|
||||
|
||||
import { requestApi } from '../lib/api';
|
||||
|
||||
const API_BASE = '/settings/connections';
|
||||
|
||||
// #region getConnections:Function [TYPE Function]
|
||||
/* @PURPOSE: Fetch a list of saved connections.
|
||||
@PRE: None.
|
||||
@POST: Returns a promise resolving to an array of connections.
|
||||
*/
|
||||
/**
|
||||
* Fetch a list of saved connections.
|
||||
* @returns {Promise<Array>} List of connections.
|
||||
*/
|
||||
export async function getConnections() {
|
||||
return requestApi(API_BASE);
|
||||
}
|
||||
// #endregion getConnections:Function
|
||||
|
||||
// #region createConnection:Function [TYPE Function]
|
||||
/* @PURPOSE: Create a new connection configuration.
|
||||
@PRE: connectionData must be a valid object.
|
||||
@POST: Returns a promise resolving to the created connection.
|
||||
*/
|
||||
/**
|
||||
* Create a new connection configuration.
|
||||
* @param {Object} connectionData - The connection data.
|
||||
* @returns {Promise<Object>} The created connection instance.
|
||||
*/
|
||||
export async function createConnection(connectionData) {
|
||||
return requestApi(API_BASE, 'POST', connectionData);
|
||||
}
|
||||
// #endregion createConnection:Function
|
||||
|
||||
// #region deleteConnection:Function [TYPE Function]
|
||||
/* @PURPOSE: Delete a connection configuration.
|
||||
@PRE: connectionId must be a valid string.
|
||||
@POST: Returns a promise that resolves when deletion is complete.
|
||||
*/
|
||||
/**
|
||||
* Delete a connection configuration.
|
||||
* @param {string} connectionId - The ID of the connection to delete.
|
||||
*/
|
||||
export async function deleteConnection(connectionId) {
|
||||
return requestApi(`${API_BASE}/${connectionId}`, 'DELETE');
|
||||
}
|
||||
// #endregion deleteConnection:Function
|
||||
// #endregion ConnectionService
|
||||
Reference in New Issue
Block a user