feat: implement plugin architecture and application settings with Svelte UI
- Added plugin base and loader for backend extensibility - Implemented application settings management with config persistence - Created Svelte-based frontend with Dashboard and Settings pages - Added API routes for plugins, tasks, and settings - Updated documentation and specifications - Improved project structure and developer tools
This commit is contained in:
144
run_mapper.py
Normal file → Executable file
144
run_mapper.py
Normal file → Executable file
@@ -1,72 +1,72 @@
|
||||
# [DEF:run_mapper:Module]
|
||||
#
|
||||
# @SEMANTICS: runner, configuration, cli, main
|
||||
# @PURPOSE: Этот модуль является CLI-точкой входа для запуска процесса меппинга метаданных датасетов.
|
||||
# @LAYER: App
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils.dataset_mapper
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
# @PUBLIC_API: main
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import argparse
|
||||
import keyring
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from superset_tool.utils.dataset_mapper import DatasetMapper
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:main:Function]
|
||||
# @PURPOSE: Парсит аргументы командной строки и запускает процесс меппинга.
|
||||
# @RELATION: CREATES_INSTANCE_OF -> DatasetMapper
|
||||
# @RELATION: CALLS -> setup_clients
|
||||
# @RELATION: CALLS -> DatasetMapper.run_mapping
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Map dataset verbose names in Superset.")
|
||||
parser.add_argument('--source', type=str, required=True, choices=['postgres', 'excel', 'both'], help='The source for the mapping.')
|
||||
parser.add_argument('--dataset-id', type=int, required=True, help='The ID of the dataset to update.')
|
||||
parser.add_argument('--table-name', type=str, help='The table name for PostgreSQL source.')
|
||||
parser.add_argument('--table-schema', type=str, help='The table schema for PostgreSQL source.')
|
||||
parser.add_argument('--excel-path', type=str, help='The path to the Excel file.')
|
||||
parser.add_argument('--env', type=str, default='dev', help='The Superset environment to use.')
|
||||
|
||||
args = parser.parse_args()
|
||||
logger = SupersetLogger(name="dataset_mapper_main")
|
||||
|
||||
# [AI_NOTE]: Конфигурация БД должна быть вынесена во внешний файл или переменные окружения.
|
||||
POSTGRES_CONFIG = {
|
||||
'dbname': 'dwh',
|
||||
'user': keyring.get_password("system", f"dwh gp user"),
|
||||
'password': keyring.get_password("system", f"dwh gp password"),
|
||||
'host': '10.66.229.201',
|
||||
'port': '5432'
|
||||
}
|
||||
|
||||
logger.info("[main][Enter] Starting dataset mapper CLI.")
|
||||
try:
|
||||
clients = setup_clients(logger)
|
||||
superset_client = clients.get(args.env)
|
||||
|
||||
if not superset_client:
|
||||
logger.error(f"[main][Failure] Superset client for '{args.env}' environment not found.")
|
||||
return
|
||||
|
||||
mapper = DatasetMapper(logger)
|
||||
mapper.run_mapping(
|
||||
superset_client=superset_client,
|
||||
dataset_id=args.dataset_id,
|
||||
source=args.source,
|
||||
postgres_config=POSTGRES_CONFIG if args.source in ['postgres', 'both'] else None,
|
||||
excel_path=args.excel_path if args.source in ['excel', 'both'] else None,
|
||||
table_name=args.table_name if args.source in ['postgres', 'both'] else None,
|
||||
table_schema=args.table_schema if args.source in ['postgres', 'both'] else None
|
||||
)
|
||||
logger.info("[main][Exit] Dataset mapper process finished.")
|
||||
|
||||
except Exception as main_exc:
|
||||
logger.error("[main][Failure] An unexpected error occurred: %s", main_exc, exc_info=True)
|
||||
# [/DEF:main]
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
# [/DEF:run_mapper]
|
||||
# [DEF:run_mapper:Module]
|
||||
#
|
||||
# @SEMANTICS: runner, configuration, cli, main
|
||||
# @PURPOSE: Этот модуль является CLI-точкой входа для запуска процесса меппинга метаданных датасетов.
|
||||
# @LAYER: App
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils.dataset_mapper
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
# @PUBLIC_API: main
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import argparse
|
||||
import keyring
|
||||
from superset_tool.utils.init_clients import setup_clients
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from superset_tool.utils.dataset_mapper import DatasetMapper
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:main:Function]
|
||||
# @PURPOSE: Парсит аргументы командной строки и запускает процесс меппинга.
|
||||
# @RELATION: CREATES_INSTANCE_OF -> DatasetMapper
|
||||
# @RELATION: CALLS -> setup_clients
|
||||
# @RELATION: CALLS -> DatasetMapper.run_mapping
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Map dataset verbose names in Superset.")
|
||||
parser.add_argument('--source', type=str, required=True, choices=['postgres', 'excel', 'both'], help='The source for the mapping.')
|
||||
parser.add_argument('--dataset-id', type=int, required=True, help='The ID of the dataset to update.')
|
||||
parser.add_argument('--table-name', type=str, help='The table name for PostgreSQL source.')
|
||||
parser.add_argument('--table-schema', type=str, help='The table schema for PostgreSQL source.')
|
||||
parser.add_argument('--excel-path', type=str, help='The path to the Excel file.')
|
||||
parser.add_argument('--env', type=str, default='dev', help='The Superset environment to use.')
|
||||
|
||||
args = parser.parse_args()
|
||||
logger = SupersetLogger(name="dataset_mapper_main")
|
||||
|
||||
# [AI_NOTE]: Конфигурация БД должна быть вынесена во внешний файл или переменные окружения.
|
||||
POSTGRES_CONFIG = {
|
||||
'dbname': 'dwh',
|
||||
'user': keyring.get_password("system", f"dwh gp user"),
|
||||
'password': keyring.get_password("system", f"dwh gp password"),
|
||||
'host': '10.66.229.201',
|
||||
'port': '5432'
|
||||
}
|
||||
|
||||
logger.info("[main][Enter] Starting dataset mapper CLI.")
|
||||
try:
|
||||
clients = setup_clients(logger)
|
||||
superset_client = clients.get(args.env)
|
||||
|
||||
if not superset_client:
|
||||
logger.error(f"[main][Failure] Superset client for '{args.env}' environment not found.")
|
||||
return
|
||||
|
||||
mapper = DatasetMapper(logger)
|
||||
mapper.run_mapping(
|
||||
superset_client=superset_client,
|
||||
dataset_id=args.dataset_id,
|
||||
source=args.source,
|
||||
postgres_config=POSTGRES_CONFIG if args.source in ['postgres', 'both'] else None,
|
||||
excel_path=args.excel_path if args.source in ['excel', 'both'] else None,
|
||||
table_name=args.table_name if args.source in ['postgres', 'both'] else None,
|
||||
table_schema=args.table_schema if args.source in ['postgres', 'both'] else None
|
||||
)
|
||||
logger.info("[main][Exit] Dataset mapper process finished.")
|
||||
|
||||
except Exception as main_exc:
|
||||
logger.error("[main][Failure] An unexpected error occurred: %s", main_exc, exc_info=True)
|
||||
# [/DEF:main]
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
# [/DEF:run_mapper]
|
||||
|
||||
Reference in New Issue
Block a user