141 lines
5.1 KiB
Python
141 lines
5.1 KiB
Python
from superset_tool.models import SupersetConfig
|
|
from superset_tool.client import SupersetClient
|
|
from superset_tool.utils.logger import SupersetLogger
|
|
from superset_tool.exceptions import AuthenticationError
|
|
from superset_tool.utils.fileio import save_and_unpack_dashboard, update_db_yaml, create_dashboard_export, create_temp_file
|
|
import os
|
|
import keyring
|
|
from pathlib import Path
|
|
import logging
|
|
|
|
log_dir = Path("H:\\dev\\Logs")
|
|
logger = SupersetLogger(
|
|
log_dir=log_dir,
|
|
level=logging.INFO,
|
|
console=True
|
|
)
|
|
|
|
database_config_click={"new":
|
|
{
|
|
"database_name": "Prod Clickhouse",
|
|
"sqlalchemy_uri": "clickhousedb+connect://clicketl:XXXXXXXXXX@rgm-s-khclk.hq.root.ad:443/dm",
|
|
"uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
|
"database_uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
|
"allow_ctas": "false",
|
|
"allow_cvas": "false",
|
|
"allow_dml": "false"
|
|
},
|
|
"old": {
|
|
"database_name": "Dev Clickhouse",
|
|
"sqlalchemy_uri": "clickhousedb+connect://dwhuser:XXXXXXXXXX@10.66.229.179:8123/dm",
|
|
"uuid": "e9fd8feb-cb77-4e82-bc1d-44768b8d2fc2",
|
|
"database_uuid": "e9fd8feb-cb77-4e82-bc1d-44768b8d2fc2",
|
|
"allow_ctas": "true",
|
|
"allow_cvas": "true",
|
|
"allow_dml": "true"
|
|
}
|
|
}
|
|
|
|
database_config_gp={"new":
|
|
{
|
|
"database_name": "Prod Greenplum",
|
|
"sqlalchemy_uri": "postgresql+psycopg2://viz_powerbi_gp_prod:XXXXXXXXXX@10.66.229.201:5432/dwh",
|
|
"uuid": "805132a3-e942-40ce-99c7-bee8f82f8aa8",
|
|
"database_uuid": "805132a3-e942-40ce-99c7-bee8f82f8aa8",
|
|
"allow_ctas": "true",
|
|
"allow_cvas": "true",
|
|
"allow_dml": "true"
|
|
},
|
|
"old": {
|
|
"database_name": "DEV Greenplum",
|
|
"sqlalchemy_uri": "postgresql+psycopg2://viz_superset_gp_dev:XXXXXXXXXX@10.66.229.171:5432/dwh",
|
|
"uuid": "97b97481-43c3-4181-94c5-b69eaaa1e11f",
|
|
"database_uuid": "97b97481-43c3-4181-94c5-b69eaaa1e11f",
|
|
"allow_ctas": "false",
|
|
"allow_cvas": "false",
|
|
"allow_dml": "false"
|
|
}
|
|
}
|
|
|
|
# Конфигурация для Dev
|
|
dev_config = SupersetConfig(
|
|
base_url="https://devta.bi.dwh.rusal.com/api/v1",
|
|
auth={
|
|
"provider": "db",
|
|
"username": "migrate_user",
|
|
"password": keyring.get_password("system", "dev migrate"),
|
|
"refresh": True
|
|
},
|
|
logger=logger,
|
|
verify_ssl=False
|
|
)
|
|
|
|
# Конфигурация для Prod
|
|
prod_config = SupersetConfig(
|
|
base_url="https://prodta.bi.dwh.rusal.com/api/v1",
|
|
auth={
|
|
"provider": "db",
|
|
"username": "migrate_user",
|
|
"password": keyring.get_password("system", "prod migrate"),
|
|
"refresh": True
|
|
},
|
|
logger=logger,
|
|
verify_ssl=False
|
|
)
|
|
|
|
# Конфигурация для Sandbox
|
|
sandbox_config = SupersetConfig(
|
|
base_url="https://sandboxta.bi.dwh.rusal.com/api/v1",
|
|
auth={
|
|
"provider": "db",
|
|
"username": "migrate_user",
|
|
"password": keyring.get_password("system", "sandbox migrate"),
|
|
"refresh": True
|
|
},
|
|
logger=logger,
|
|
verify_ssl=False
|
|
)
|
|
|
|
# Инициализация клиента
|
|
|
|
dev_client = SupersetClient(dev_config)
|
|
sandbox_client = SupersetClient(sandbox_config)
|
|
prod_client = SupersetClient(prod_config)
|
|
|
|
from_c = dev_client
|
|
to_c = sandbox_client
|
|
dashboard_slug = "FI0050"
|
|
dashboard_id = 53
|
|
|
|
dashboard_meta = from_c.get_dashboard(dashboard_slug)
|
|
#print(dashboard_meta)
|
|
#print(dashboard_meta["dashboard_title"])
|
|
|
|
dashboard_id = dashboard_meta["id"]
|
|
|
|
with create_temp_file(suffix='.dir', logger=logger) as temp_root:
|
|
# Экспорт дашборда во временную директорию
|
|
zip_content, filename = from_c.export_dashboard(dashboard_id, logger=logger)
|
|
|
|
# Сохранение и распаковка во временную директорию
|
|
zip_path, unpacked_path = save_and_unpack_dashboard(
|
|
zip_content=zip_content,
|
|
original_filename=filename,
|
|
unpack=True,
|
|
logger=logger,
|
|
output_dir=temp_root
|
|
)
|
|
|
|
# Обновление конфигураций
|
|
source_path = unpacked_path / Path(filename).stem
|
|
update_db_yaml(database_config_click, path=source_path, logger=logger)
|
|
update_db_yaml(database_config_gp, path=source_path, logger=logger)
|
|
|
|
# Создание нового экспорта во временной директории
|
|
temp_zip = temp_root / f"{dashboard_slug}.zip"
|
|
create_dashboard_export(temp_zip, [source_path], logger=logger)
|
|
|
|
# Импорт обновленного дашборда
|
|
to_c.import_dashboard(temp_zip)
|
|
|