Работающий импорт с изменением БД
This commit is contained in:
Volobuev Andrey
@@ -1,31 +1,62 @@
|
||||
from superset_tool.models import SupersetConfig
|
||||
from superset_tool.client import SupersetClient
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from superset_tool.exceptions import AuthenticationError
|
||||
from superset_tool.utils.fileio import save_and_unpack_dashboard, update_db_yaml, archive_exports, sync_for_git
|
||||
from superset_tool.utils.fileio import save_and_unpack_dashboard, update_db_yaml, create_dashboard_export
|
||||
import os
|
||||
import keyring
|
||||
from pathlib import Path
|
||||
import logging
|
||||
|
||||
log_dir = Path("H:\\dev\\Logs")
|
||||
logger = SupersetLogger(
|
||||
log_dir=log_dir,
|
||||
level=logging.INFO,
|
||||
console=True
|
||||
)
|
||||
|
||||
database_config={"PROD":
|
||||
database_config_click={"new":
|
||||
{
|
||||
"database_name": "Prod Clickhouse",
|
||||
"sqlalchemy_uri": "clickhousedb+connect://clicketl:XXXXXXXXXX@rgm-s-khclk.hq.root.ad:443/dm",
|
||||
"uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
||||
"database_uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
||||
"allow_ctas": "true",
|
||||
"allow_cvas": "true",
|
||||
"allow_dml": "true"
|
||||
},
|
||||
"DEV": {
|
||||
"old": {
|
||||
"database_name": "Dev Clickhouse",
|
||||
"sqlalchemy_uri": "clickhousedb+connect://dwhuser:XXXXXXXXXX@10.66.229.179:8123/dm",
|
||||
"uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
||||
"database_uuid": "b9b67cb5-9874-4dc6-87bd-354fc33be6f9",
|
||||
"uuid": "e9fd8feb-cb77-4e82-bc1d-44768b8d2fc2",
|
||||
"database_uuid": "e9fd8feb-cb77-4e82-bc1d-44768b8d2fc2",
|
||||
"allow_ctas": "true",
|
||||
"allow_cvas": "true",
|
||||
"allow_dml": "true"
|
||||
}
|
||||
}
|
||||
|
||||
database_config_gp={"new":
|
||||
{
|
||||
"database_name": "Prod Greenplum",
|
||||
"sqlalchemy_uri": "postgresql+psycopg2://viz_powerbi_gp_prod:XXXXXXXXXX@10.66.229.201:5432/dwh",
|
||||
"uuid": "805132a3-e942-40ce-99c7-bee8f82f8aa8",
|
||||
"database_uuid": "805132a3-e942-40ce-99c7-bee8f82f8aa8",
|
||||
"allow_ctas": "true",
|
||||
"allow_cvas": "true",
|
||||
"allow_dml": "true"
|
||||
},
|
||||
"old": {
|
||||
"database_name": "DEV Greenplum",
|
||||
"sqlalchemy_uri": "postgresql+psycopg2://viz_superset_gp_dev:XXXXXXXXXX@10.66.229.171:5432/dwh",
|
||||
"uuid": "97b97481-43c3-4181-94c5-b69eaaa1e11f",
|
||||
"database_uuid": "97b97481-43c3-4181-94c5-b69eaaa1e11f",
|
||||
"allow_ctas": "false",
|
||||
"allow_cvas": "false",
|
||||
"allow_dml": "false"
|
||||
}
|
||||
}
|
||||
|
||||
# Конфигурация для Dev
|
||||
dev_config = SupersetConfig(
|
||||
base_url="https://devta.bi.dwh.rusal.com/api/v1",
|
||||
@@ -35,6 +66,7 @@ dev_config = SupersetConfig(
|
||||
"password": keyring.get_password("system", "dev migrate"),
|
||||
"refresh": True
|
||||
},
|
||||
logger=logger,
|
||||
verify_ssl=False
|
||||
)
|
||||
|
||||
@@ -47,6 +79,7 @@ prod_config = SupersetConfig(
|
||||
"password": keyring.get_password("system", "prod migrate"),
|
||||
"refresh": True
|
||||
},
|
||||
logger=logger,
|
||||
verify_ssl=False
|
||||
)
|
||||
|
||||
@@ -59,41 +92,45 @@ sandbox_config = SupersetConfig(
|
||||
"password": keyring.get_password("system", "sandbox migrate"),
|
||||
"refresh": True
|
||||
},
|
||||
logger=logger,
|
||||
verify_ssl=False
|
||||
)
|
||||
|
||||
# Инициализация клиента
|
||||
|
||||
dev_client = SupersetClient(dev_config)
|
||||
#prod_client = SupersetClient(prod_config)
|
||||
sandbox_client = SupersetClient(sandbox_config)
|
||||
prod_client = SupersetClient(prod_config)
|
||||
|
||||
dashboard_slug = "IM0010"
|
||||
from_c = dev_client
|
||||
to_c = sandbox_client
|
||||
dashboard_slug = "FI0050"
|
||||
#dashboard_id = 53
|
||||
|
||||
dashboard_meta = dev_client.get_dashboard(dashboard_slug)
|
||||
print(dashboard_meta)
|
||||
# print(dashboard_meta["dashboard_title"])
|
||||
dashboard_meta = from_c.get_dashboard(dashboard_slug)
|
||||
#print(dashboard_meta)
|
||||
print(dashboard_meta["dashboard_title"])
|
||||
|
||||
#dashboard_id = dashboard_meta["id"]
|
||||
# zip_content, filename = prod_client.export_dashboard(dashboard_id)
|
||||
# superset_repo = "H:\\Superset\\repo\\"
|
||||
# # print(f"Экспортируем дашборд ID = {dashboard_id}...")
|
||||
# # #Сохранение и распаковка
|
||||
# zip_path, unpacked_path = save_and_unpack_dashboard(
|
||||
# zip_content=zip_content,
|
||||
# original_filename=filename,
|
||||
# output_dir=f"dashboards\{dashboard_slug}"
|
||||
# )
|
||||
# dest_path = os.path.join(superset_repo,dashboard_slug)
|
||||
# source_path = os.path.join(unpacked_path,Path(filename).stem)
|
||||
# print(dest_path)
|
||||
# print(source_path)
|
||||
# sync_for_git(source_path=source_path,destination_path=dest_path,verbose=True)
|
||||
dashboard_id = dashboard_meta["id"]
|
||||
zip_content, filename = from_c.export_dashboard(dashboard_id, logger=logger)
|
||||
superset_repo = Path("H:\\dev\\dashboards\\")
|
||||
# print(f"Экспортируем дашборд ID = {dashboard_id}...")
|
||||
# #Сохранение и распаковка
|
||||
zip_path, unpacked_path = save_and_unpack_dashboard(
|
||||
zip_content=zip_content,
|
||||
original_filename=filename,
|
||||
unpack=True,
|
||||
logger=logger,
|
||||
output_dir=os.path.join(superset_repo,dashboard_slug)
|
||||
)
|
||||
dest_path = os.path.join(superset_repo,dashboard_slug)
|
||||
source_path = os.path.join(unpacked_path,Path(filename).stem)
|
||||
|
||||
# print(f"Сохранено в: {zip_path}")
|
||||
# print(f"Распаковано в: {unpacked_path}")
|
||||
# update_db_yaml(prod_config.database_config, path = unpacked_path,verbose=False)
|
||||
update_db_yaml(database_config_click, path = source_path, logger=logger)
|
||||
update_db_yaml(database_config_gp, path = source_path, logger=logger)
|
||||
|
||||
# prod_client.import_dashboard(zip_path)
|
||||
create_dashboard_export(f"{dashboard_slug}.zip",[source_path],logger=logger)
|
||||
|
||||
zip_path = Path(f"{dashboard_slug}.zip")
|
||||
to_c.import_dashboard(zip_path)
|
||||
|
||||
# archive_exports("dashboards", max_files=3)
|
||||
|
||||
Reference in New Issue
Block a user