busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

032: deep async propagation — orchestrator, insert, mapper, batch chains

Browse Source 
Full async conversion for all sync callers of async SupersetClient methods:

orchestrator_sql.py: generate_and_insert_sql + _resolve_dialect → async
orchestrator_run_completion.py: complete_success → async (calls generate_and_insert_sql)
orchestrator_exec.py: execute_run → async (awaits complete_success)
orchestrator_runner.py: execute_run → async (delegates to engine)
orchestrator.py: execute_run + _generate_and_insert_sql → async
executor.py: _insert_batch_to_target → async (awaits batch insert)
_batch_proc.py: insert_batch_to_target → async
_batch_insert.py: insert_batch_to_target + _resolve_insert_backend + _execute_insert_sql → async
dataset_mapper.py: get_sqllab_mappings + run_mapping → async
  + await get_dataset, update_dataset, execute_and_poll
mapper.py: await on run_mapping + resolve_database_id calls
_run_routes.py: threading.Thread → asyncio.create_task (_background_execute async)
This commit is contained in:
busya
2026-06-05 00:13:01 +03:00
parent f416583a8c
commit ee9123bcf2
11 changed files with 51 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/src/core/utils/dataset_mapper.py
View File

@@ -35,7 +35,7 @@ class DatasetMapper:
# @PARAM database_id (int) - ID базы данных в Superset.
# @PARAM sql_query (Optional[str]) - Произвольный SQL-запрос (должен вернуть column_name + verbose_name).
# @RETURN Dict[str, str] - Словарь column_name -> verbose_name.
def get_sqllab_mappings(
async def get_sqllab_mappings(
self,
client: Any,
dataset_id: int,
@@ -72,7 +72,7 @@ class DatasetMapper:
# Выполняем запрос через SQL Lab
app_logger.info("[get_sqllab_mappings] Executing SQL Lab query on database %d...", database_id)
result = sqllab_executor.execute_and_poll(sql_query, database_id)
result = await sqllab_executor.execute_and_poll(sql_query, database_id)
rows = result.get("results") or result.get("data") or result.get("result", [])
app_logger.info("[get_sqllab_mappings] SQL Lab returned %d rows", len(rows))
@@ -123,7 +123,7 @@ class DatasetMapper:
# @PARAM database_id (Optional[int]) - ID базы данных Superset (для sqllab source).
# @PARAM sql_query (Optional[str]) - Произвольный SQL-запрос (для sqllab source).
# @PARAM excel_path (Optional[str]) - Путь к XLSX файлу.
def run_mapping(
async def run_mapping(
self,
superset_client: Any,
dataset_id: int,
@@ -140,7 +140,7 @@ class DatasetMapper:
try:
if source == "sqllab":
assert sqllab_executor and database_id, "sqllab_executor and database_id are required for sqllab source."
mappings.update(self.get_sqllab_mappings(superset_client, dataset_id, sqllab_executor, database_id, sql_query))
mappings.update(await self.get_sqllab_mappings(superset_client, dataset_id, sqllab_executor, database_id, sql_query))
elif source == "excel":
assert excel_path, "excel_path is required."
mappings.update(self.load_excel_mappings(excel_path))
@@ -148,7 +148,7 @@ class DatasetMapper:
app_logger.error("[run_mapping][Failure] Invalid source: %s.", source)
return
dataset_response = superset_client.get_dataset(dataset_id)
dataset_response = await superset_client.get_dataset(dataset_id)
dataset_data = dataset_response['result']
original_columns = dataset_data.get('columns', [])
@@ -226,7 +226,7 @@ class DatasetMapper:
}
payload_for_update = {k: v for k, v in payload_for_update.items() if v is not None}
superset_client.update_dataset(dataset_id, payload_for_update)
await superset_client.update_dataset(dataset_id, payload_for_update)
app_logger.info("[run_mapping][Success] Dataset %d columns' verbose_name updated.", dataset_id)
else:
app_logger.info("[run_mapping][State] No changes in columns' verbose_name, skipping update.")