busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

032: deep async propagation — orchestrator, insert, mapper, batch chains

Browse Source 
Full async conversion for all sync callers of async SupersetClient methods:

orchestrator_sql.py: generate_and_insert_sql + _resolve_dialect → async
orchestrator_run_completion.py: complete_success → async (calls generate_and_insert_sql)
orchestrator_exec.py: execute_run → async (awaits complete_success)
orchestrator_runner.py: execute_run → async (delegates to engine)
orchestrator.py: execute_run + _generate_and_insert_sql → async
executor.py: _insert_batch_to_target → async (awaits batch insert)
_batch_proc.py: insert_batch_to_target → async
_batch_insert.py: insert_batch_to_target + _resolve_insert_backend + _execute_insert_sql → async
dataset_mapper.py: get_sqllab_mappings + run_mapping → async
  + await get_dataset, update_dataset, execute_and_poll
mapper.py: await on run_mapping + resolve_database_id calls
_run_routes.py: threading.Thread → asyncio.create_task (_background_execute async)
This commit is contained in:
busya
2026-06-05 00:13:01 +03:00
parent f416583a8c
commit ee9123bcf2
11 changed files with 51 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/plugins/mapper.py
View File

@@ -169,7 +169,7 @@ class MapperPlugin(PluginBase):
raise ValueError("database_id is required for sqllab source.")
await executor.resolve_database_id(target_database_id=str(database_id))
mapper.run_mapping(
await mapper.run_mapping(
superset_client=client,
dataset_id=dataset_id,
source="sqllab",
@@ -179,7 +179,7 @@ class MapperPlugin(PluginBase):
)
else:
# Excel source
mapper.run_mapping(
await mapper.run_mapping(
superset_client=client,
dataset_id=dataset_id,
source="excel",

16
backend/src/plugins/translate/_batch_insert.py
View File

@@ -25,7 +25,7 @@ from .superset_executor import SupersetSqlLabExecutor
# #region insert_batch_to_target [C:3] [TYPE Function] [SEMANTICS translate, insert, orchestrate]
# @BRIEF Insert successful records from a single batch into the target table.
def insert_batch_to_target(
async def insert_batch_to_target(
db: Session, config_manager: ConfigManager,
job: TranslationJob, batch_id: str, run_id: str,
) -> None:
@@ -45,7 +45,7 @@ def insert_batch_to_target(
columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
dialect, executor = _resolve_insert_backend(config_manager, job, batch_id)
dialect, executor = await _resolve_insert_backend(config_manager, job, batch_id)
if dialect is None:
return
@@ -64,7 +64,7 @@ def insert_batch_to_target(
for sql, chunk_count in statements:
if sql is None:
continue
_execute_insert_sql(executor, sql, batch_id, chunk_count)
await _execute_insert_sql(executor, sql, batch_id, chunk_count)
total_inserted += chunk_count
logger.reason(f"Batch {batch_id[:12]} inserted {total_inserted} rows",
{"batch_id": batch_id, "rows": total_inserted, "chunks": len(statements)})
@@ -193,14 +193,14 @@ def _build_insert_rows(
# #region _resolve_insert_backend [C:1] [TYPE Function]
def _resolve_insert_backend(
async def _resolve_insert_backend(
config_manager: ConfigManager, job: TranslationJob, batch_id: str,
) -> tuple[str | None, SupersetSqlLabExecutor | None]:
"""Resolve the database backend and SQL executor for batch insert."""
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(config_manager, env_id)
executor.resolve_database_id(target_database_id=job.target_database_id)
await executor.resolve_database_id(target_database_id=job.target_database_id)
real_backend = executor.get_database_backend()
except Exception as e:
logger.explore("Failed to resolve database backend for batch insert",
@@ -233,14 +233,14 @@ def _generate_insert_sql(
# #region _execute_insert_sql [C:1] [TYPE Function]
def _execute_insert_sql(executor: SupersetSqlLabExecutor, sql: str, batch_id: str, row_count: int) -> None:
async def _execute_insert_sql(executor: SupersetSqlLabExecutor, sql: str, batch_id: str, row_count: int) -> None:
"""Execute the INSERT SQL via Superset SQL Lab."""
try:
result = executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
result = await executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
except Exception as e:
logger.explore("Superset SQL submission failed for batch", {"batch_id": batch_id, "error": str(e)})
return
logger.reason(f"Batch {batch_id[:12]} inserted {row_count} rows",
logger.reason(f"Chunk inserted for batch {batch_id[:12]}",
{"batch_id": batch_id, "rows": row_count, "status": result.get("status")})
# #endregion _execute_insert_sql
# #endregion BatchInsertService

4
backend/src/plugins/translate/_batch_proc.py
View File

@@ -253,7 +253,7 @@ class BatchProcessingService:
return result
# -- Batch insert (delegation) --
def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
async def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
await insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
# #endregion BatchProcessingService
# #endregion BatchProcessingService

6
backend/src/plugins/translate/executor.py
View File

@@ -148,7 +148,7 @@ class TranslationExecutor:
batch_id = batch_result.get("batch_id")
if batch_id and batch_result.get("successful", 0) > 0:
try:
self._insert_batch_to_target(job, batch_id, run.id)
await self._insert_batch_to_target(job, batch_id, run.id)
except Exception as e:
logger.explore("Batch INSERT failed (non-fatal)", {"batch_id": batch_id, "error": str(e)})
@@ -247,9 +247,9 @@ class TranslationExecutor:
job, run_id, batch_index, batch_rows, dict_snapshot_hash, config_hash,
preview_edits_cache=self._preview_edits_cache)
def _insert_batch_to_target(self, job, batch_id, run_id) -> None:
async def _insert_batch_to_target(self, job, batch_id, run_id) -> None:
from ._batch_proc import BatchProcessingService
BatchProcessingService(self.db, self.config_manager).insert_batch_to_target(job, batch_id, run_id)
await BatchProcessingService(self.db, self.config_manager).insert_batch_to_target(job, batch_id, run_id)
async def _call_llm_for_batch(self, job, run_id, batch_rows, dict_matches, batch_id, max_tokens=8192, _recursion_depth=0) -> dict:
from ._llm_call import LLMTranslationService

10
backend/src/plugins/translate/orchestrator.py
View File

@@ -84,14 +84,12 @@ class TranslationOrchestrator:
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
async def execute_run(
self,
run: TranslationRun,
on_batch_progress: Callable[[str, int, int, int, int], None] | None = None,
skip_insert: bool = False,
) -> TranslationRun:
with belief_scope("TranslationOrchestrator.execute_run"):
return self._runner.execute_run(
return await self._runner.execute_run(
run=run,
on_batch_progress=on_batch_progress,
skip_insert=skip_insert,
@@ -119,7 +117,7 @@ class TranslationOrchestrator:
# region _generate_and_insert_sql [TYPE Function] [SEMANTICS backward-compat wrapper]
# @PURPOSE: Backward-compatible delegating wrapper for SQL generation and insert.
# @SIDE_EFFECT Delegates to SQLInsertService. May call Superset API.
def _generate_and_insert_sql(
async def _generate_and_insert_sql(
self,
job: Any,
run: TranslationRun,
@@ -127,7 +125,7 @@ class TranslationOrchestrator:
"""Backward-compatible wrapper — delegates to SQLInsertService."""
from .orchestrator_sql import SQLInsertService
svc = SQLInsertService(self.db, self.config_manager, self.event_log)
return svc.generate_and_insert_sql(job, run)
return await svc.generate_and_insert_sql(job, run)
# endregion _generate_and_insert_sql
# region _update_language_stats [TYPE Function] [SEMANTICS backward-compat wrapper]

4
backend/src/plugins/translate/orchestrator_exec.py
View File

@@ -55,7 +55,7 @@ class TranslationExecutionEngine:
# @PRE run is in PENDING status.
# @POST Run executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
async def execute_run(
self,
run: TranslationRun,
on_batch_progress: Callable[[str, int, int, int, int], None] | None = None,
@@ -89,7 +89,7 @@ class TranslationExecutionEngine:
if run.status == "CANCELLED":
return _complete_cancelled(self.db, self.event_log, self._aggregator, run, language_stats_map, self.current_user)
return _complete_success(
return await _complete_success(
self.db, self.event_log, self._aggregator, self._sql_service,
run, job, skip_insert, language_stats_map, self.current_user,
)

4
backend/src/plugins/translate/orchestrator_run_completion.py
View File

@@ -69,7 +69,7 @@ def complete_cancelled(
# region complete_success [TYPE Function]
# @PURPOSE: Finalize a successful run — update stats, optionally insert SQL, commit.
# @SIDE_EFFECT DB writes; event log; Superset API call if skip_insert is False.
def complete_success(
async def complete_success(
db,
event_log,
aggregator,
@@ -102,7 +102,7 @@ def complete_success(
db.commit()
return run
insert_result = sql_service.generate_and_insert_sql(job, run)
insert_result = await sql_service.generate_and_insert_sql(job, run)
run.insert_status = insert_result.get("status")
run.superset_execution_id = str(insert_result.get("query_id") or "")
run.superset_execution_log = insert_result

4
backend/src/plugins/translate/orchestrator_runner.py
View File

@@ -41,14 +41,14 @@ class TranslationStageRunner:
# @PRE run is in PENDING status.
# @POST Run is executed, SQL generated, Superset submission attempted.
# @SIDE_EFFECT LLM calls, DB writes, Superset API calls.
def execute_run(
async def execute_run(
self,
run: TranslationRun,
on_batch_progress: Callable[[str, int, int, int, int], None] | None = None,
skip_insert: bool = False,
) -> TranslationRun:
with belief_scope("TranslationStageRunner.execute_run"):
return self._executor_engine.execute_run(
return await self._executor_engine.execute_run(
run=run,
on_batch_progress=on_batch_progress,
skip_insert=skip_insert,

12
backend/src/plugins/translate/orchestrator_sql.py
View File

@@ -32,7 +32,7 @@ class SQLInsertService:
# region generate_and_insert_sql [TYPE Function]
# @PURPOSE: Generate INSERT SQL and submit to Superset SQL Lab.
def generate_and_insert_sql(
async def generate_and_insert_sql(
self,
job: TranslationJob,
run: TranslationRun,
@@ -64,7 +64,7 @@ class SQLInsertService:
columns = [effective_target or "translated_text"]
rows_for_sql = [{columns[0]: rec.target_sql or ""} for rec in records]
dialect = self._resolve_dialect(job)
dialect = await self._resolve_dialect(job)
try:
sql, row_count = SQLGenerator.generate(
dialect=dialect,
@@ -88,8 +88,8 @@ class SQLInsertService:
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
executor.resolve_database_id(target_database_id=job.target_database_id)
result = executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
await executor.resolve_database_id(target_database_id=job.target_database_id)
result = await executor.execute_and_poll(sql=sql, max_polls=30, poll_interval_seconds=2.0)
except Exception as e:
logger.explore("Superset SQL submission failed", {"run_id": run.id, "error": str(e)})
result = {"status": "failed", "error_message": str(e), "query_id": None}
@@ -103,11 +103,11 @@ class SQLInsertService:
# endregion generate_and_insert_sql
# region _resolve_dialect [TYPE Function]
def _resolve_dialect(self, job: TranslationJob) -> str:
async def _resolve_dialect(self, job: TranslationJob) -> str:
try:
env_id = job.environment_id or job.source_dialect or ""
executor = SupersetSqlLabExecutor(self.config_manager, env_id)
executor.resolve_database_id(target_database_id=job.target_database_id)
await executor.resolve_database_id(target_database_id=job.target_database_id)
real_backend = executor.get_database_backend()
except Exception as e:
logger.explore("Failed to resolve database backend, falling back to job dialect", {"error": str(e)})