busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(translate,scheduler,trace): new-key-only mode, stale PENDING protection, trace_id propagation

Browse Source 
Translations:
- FR-045: new-key-only execution mode — translate only rows with unseen keys
- Compare source row keys against TranslationRecord.source_data from last succeeded run
- Baseline expired (>90 days) fallback to full mode with baseline_expired event
- run_noop early return when zero new keys (skip LLM + SQL)

Scheduler reliability:
- Stale PENDING protection: runs older than 1h auto-marked FAILED, no longer block schedule
- load_schedules() reloads active translation schedules from DB on restart
- add_translation_job/remove_translation_job register/unregister with APScheduler
- execution_mode column on translation_schedules with additive DB migration

Automation view:
- GET /settings/automation/translation-schedules endpoint
- Translation schedules displayed on Automation page below validation policies

Trace propagation:
- seed_trace_id() in all background entry points:
  TaskManager._run_task/_flusher_loop, SchedulerService._trigger_backup,
  websocket_endpoint, IdMappingService, all standalone scripts

Migrations:
- dictionary_entries: origin_run_id, origin_row_key, origin_user_id
- translation_schedules: execution_mode

Tests:
- 3 new test modules (22 tests): core_scheduler, executor_filter, scheduler_execution+guard
- Fix pre-existing translate test isolation (conftest.py)
- Fix test_list_runs_filter_status parameter
This commit is contained in:
busya
2026-05-14 10:13:56 +03:00
parent d1695fe536
commit 8d0bc6fb93
26 changed files with 1525 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
backend/src/plugins/translate/executor.py
View File

@@ -40,6 +40,7 @@ from .preview import DEFAULT_EXECUTION_PROMPT_TEMPLATE
# #region MAX_RETRIES_PER_BATCH [TYPE Constant]
# @BRIEF Maximum number of retries for a single batch before marking it failed.
MAX_RETRIES_PER_BATCH = 3
# #endregion MAX_RETRIES_PER_BATCH
# #region TranslationExecutor [C:4] [TYPE Class]
@@ -97,6 +98,20 @@ class TranslationExecutor:
self.db.flush()
return run
# Apply new-key-only filtering for scheduled runs
if run.trigger_type == "new_key_only":
source_rows = self._filter_new_keys(job, run.id, source_rows)
if not source_rows:
logger.reason("run_noop — no new rows to translate", {
"job_id": job.id, "run_id": run.id,
})
run.status = "COMPLETED"
run.insert_status = "skipped"
run.total_records = 0
run.completed_at = datetime.now(timezone.utc)
self.db.commit()
return
total_rows = len(source_rows)
run.total_records = total_rows
@@ -303,6 +318,74 @@ class TranslationExecutor:
return source_rows
# endregion _fetch_source_rows
# region _filter_new_keys [TYPE Function]
# @PURPOSE: Filter source rows to only include those with keys absent from the last successful run.
# @PRE: job and run are persisted; source_rows is a list of dicts with source_data containing key columns.
# @POST: Returns filtered list of source rows with only new keys. Logs skip count.
# @SIDE_EFFECT: Queries TranslationRecord from previous runs; no writes.
def _filter_new_keys(self, job, run_id: str, source_rows: list) -> list:
with belief_scope("TranslationExecutor._filter_new_keys"):
# Find most recent COMPLETED run with successful insert for this job
prev_run = (
self.db.query(TranslationRun)
.filter(
TranslationRun.job_id == job.id,
TranslationRun.status == "COMPLETED",
TranslationRun.insert_status == "succeeded",
TranslationRun.id != run_id,
)
.order_by(TranslationRun.created_at.desc())
.first()
)
if not prev_run:
logger.reason("No prior successful run — all keys treated as new", {
"job_id": job.id,
})
return source_rows
# Get successful records from that run
prev_records = (
self.db.query(TranslationRecord)
.filter(
TranslationRecord.run_id == prev_run.id,
TranslationRecord.status == "SUCCESS",
)
.all()
)
if not prev_records:
return source_rows
# Build set of already-translated composite keys
key_cols = job.target_key_cols or job.source_key_cols or []
if not key_cols:
logger.explore("No key columns configured — skipping new-key-only filter",
{"job_id": job.id})
return source_rows
existing_keys = set()
for rec in prev_records:
sd = rec.source_data or {}
key_tuple = tuple(str(sd.get(k, "")) for k in key_cols)
existing_keys.add(key_tuple)
# Filter: keep only rows whose keys are NOT in the existing set
filtered = []
skipped = 0
for row in source_rows:
sd = row.get("source_data", {}) or {}
key_tuple = tuple(str(sd.get(k, "")) for k in key_cols)
if key_tuple not in existing_keys:
filtered.append(row)
else:
skipped += 1
logger.reason(f"New-key-only filter: {len(source_rows)} total → {len(filtered)} new, {skipped} skipped", {
"job_id": job.id,
"prev_run_id": prev_run.id,
"key_cols": key_cols,
})
return filtered
# endregion _filter_new_keys
# region _extract_chart_data_rows [TYPE Function]
# @PURPOSE: Extract data rows from Superset chart data API response.
# @POST: Returns list of dicts with column-value pairs.
@@ -763,4 +846,3 @@ class TranslationExecutor:
# #endregion TranslationExecutor
# #endregion TranslationExecutor
# #endregion TranslationExecutor

71
backend/src/plugins/translate/scheduler.py
View File

@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional
from sqlalchemy.orm import Session
from ...core.logger import logger, belief_scope
from ...core.cot_logger import seed_trace_id
from ...core.config_manager import ConfigManager
from ...models.translate import TranslationSchedule, TranslationJob, TranslationRun
from .events import TranslationEventLog
@@ -44,6 +45,7 @@ class TranslationScheduler:
cron_expression: str,
timezone: str = "UTC",
is_active: bool = True,
execution_mode: str = "full",
) -> TranslationSchedule:
with belief_scope("TranslationScheduler.create_schedule"):
# Verify job exists
@@ -57,6 +59,7 @@ class TranslationScheduler:
cron_expression=cron_expression,
timezone=timezone,
is_active=is_active,
execution_mode=execution_mode,
created_by=self.current_user,
)
self.db.add(schedule)
@@ -88,6 +91,7 @@ class TranslationScheduler:
cron_expression: Optional[str] = None,
timezone_str: Optional[str] = None,
is_active: Optional[bool] = None,
execution_mode: Optional[str] = None,
) -> TranslationSchedule:
with belief_scope("TranslationScheduler.update_schedule"):
schedule = self.db.query(TranslationSchedule).filter(
@@ -102,6 +106,8 @@ class TranslationScheduler:
schedule.timezone = timezone_str
if is_active is not None:
schedule.is_active = is_active
if execution_mode is not None:
schedule.execution_mode = execution_mode
schedule.updated_at = datetime.now(timezone.utc)
self.db.commit()
self.db.refresh(schedule)
@@ -245,8 +251,10 @@ def execute_scheduled_translation(
job_id: str,
db_session_maker,
config_manager: ConfigManager,
execution_mode: str = "full",
) -> None:
"""APScheduler job callback for scheduled translations."""
seed_trace_id()
db: Session = db_session_maker()
try:
with belief_scope("execute_scheduled_translation"):
@@ -265,26 +273,57 @@ def execute_scheduled_translation(
})
# Concurrency check: max 1 pending/running run per job
STALE_THRESHOLD = timedelta(hours=1)
now = datetime.now(timezone.utc)
active_run = (
db.query(TranslationRun)
.filter(
TranslationRun.job_id == job_id,
TranslationRun.status.in_(["PENDING", "RUNNING"]),
)
.order_by(TranslationRun.created_at.asc()) # oldest first
.first()
)
if active_run:
logger.explore("Skipping scheduled run — concurrent run in progress", {
"job_id": job_id,
"existing_run_id": active_run.id,
})
event_log = TranslationEventLog(db)
event_log.log_event(
job_id=job_id,
event_type="RUN_STARTED",
payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id},
is_stale = (
active_run.status == "PENDING"
and active_run.created_at
and (now - active_run.created_at) > STALE_THRESHOLD
)
return
if is_stale:
# Mark ALL stale PENDING runs as FAILED
stale_runs = (
db.query(TranslationRun)
.filter(
TranslationRun.job_id == job_id,
TranslationRun.status == "PENDING",
TranslationRun.created_at < (now - STALE_THRESHOLD),
)
.all()
)
for sr in stale_runs:
sr.status = "FAILED"
sr.error_message = "Stale: concurrency deadlock — marked by scheduled trigger"
sr.completed_at = now
db.commit()
logger.explore(f"Cleared {len(stale_runs)} stale PENDING run(s), proceeding with scheduled trigger", {
"job_id": job_id,
"stale_run_ids": [sr.id for sr in stale_runs],
})
# Proceed to execute the scheduled run (do NOT return)
else:
logger.explore("Skipping scheduled run — concurrent run in progress", {
"job_id": job_id,
"existing_run_id": active_run.id,
})
event_log = TranslationEventLog(db)
event_log.log_event(
job_id=job_id,
event_type="RUN_STARTED",
payload={"reason": "skipped_concurrent", "existing_run_id": active_run.id},
)
return
# Check baseline expiry
baseline_expired = False
@@ -315,7 +354,17 @@ def execute_scheduled_translation(
job_id=job_id,
is_scheduled=True,
)
run.trigger_type = "scheduled"
# Determine trigger type based on execution_mode and baseline expiry
if execution_mode == "new_key_only" and not baseline_expired:
run.trigger_type = "new_key_only"
elif execution_mode == "new_key_only" and baseline_expired:
logger.reason("Baseline expired — falling back to full translation", {
"job_id": job_id, "schedule_id": schedule_id, "age_days": age.days,
})
run.trigger_type = "scheduled"
else:
run.trigger_type = "scheduled"
db.flush()
if baseline_expired: