busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(health): suppress 404 when health monitor disabled + fix audit test assertion

Browse Source 
- Sidebar.svelte: defer healthStore.refresh() until feature flags confirm
  health_monitor is enabled; skip entirely when disabled
- health.js: remove throw error from catch block — log and return null
  instead, preventing 'Uncaught (in promise)' on expected 404
- test_report_audit_immutability.py: fix mock assertions —
  audit_service uses logger.reason/reflect/explore, not logger.info
- HealthStore and Sidebar now produce zero network noise when
  FEATURES__HEALTH_MONITOR=false
This commit is contained in:
busya
2026-05-17 14:18:02 +03:00
parent 58ac89c21e
commit cd868df261
141 changed files with 9631 additions and 10165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/api/routes/git/_repo_operations_routes.py
View File

@@ -293,14 +293,14 @@ async def generate_commit_message(
history_objs = _gs.get_commit_history(dashboard_id, limit=5)
history = [h.message for h in history_objs if hasattr(h, "message")]
from ...plugins.llm_analysis.models import LLMProviderType
from ...plugins.llm_analysis.service import LLMClient
from ...services.llm_prompt_templates import (
from src.plugins.llm_analysis.models import LLMProviderType
from src.plugins.llm_analysis.service import LLMClient
from src.services.llm_prompt_templates import (
DEFAULT_LLM_PROMPTS,
normalize_llm_settings,
resolve_bound_provider_id,
)
from ...services.llm_provider import LLMProviderService
from src.services.llm_provider import LLMProviderService
llm_service = LLMProviderService(db)
providers = llm_service.get_all_providers()
@@ -325,7 +325,7 @@ async def generate_commit_message(
default_model=provider.default_model,
)
from ...plugins.git.llm_extension import GitLLMExtension
from src.plugins.git.llm_extension import GitLLMExtension
extension = GitLLMExtension(client)
git_prompt = llm_settings["prompts"].get(

21
backend/src/api/routes/translate/__init__.py
View File

@@ -1,15 +1,18 @@
# #region TranslateRoutes [C:4] [TYPE Module] [SEMANTICS translate, api, package, schedule, dashboard, llm]
# @BRIEF API routes for LLM-based SQL/dashboard translation management including terminology dictionary CRUD and import.
# @LAYER: UI (API)
# @RELATION DEPENDS_ON -> [TranslateJobResponse]
# @RELATION DEPENDS_ON -> [DictionaryManager:Class]
# @RELATION DEPENDS_ON -> [get_current_user]
# @RELATION DEPENDS_ON -> [get_db]
# @RELATION DEPENDS_ON -> [has_permission]
# @RELATION DEPENDS_ON -> [ConfigManager]
# @LAYER API
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RATIONALE: Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED: Invalidating in-progress runs on config edit would break scheduled run continuity.
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RELATION DEPENDS_ON -> [SupersetClient]
# @RATIONALE Snapshot isolation — in-progress runs use config snapshot; config edits affect future runs only.
# @REJECTED Invalidating in-progress runs on config edit would break scheduled run continuity.
# @POST Translation job, run, dictionary, correction, preview, and schedule CRUD operations are executed.
# @PRE API server is running, user is authenticated, database is accessible.
# @SIDE_EFFECT Creates/modifies DB rows; triggers background translation runs; queries Superset API.
"""
Translate routes package.

18
backend/src/api/routes/translate/_correction_routes.py
View File

@@ -20,10 +20,12 @@ from ._router import router
# Corrections
# ============================================================
# #region submit_correction [TYPE Function]
# #region submit_correction [C:4] [TYPE Function]
# @BRIEF Submit a term correction from a run result review.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Correction applied with conflict detection.
# @PRE User has translate.dictionary.edit permission.
# @POST Correction applied with conflict detection.
# @SIDE_EFFECT Creates/updates DictionaryEntry row; commits.
# @RELATION DEPENDS_ON -> [DictionaryManager]
@router.post("/corrections")
async def submit_correction(
payload: TermCorrectionSubmit,
@@ -55,10 +57,14 @@ async def submit_correction(
# #endregion submit_correction
# #region submit_bulk_corrections [TYPE Function]
# #region submit_bulk_corrections [C:4] [TYPE Function]
# @BRIEF Submit multiple term corrections atomically.
# @PRE: User has translate.dictionary.edit permission.
# @POST: All corrections applied or none with conflict list.
# @PRE User has translate.dictionary.edit permission.
# @POST All corrections applied or none with conflict list.
# @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
# @RELATION DEPENDS_ON -> [DictionaryManager]
# @COMPLEXITY 4
@router.post("/corrections/bulk")
async def submit_bulk_corrections(
payload: TermCorrectionBulkSubmit,

26
backend/src/api/routes/translate/_dictionary_routes.py
View File

@@ -22,7 +22,7 @@ from ._router import router
# Terminology Dictionaries
# ============================================================
# #region list_dictionaries [TYPE Function]
# #region list_dictionaries [C:4] [TYPE Function]
# @BRIEF List all terminology dictionaries.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns list of dictionaries with total count.
@@ -46,7 +46,7 @@ async def list_dictionaries(
# #endregion list_dictionaries
# #region get_dictionary [TYPE Function]
# #region get_dictionary [C:4] [TYPE Function]
# @BRIEF Get a single terminology dictionary by ID.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns the dictionary with entry_count.
@@ -71,7 +71,7 @@ async def get_dictionary(
# #endregion get_dictionary
# #region create_dictionary [TYPE Function]
# #region create_dictionary [C:4] [TYPE Function]
# @BRIEF Create a new terminology dictionary.
# @PRE: User has translate.dictionary.create permission.
# @POST: Returns the created dictionary.
@@ -99,7 +99,7 @@ async def create_dictionary(
# #endregion create_dictionary
# #region update_dictionary [TYPE Function]
# #region update_dictionary [C:4] [TYPE Function]
# @BRIEF Update an existing terminology dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Returns the updated dictionary.
@@ -133,7 +133,7 @@ async def update_dictionary(
# #endregion update_dictionary
# #region delete_dictionary [TYPE Function]
# #region delete_dictionary [C:4] [TYPE Function]
# @BRIEF Delete a terminology dictionary, blocked if attached to active/scheduled jobs.
# @PRE: User has translate.dictionary.delete permission.
# @POST: Dictionary is deleted.
@@ -162,7 +162,7 @@ async def delete_dictionary(
# Dictionary Entries CRUD
# ============================================================
# #region list_dictionary_entries [TYPE Function]
# #region list_dictionary_entries [C:4] [TYPE Function]
# @BRIEF List entries for a dictionary, optionally filtered by language pair.
# @PRE: User has translate.dictionary.view permission.
# @POST: Returns paginated list of entries with language pair fields.
@@ -220,7 +220,7 @@ async def list_dictionary_entries(
# #endregion list_dictionary_entries
# #region add_dictionary_entry [TYPE Function]
# #region add_dictionary_entry [C:4] [TYPE Function]
# @BRIEF Add a new entry to a dictionary.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is created.
@@ -269,7 +269,7 @@ async def add_dictionary_entry(
# #endregion add_dictionary_entry
# #region edit_dictionary_entry [TYPE Function]
# #region edit_dictionary_entry [C:4] [TYPE Function]
# @BRIEF Update an existing dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is updated.
@@ -319,7 +319,7 @@ async def edit_dictionary_entry(
# #endregion edit_dictionary_entry
# #region delete_dictionary_entry [TYPE Function]
# #region delete_dictionary_entry [C:4] [TYPE Function]
# @BRIEF Delete a dictionary entry.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entry is deleted.
@@ -343,10 +343,12 @@ async def delete_dictionary_entry(
# #endregion delete_dictionary_entry
# #region import_dictionary_entries [TYPE Function]
# #region import_dictionary_entries [C:4] [TYPE Function]
# @BRIEF Import entries into a terminology dictionary from CSV/TSV content.
# @PRE: User has translate.dictionary.edit permission.
# @POST: Entries are imported per conflict mode.
# @PRE User has translate.dictionary.edit permission.
# @POST Entries are imported per conflict mode.
# @COMPLEXITY 4
@router.post("/dictionaries/{dictionary_id}/import")
async def import_dictionary_entries(
dictionary_id: str,

2
backend/src/api/routes/translate/_helpers.py
View File

@@ -1,6 +1,6 @@
# #region TranslateHelpersModule [C:2] [TYPE Module] [SEMANTICS sqlalchemy, translate, helper, query, mapping]
# @BRIEF Shared helper functions for translate route handlers.
# @LAYER: API
# @LAYER API
# @RELATION DEPENDS_ON -> [models.translate]
from typing import Any

25
backend/src/api/routes/translate/_job_routes.py
View File

@@ -31,7 +31,7 @@ from ._router import router
# Translation Job CRUD
# ============================================================
# #region list_jobs [TYPE Function]
# #region list_jobs [C:4] [TYPE Function]
# @BRIEF List all translation jobs.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of translation jobs.
@@ -64,7 +64,7 @@ async def list_jobs(
# #endregion list_jobs
# #region get_job [TYPE Function]
# #region get_job [C:4] [TYPE Function]
# @BRIEF Get a single translation job by ID.
# @PRE: User has translate.job.view permission.
# @POST: Returns the translation job.
@@ -88,7 +88,7 @@ async def get_job(
# #endregion get_job
# #region create_job [TYPE Function]
# #region create_job [C:4] [TYPE Function]
# @BRIEF Create a new translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the created translation job.
@@ -113,7 +113,7 @@ async def create_job(
# #endregion create_job
# #region update_job [TYPE Function]
# #region update_job [C:4] [TYPE Function]
# @BRIEF Update an existing translation job.
# @PRE: User has translate.job.edit permission.
# @POST: Returns the updated translation job.
@@ -139,7 +139,7 @@ async def update_job(
# #endregion update_job
# #region delete_job [TYPE Function]
# #region delete_job [C:4] [TYPE Function]
# @BRIEF Delete a translation job.
# @PRE: User has translate.job.delete permission.
# @POST: Job is deleted.
@@ -161,7 +161,7 @@ async def delete_job(
# #endregion delete_job
# #region duplicate_job [TYPE Function]
# #region duplicate_job [C:4] [TYPE Function]
# @BRIEF Duplicate a translation job.
# @PRE: User has translate.job.create permission.
# @POST: Returns the duplicated job with status DRAFT.
@@ -188,13 +188,14 @@ async def duplicate_job(
# #endregion duplicate_job
# #region get_datasource_columns [TYPE Function]
# #region get_datasource_columns [C:4] [TYPE Function]
# @BRIEF Get column metadata and database dialect for a Superset datasource.
# @PRE: User has translate.job.view permission.
# @POST: Returns column list with metadata and database_dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# @RATIONALE: Dialect detection from Superset connection cached on TranslationJob at save time.
# @REJECTED: Hardcoding dialect list per database — would drift from actual Superset config.
# @PRE User has translate.job.view permission.
# @POST Returns column list with metadata and database_dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# @RATIONALE Dialect detection from Superset connection cached on TranslationJob at save time.
# @REJECTED Hardcoding dialect list per database — would drift from actual Superset config.
# @COMPLEXITY 4
@router.get("/datasources")
async def list_datasources(

10
backend/src/api/routes/translate/_metrics_routes.py
View File

@@ -17,7 +17,7 @@ from ._router import router
# Metrics
# ============================================================
# #region get_metrics [TYPE Function]
# #region get_metrics [C:4] [TYPE Function]
# @BRIEF Get translation metrics, optionally filtered by job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics data.
@@ -40,10 +40,12 @@ async def get_metrics(
# #endregion get_metrics
# #region get_job_metrics [TYPE Function]
# #region get_job_metrics [C:4] [TYPE Function]
# @BRIEF Get aggregated metrics for a specific job.
# @PRE: User has translate.metrics.view permission.
# @POST: Returns metrics dict.
# @PRE User has translate.metrics.view permission.
# @POST Returns metrics dict.
# @COMPLEXITY 4
@router.get("/jobs/{job_id}/metrics")
async def get_job_metrics(
job_id: str,

16
backend/src/api/routes/translate/_preview_routes.py
View File

@@ -23,7 +23,7 @@ from ._router import router
# Preview
# ============================================================
# #region preview_translation [TYPE Function]
# #region preview_translation [C:4] [TYPE Function]
# @BRIEF Preview a translation before applying it.
# @PRE: User has translate.job.execute permission.
# @POST: Returns a preview session with records and cost estimation.
@@ -59,7 +59,7 @@ async def preview_translation(
# #endregion preview_translation
# #region update_preview_row [TYPE Function]
# #region update_preview_row [C:4] [TYPE Function]
# @BRIEF Approve, edit, or reject a preview row (optionally per language).
# @PRE: User has translate.job.execute permission.
# @POST: Preview row status is updated.
@@ -91,7 +91,7 @@ async def update_preview_row(
# #endregion update_preview_row
# #region accept_preview_session [TYPE Function]
# #region accept_preview_session [C:4] [TYPE Function]
# @BRIEF Accept a preview session, marking it as the quality gate for full execution.
# @PRE: User has translate.job.execute permission. Job has an ACTIVE preview session.
# @POST: Preview session is marked as APPLIED; full execution can proceed.
@@ -114,7 +114,7 @@ async def accept_preview_session(
# #endregion accept_preview_session
# #region apply_preview [TYPE Function]
# #region apply_preview [C:4] [TYPE Function]
# @BRIEF Apply a preview session (alias for accept when accepting at session level).
# @PRE: User has translate.job.execute permission.
# @POST: Preview is applied.
@@ -146,10 +146,12 @@ async def apply_preview(
# Preview Records
# ============================================================
# #region get_preview_records [TYPE Function]
# #region get_preview_records [C:4] [TYPE Function]
# @BRIEF Get records for a preview session.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of preview records.
# @PRE User has translate.job.view permission.
# @POST Returns list of preview records.
# @COMPLEXITY 4
@router.get("/preview/{session_id}/records", response_model=list[PreviewRow])
async def get_preview_records(
session_id: str,

1
backend/src/api/routes/translate/_router.py
View File

@@ -6,6 +6,7 @@ from fastapi import APIRouter
# #region translate_router [TYPE Global]
# @BRIEF APIRouter instance for all translate sub-routes.
router = APIRouter(prefix="/api/translate", tags=["translate"])
# #endregion translate_router

12
backend/src/api/routes/translate/_run_list_routes.py
View File

@@ -19,7 +19,7 @@ from ._router import router
# List Runs (cross-job)
# ============================================================
# #region list_runs [TYPE Function]
# #region list_runs [C:4] [TYPE Function]
# @BRIEF List all runs with cross-job filtering and pagination.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated list of runs.
@@ -126,7 +126,7 @@ async def list_runs(
# Run Detail
# ============================================================
# #region get_run_detail [TYPE Function]
# #region get_run_detail [C:4] [TYPE Function]
# @BRIEF Get detailed run info with config_snapshot, records, events.
# @PRE: User has translate.history.view permission.
# @POST: Returns run detail with records and events.
@@ -169,10 +169,12 @@ async def get_run_detail(
# Download Skipped CSV
# ============================================================
# #region download_skipped_csv [TYPE Function]
# #region download_skipped_csv [C:4] [TYPE Function]
# @BRIEF Download a CSV of skipped translation records from a run.
# @PRE: User has translate.job.view permission.
# @POST: Returns CSV file of skipped records.
# @PRE User has translate.job.view permission.
# @POST Returns CSV file of skipped records.
# @COMPLEXITY 4
@router.get("/runs/{run_id}/skipped.csv")
async def download_skipped_csv(
run_id: str,

28
backend/src/api/routes/translate/_run_routes.py
View File

@@ -25,7 +25,7 @@ from ._router import router
# Translation Run / Execute
# ============================================================
# #region run_translation [TYPE Function]
# #region run_translation [C:4] [TYPE Function]
# @BRIEF Execute a translation job (trigger a run).
# @PRE: User has translate.job.execute permission.
# @POST: Returns the created translation run.
@@ -164,7 +164,7 @@ def run_translation(
# #endregion run_translation
# #region retry_run [TYPE Function]
# #region retry_run [C:4] [TYPE Function]
# @BRIEF Retry failed batches in a translation run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated translation run.
@@ -190,7 +190,7 @@ def retry_run(
# #endregion retry_run
# #region retry_insert [TYPE Function]
# #region retry_insert [C:4] [TYPE Function]
# @BRIEF Retry the SQL insert phase for a completed run.
# @PRE: User has translate.job.execute permission.
# @POST: Returns the updated run.
@@ -216,7 +216,7 @@ def retry_insert(
# #endregion retry_insert
# #region cancel_run [TYPE Function]
# #region cancel_run [C:4] [TYPE Function]
# @BRIEF Cancel a running translation.
# @PRE: User has translate.job.execute permission.
# @POST: Run is cancelled.
@@ -239,7 +239,7 @@ def cancel_run(
# #endregion cancel_run
# #region get_run_history [TYPE Function]
# #region get_run_history [C:4] [TYPE Function]
# @BRIEF Get run history for a translation job.
# @PRE: User has translate.history.view permission.
# @POST: Returns list of runs.
@@ -264,7 +264,7 @@ def get_run_history(
# #endregion get_run_history
# #region get_run_status [TYPE Function]
# #region get_run_status [C:4] [TYPE Function]
# @BRIEF Get status and statistics for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns run details with statistics.
@@ -286,7 +286,7 @@ def get_run_status(
# #endregion get_run_status
# #region get_run_records [TYPE Function]
# #region get_run_records [C:4] [TYPE Function]
# @BRIEF Get paginated records for a translation run.
# @PRE: User has translate.history.view permission.
# @POST: Returns paginated records.
@@ -315,7 +315,7 @@ def get_run_records(
# Batches
# ============================================================
# #region get_batches [TYPE Function]
# #region get_batches [C:4] [TYPE Function]
# @BRIEF Get batches for a translation run.
# @PRE: User has translate.job.view permission.
# @POST: Returns list of batches.
@@ -360,7 +360,7 @@ def get_batches(
# Manual Language Override
# ============================================================
# #region override_detected_language [TYPE Function]
# #region override_detected_language [C:4] [TYPE Function]
# @BRIEF Manually override the detected source language for a specific translation language entry.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.source_language_detected is updated; language_overridden is set to True.
@@ -439,7 +439,7 @@ def override_detected_language(
# Inline Correction
# ============================================================
# #region inline_edit_translation [C:3] [TYPE Function] [SEMANTICS api,translate,correction]
# #region inline_edit_translation [C:4] [TYPE Function] [SEMANTICS api,translate,correction]
# @BRIEF Apply an inline correction to a translated value on a completed run result.
# @PRE: User has translate.job.execute permission. Run, record, and language entry exist.
# @POST: TranslationLanguage.final_value and user_edit are updated. Optional dictionary submission.
@@ -488,10 +488,12 @@ def inline_edit_translation(
# Bulk Find-and-Replace
# ============================================================
# #region bulk_find_replace [C:3] [TYPE Function] [SEMANTICS api,translate,bulk,replace]
# #region bulk_find_replace [C:4] [TYPE Function] [SEMANTICS api,translate,bulk,replace]
# @BRIEF Perform bulk find-and-replace on translated values within a run.
# @PRE: User has translate.job.execute permission. Run exists.
# @POST: If preview=false, matching translations are updated. Optional dictionary submission.
# @PRE User has translate.job.execute permission. Run exists.
# @POST If preview=false, matching translations are updated. Optional dictionary submission.
# @COMPLEXITY 4
@router.post("/runs/{run_id}/bulk-replace")
def bulk_find_replace(
run_id: str,

18
backend/src/api/routes/translate/_schedule_routes.py
View File

@@ -18,7 +18,7 @@ from ._router import router
# Schedule
# ============================================================
# #region get_schedule [TYPE Function]
# #region get_schedule [C:4] [TYPE Function]
# @BRIEF Get the schedule for a translation job.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns the schedule configuration.
@@ -53,7 +53,7 @@ async def get_schedule(
# #endregion get_schedule
# #region set_schedule [TYPE Function]
# #region set_schedule [C:4] [TYPE Function]
# @BRIEF Set or update the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is created or updated.
@@ -119,7 +119,7 @@ async def set_schedule(
# #endregion set_schedule
# #region enable_schedule [TYPE Function]
# #region enable_schedule [C:4] [TYPE Function]
# @BRIEF Enable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is enabled.
@@ -151,7 +151,7 @@ async def enable_schedule(
# #endregion enable_schedule
# #region disable_schedule [TYPE Function]
# #region disable_schedule [C:4] [TYPE Function]
# @BRIEF Disable a schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is disabled.
@@ -177,7 +177,7 @@ async def disable_schedule(
# #endregion disable_schedule
# #region delete_schedule [TYPE Function]
# #region delete_schedule [C:4] [TYPE Function]
# @BRIEF Delete the schedule for a translation job.
# @PRE: User has translate.schedule.manage permission.
# @POST: Schedule is removed.
@@ -203,10 +203,12 @@ async def delete_schedule(
# #endregion delete_schedule
# #region get_next_executions [TYPE Function]
# #region get_next_executions [C:4] [TYPE Function]
# @BRIEF Preview next N executions for a job's schedule.
# @PRE: User has translate.schedule.view permission.
# @POST: Returns next execution times.
# @PRE User has translate.schedule.view permission.
# @POST Returns next execution times.
# @COMPLEXITY 4
@router.get("/jobs/{job_id}/schedule/next-executions")
async def get_next_executions(
job_id: str,

2
backend/src/models/translate.py
View File

@@ -1,7 +1,7 @@
# #region TranslateModels [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, model, schema, dashboard, llm]
# @BRIEF SQLAlchemy ORM models for LLM-based SQL/dashboard translation across dialects.
# @LAYER: Domain
# @RELATION INHERITS_FROM -> MappingModels:Base
# @RELATION INHERITS -> [MappingModels]
import uuid
from datetime import UTC, datetime

258
backend/src/plugins/git_plugin.py
View File

@@ -1,4 +1,4 @@
# #region GitPluginModule [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset]
# #region GitPluginModule [C:4] [TYPE Module] [SEMANTICS git, versioning, deploy, sync, superset, backup, transactional]
#
# @BRIEF Предоставляет плагин для версионирования и развертывания дашбордов Superset.
# @LAYER: Plugin
@@ -10,10 +10,13 @@
#
# @INVARIANT: Все операции с Git должны выполняться через GitService.
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
# @INVARIANT: _handle_sync сохраняет backup управляемых директорий перед удалением;
# при ошибке распаковки backup восстанавливается.
import io
import os
import shutil
import time
import zipfile
from pathlib import Path
from typing import Any
@@ -62,52 +65,33 @@ class GitPlugin(PluginBase):
# endregion __init__
@property
# region id [TYPE Function]
# @PURPOSE: Returns the plugin identifier.
# @PRE: GitPlugin is initialized.
# @POST: Returns 'git-integration'.
# region id [C:1] [TYPE Function]
def id(self) -> str:
with belief_scope("GitPlugin.id"):
return "git-integration"
return "git-integration"
# endregion id
@property
# region name [TYPE Function]
# @PURPOSE: Returns the plugin name.
# @PRE: GitPlugin is initialized.
# @POST: Returns the human-readable name.
# region name [C:1] [TYPE Function]
def name(self) -> str:
with belief_scope("GitPlugin.name"):
return "Git Integration"
return "Git Integration"
# endregion name
@property
# region description [TYPE Function]
# @PURPOSE: Returns the plugin description.
# @PRE: GitPlugin is initialized.
# @POST: Returns the plugin's purpose description.
# region description [C:1] [TYPE Function]
def description(self) -> str:
with belief_scope("GitPlugin.description"):
return "Version control for Superset dashboards"
return "Version control for Superset dashboards"
# endregion description
@property
# region version [TYPE Function]
# @PURPOSE: Returns the plugin version.
# @PRE: GitPlugin is initialized.
# @POST: Returns the version string.
# region version [C:1] [TYPE Function]
def version(self) -> str:
with belief_scope("GitPlugin.version"):
return "0.1.0"
return "0.1.0"
# endregion version
@property
# region ui_route [TYPE Function]
# @PURPOSE: Returns the frontend route for the git plugin.
# @RETURN: str - "/git"
# region ui_route [C:1] [TYPE Function]
def ui_route(self) -> str:
with belief_scope("GitPlugin.ui_route"):
return "/git"
return "/git"
# endregion ui_route
# region get_schema [TYPE Function]
@@ -129,21 +113,13 @@ class GitPlugin(PluginBase):
}
# endregion get_schema
# region initialize [TYPE Function]
# @PURPOSE: Выполняет начальную настройку плагина.
# @PRE: GitPlugin is initialized.
# @POST: Плагин готов к выполнению задач.
# region initialize [C:1] [TYPE Function]
async def initialize(self):
with belief_scope("GitPlugin.initialize"):
app_logger.reason("Initializing Git Integration Plugin logic.", extra={"src": "GitPlugin.initialize"})
app_logger.reason("Initializing Git Integration Plugin logic.", extra={"src": "GitPlugin.initialize"})
# endregion initialize
# region execute [TYPE Function]
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
# @PRE: task_data содержит 'operation' и 'dashboard_id'.
# @POST: Возвращает результат выполнения операции.
# @PARAM: task_data (Dict[str, Any]) - Данные задачи.
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
# @RETURN: Dict[str, Any] - Статус и сообщение.
# region execute [C:3] [TYPE Function]
# @PURPOSE: Main task executor with TaskContext support.
# @RELATION: CALLS -> self._handle_sync
# @RELATION: CALLS -> self._handle_deploy
async def execute(self, task_data: dict[str, Any], context: TaskContext | None = None) -> dict[str, Any]:
@@ -176,41 +152,48 @@ class GitPlugin(PluginBase):
return result
# endregion execute
# region _handle_sync [TYPE Function]
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.
# region _handle_sync [C:4] [TYPE Function] [SEMANTICS git,sync,backup,transactional]
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий с backup/restore.
# @PRE: Репозиторий для дашборда должен существовать.
# @POST: Файлы в репозитории обновлены до текущего состояния в Superset.
# @PARAM: dashboard_id (int) - ID дашборда.
# @PARAM: source_env_id (Optional[str]) - ID исходного окружения.
# @RETURN: Dict[str, str] - Результат синхронизации.
# Управляемые директории backup-ируются перед удалением; при ошибке распаковки backup восстанавливается.
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
# Создаёт временный backup в /tmp/ss-tools-backup-{dashboard_id}-{timestamp}/.
# @RETURN: Dict[str, str] - Результат синхронизации.
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
async def _handle_sync(self, dashboard_id: int, source_env_id: str | None = None, log=None, git_log=None, superset_log=None) -> dict[str, str]:
with belief_scope("GitPlugin._handle_sync"):
try:
# 1. Получение репозитория
repo = self.git_service.get_repo(dashboard_id)
repo_path = Path(repo.working_dir)
git_log.info(f"Target repo path: {repo_path}")
# 2. Настройка клиента Superset
env = self._get_env(source_env_id)
client = SupersetClient(env)
client.authenticate()
# 3. Экспорт дашборда
superset_log.info(f"Exporting dashboard {dashboard_id} from {env.name}")
zip_bytes, _ = client.export_dashboard(dashboard_id)
# 4. Распаковка с выравниванием структуры (flattening)
git_log.info(f"Unpacking export to {repo_path}")
# Список папок/файлов, которые мы ожидаем от Superset
managed_dirs = ["dashboards", "charts", "datasets", "databases"]
managed_files = ["metadata.yaml"]
# Очистка старых данных перед распаковкой, чтобы не оставалось "призраков"
# Fix 1: Create backup before deleting managed files
backup_dir = Path(f"/tmp/ss-tools-backup-{dashboard_id}-{time.time_ns()}")
for d in managed_dirs:
d_path = repo_path / d
if d_path.exists() and d_path.is_dir():
shutil.copytree(d_path, backup_dir / d)
for f in managed_files:
f_path = repo_path / f
if f_path.exists():
(backup_dir / f).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(f_path, backup_dir / f)
# Delete old managed files
for d in managed_dirs:
d_path = repo_path / d
if d_path.exists() and d_path.is_dir():
@@ -220,30 +203,45 @@ class GitPlugin(PluginBase):
if f_path.exists():
f_path.unlink()
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
# Superset экспортирует всё в подпапку dashboard_export_timestamp/
# Нам нужно найти это имя папки
namelist = zf.namelist()
if not namelist:
raise ValueError("Export ZIP is empty")
try:
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
namelist = zf.namelist()
if not namelist:
raise ValueError("Export ZIP is empty")
root_folder = namelist[0].split('/')[0]
git_log.info(f"Detected root folder in ZIP: {root_folder}")
for member in zf.infolist():
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
relative_path = member.filename[len(root_folder)+1:]
target_path = repo_path / relative_path
if member.is_dir():
target_path.mkdir(parents=True, exist_ok=True)
else:
target_path.parent.mkdir(parents=True, exist_ok=True)
with zf.open(member) as source, open(target_path, "wb") as target:
shutil.copyfileobj(source, target)
except Exception:
# Fix 1: Restore backup on unzip failure
if backup_dir.exists():
for d in managed_dirs:
src = backup_dir / d
if src.exists():
dst = repo_path / d
if dst.exists():
shutil.rmtree(dst)
shutil.copytree(src, dst)
for f in managed_files:
src = backup_dir / f
if src.exists():
dst = repo_path / f
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
raise
root_folder = namelist[0].split('/')[0]
git_log.info(f"Detected root folder in ZIP: {root_folder}")
# Fix 1: Remove backup on success
if backup_dir.exists():
shutil.rmtree(backup_dir, ignore_errors=True)
for member in zf.infolist():
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
# Убираем префикс папки
relative_path = member.filename[len(root_folder)+1:]
target_path = repo_path / relative_path
if member.is_dir():
target_path.mkdir(parents=True, exist_ok=True)
else:
target_path.parent.mkdir(parents=True, exist_ok=True)
with zf.open(member) as source, open(target_path, "wb") as target:
shutil.copyfileobj(source, target)
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
try:
repo.git.add(A=True)
app_logger.reason("Changes staged in git", extra={"src": "_handle_sync"})
@@ -258,35 +256,21 @@ class GitPlugin(PluginBase):
raise
# endregion _handle_sync
# region _handle_deploy [TYPE Function]
# @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.
# @PRE: environment_id должен соответствовать настроенному окружению.
# @POST: Дашборд импортирован в целевой Superset.
# @PARAM: dashboard_id (int) - ID дашборда.
# @PARAM: env_id (str) - ID целевого окружения.
# @PARAM: log - Main logger instance.
# @PARAM: git_log - Git-specific logger instance.
# @PARAM: superset_log - Superset API-specific logger instance.
# @RETURN: Dict[str, Any] - Результат деплоя.
# @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.
# region _handle_deploy [C:4] [TYPE Function] [SEMANTICS git,deploy,zip]
# @PURPOSE: Packages repository into ZIP and imports into target Superset environment.
# @POST: Dashboard imported into target Superset.
# @SIDE_EFFECT: Creates and removes temporary ZIP file.
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> dict[str, Any]:
with belief_scope("GitPlugin._handle_deploy"):
try:
if not env_id:
raise ValueError("Target environment ID required for deployment")
# 1. Получение репозитория
repo = self.git_service.get_repo(dashboard_id)
repo_path = Path(repo.working_dir)
# 2. Упаковка в ZIP
git_log.info(f"Packing repository {repo_path} for deployment.")
zip_buffer = io.BytesIO()
# Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)
root_dir_name = f"dashboard_export_{dashboard_id}"
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
for root, dirs, files in os.walk(repo_path):
if ".git" in dirs:
@@ -295,21 +279,14 @@ class GitPlugin(PluginBase):
if file == ".git" or file.endswith(".zip"):
continue
file_path = Path(root) / file
# Prepend the root directory name to the archive path
arcname = Path(root_dir_name) / file_path.relative_to(repo_path)
zf.write(file_path, arcname)
zip_buffer.seek(0)
# 3. Настройка клиента Superset
env = self.config_manager.get_environment(env_id)
if not env:
raise ValueError(f"Environment {env_id} not found")
client = SupersetClient(env)
client.authenticate()
# 4. Импорт
temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip"
git_log.info(f"Saving temporary zip to {temp_zip_path}")
with open(temp_zip_path, "wb") as f:
@@ -329,11 +306,12 @@ class GitPlugin(PluginBase):
raise
# endregion _handle_deploy
# region _get_env [TYPE Function]
# region _get_env [C:4] [TYPE Function] [SEMANTICS env,config,strict]
# @PURPOSE: Вспомогательный метод для получения конфигурации окружения.
# @PARAM: env_id (Optional[str]) - ID окружения.
# @PRE: env_id is a string or None.
# @POST: Returns an Environment object from config or DB.
# When env_id is explicitly provided and not found, raises ValueError (no fallback).
# @RETURN: Environment - Объект конфигурации окружения.
def _get_env(self, env_id: str | None = None):
with belief_scope("GitPlugin._get_env"):
@@ -346,54 +324,46 @@ class GitPlugin(PluginBase):
app_logger.reflect(f"Found environment by ID in ConfigManager: {env.name}", extra={"src": "_get_env"})
return env
# Priority 2: Database (DeploymentEnvironment)
from src.core.database import SessionLocal
from src.models.git import DeploymentEnvironment
# Priority 2: Database (DeploymentEnvironment)
from src.core.database import SessionLocal
from src.models.git import DeploymentEnvironment
db = SessionLocal()
try:
if env_id:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
else:
# If no ID, try to find active or any environment in DB
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
if not db_env:
db_env = db.query(DeploymentEnvironment).first()
db = SessionLocal()
try:
if env_id:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
else:
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
if not db_env:
db_env = db.query(DeploymentEnvironment).first()
if db_env:
app_logger.reflect(f"Found environment in DB: {db_env.name}", extra={"src": "_get_env"})
from src.core.config_models import Environment
# Use token as password for SupersetClient
return Environment(
id=db_env.id,
name=db_env.name,
url=db_env.superset_url,
username="admin",
password=db_env.superset_token,
verify_ssl=True
)
finally:
db.close()
if db_env:
app_logger.reflect(f"Found environment in DB: {db_env.name}", extra={"src": "_get_env"})
from src.core.config_models import Environment
return Environment(
id=db_env.id,
name=db_env.name,
url=db_env.superset_url,
username="admin",
password=db_env.superset_token,
verify_ssl=True
)
# Priority 3: ConfigManager Default (if no env_id provided)
envs = self.config_manager.get_environments()
if envs:
if env_id:
# If env_id was provided but not found in DB or specifically by ID in config,
# but we have other envs, maybe it's one of them?
env = next((e for e in envs if e.id == env_id), None)
if env:
app_logger.reflect(f"Found environment {env_id} in ConfigManager list", extra={"src": "_get_env"})
return env
# Fix 7: Strict check — if env_id was explicitly provided but not found, raise immediately
if env_id:
raise ValueError(f"Environment '{env_id}' not found in ConfigManager or database")
finally:
db.close()
if not env_id:
app_logger.reflect(f"Using first environment from ConfigManager: {envs[0].name}", extra={"src": "_get_env"})
return envs[0]
# Priority 3: fallback to first env only when no specific env_id was requested
envs = self.config_manager.get_environments()
if envs and not env_id:
app_logger.reflect(f"Using first environment from ConfigManager: {envs[0].name}", extra={"src": "_get_env"})
return envs[0]
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
# endregion _get_env
# endregion initialize
# #endregion GitPlugin
# #endregion GitPluginModule

1
backend/src/plugins/translate/__init__.py
View File

@@ -1,3 +1,2 @@
# #region TranslatePluginPackage [TYPE Package] [SEMANTICS translate, plugin, package, sql, dashboard]
# @BRIEF Translation plugin package for LLM-based cross-Superset SQL/dashboard translation.
# #endregion TranslatePluginPackage

44
backend/src/plugins/translate/_token_budget.py
View File

@@ -9,24 +9,24 @@
# Fixed batch_size of 50 — causes truncation on long-content rows.
# #region DEFAULT_CONTEXT_WINDOW [TYPE Constant]
# @BRIEF Default context window for DeepSeek v4 Flash: up to 64K tokens.
DEFAULT_CONTEXT_WINDOW = 64000
# #endregion DEFAULT_CONTEXT_WINDOW
# #region DEFAULT_MAX_OUTPUT_TOKENS [TYPE Constant]
# @BRIEF Default max_tokens setting for LLM output (16384 — sufficient for 50 rows x 4 languages).
# #region DEFAULT_MAX_OUTPUT_TOKENS [TYPE Constant]
DEFAULT_MAX_OUTPUT_TOKENS = 16384
# #endregion DEFAULT_MAX_OUTPUT_TOKENS
# #region REASONING_OVERHEAD [TYPE Constant]
# @BRIEF CoT reasoning overhead tokens for DeepSeek models (~2000 tokens for chain-of-thought).
# #region REASONING_OVERHEAD [TYPE Constant]
REASONING_OVERHEAD = 2000
# #endregion REASONING_OVERHEAD
# #region PROVIDER_DEFAULTS [TYPE Constant]
# @BRIEF Provider-aware defaults for context_window and max_output_tokens.
# Maps model name (or "default" fallback) to capacity limits.
# @RATIONALE: Different providers have drastically different context windows and
# @RATIONALE Different providers have drastically different context windows and
# output limits. Using a single default for all causes either wasted
# capacity (underestimation) or truncation (overestimation).
PROVIDER_DEFAULTS: dict[str, dict[str, int]] = {
@@ -38,46 +38,46 @@ PROVIDER_DEFAULTS: dict[str, dict[str, int]] = {
"default": {"context_window": 64000, "max_output_tokens": 16384},
}
# #endregion PROVIDER_DEFAULTS
# Increased from 60 to 120 because SQL/dashboard text and JSON structure need more.
# #region OUTPUT_PER_ROW_PER_LANG [TYPE Constant]
# @BRIEF Estimated output tokens per row per language in JSON response format.
# Increased from 60 to 120 because SQL/dashboard text and JSON structure need more.
OUTPUT_PER_ROW_PER_LANG = 120
# #endregion OUTPUT_PER_ROW_PER_LANG
# #endregion JSON_OVERHEAD_PER_ROW
# #region JSON_OVERHEAD_PER_ROW [TYPE Constant]
# @BRIEF Estimated overhead tokens for JSON keys, brackets, and formatting per row.
JSON_OVERHEAD_PER_ROW = 50
# #endregion JSON_OVERHEAD_PER_ROW
# #endregion PROMPT_BASE_TOKENS
# #region PROMPT_BASE_TOKENS [TYPE Constant]
# @BRIEF Base tokens for system prompt + instructions + dictionary section + JSON format specification.
# Increased from 300 to 600 to account for longer template, system msg, and dict preamble.
PROMPT_BASE_TOKENS = 600
# #endregion PROMPT_BASE_TOKENS
# @BRIEF Cap for dictionary tokens in estimation to avoid overestimating.
# #region DICT_TOKENS_PER_ENTRY [TYPE Constant]
# @BRIEF Estimated tokens per dictionary entry in the glossary section.
DICT_TOKENS_PER_ENTRY = 20
# #endregion DICT_TOKENS_PER_ENTRY
# #region DICT_TOKENS_MAX [TYPE Constant]
# @BRIEF Cap for dictionary tokens in estimation to avoid overestimating.
DICT_TOKENS_MAX = 5000
# #endregion DICT_TOKENS_MAX
MIN_MAX_TOKENS = 4096
# #region CHARS_PER_TOKEN_MIXED [TYPE Constant]
# @BRIEF Characters per token for mixed Russian/English text (empirical ~2.2 for mixed, ~2 for Russian, ~4 for English).
CHARS_PER_TOKEN_MIXED = 2.2
# #endregion CHARS_PER_TOKEN_MIXED
MAX_OUTPUT_HEADROOM = 3000
# #region MIN_MAX_TOKENS [TYPE Constant]
# @BRIEF Minimum max_tokens to allow (4096 ensures reasonable output even for small batches).
MIN_MAX_TOKENS = 4096
# #endregion MIN_MAX_TOKENS
# #endregion MIN_MAX_TOKENS
# #region MAX_OUTPUT_HEADROOM [TYPE Constant]
# @BRIEF Extra headroom added to max_output_needed beyond the estimate (3000 = 10-20% for variance).
# Increased from 1000 to 3000 because SQL/dashboard text output varies significantly.
MAX_OUTPUT_HEADROOM = 3000
# #endregion MAX_OUTPUT_HEADROOM

139
backend/src/plugins/translate/dictionary.py
View File

@@ -1,13 +1,16 @@
# #region DictionaryManagerModule [C:4] [TYPE Module] [SEMANTICS sqlalchemy, translate, dictionary, batch, term]
# @BRIEF Business logic for terminology dictionary management, entry CRUD, CSV/TSV import with conflict detection, and per-batch filtering.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class]
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RELATION DEPENDS_ON -> [TranslationJobDictionary:Class]
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE: C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED: Pure C3 CRUD without state guards would allow orphaned job-dictionary links.
# @REJECTED: "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RELATION DEPENDS_ON -> [TranslationJob:Class]
# @RATIONALE C4 complexity because dictionary CRUD is stateful with referential integrity enforcement on deletion and unique constraint on normalized source term.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @REJECTED "Keep both" as conflict option — UniqueConstraint(dictionary_id, source_term_normalized) prohibits variants; only overwrite/keep existing.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @PRE Database session is open and valid. Dictionary manager is properly initialized.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
import csv
import io
@@ -42,19 +45,18 @@ def _validate_bcp47(tag: str, field_name: str) -> None:
f"{field_name} is not a valid BCP-47 tag: '{tag}'. "
"Expected format like 'en', 'ru', 'zh-CN', 'pt-BR'."
)
# #endregion _validate_bcp47
# #region DictionaryManager [C:4] [TYPE Class]
# @BRIEF Manages terminology dictionaries and their entries with referential integrity.
# @PRE: Database session is open and valid.
# @POST: Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT: Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @PRE Database session is open and valid.
# @POST Dictionary and entry mutations are persisted with conflict detection.
# @SIDE_EFFECT Creates, updates, deletes TerminologyDictionary and DictionaryEntry rows; enforces deletion guards.
# @RELATION DEPENDS_ON -> [TerminologyDictionary:Class], DEPENDS_ON -> [DictionaryEntry:Class], DEPENDS_ON -> [TranslationJobDictionary:Class], DEPENDS_ON -> [TranslationJob:Class]
class DictionaryManager:
# region DictionaryManager.create_dictionary [TYPE Function]
# @PURPOSE: Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE: name is provided.
# @POST: New TerminologyDictionary row is created and returned.
# @PURPOSE Create a new terminology dictionary (language independent at dictionary level; language pairs are per-entry).
# @PRE name is provided.
# @POST New TerminologyDictionary row is created and returned.
@staticmethod
def create_dictionary(
db: Session, name: str,
@@ -81,9 +83,9 @@ class DictionaryManager:
# endregion DictionaryManager.create_dictionary
# region DictionaryManager.update_dictionary [TYPE Function]
# @PURPOSE: Update an existing terminology dictionary.
# @PRE: dict_id exists in terminology_dictionaries table.
# @POST: Dictionary metadata is updated and returned.
# @PURPOSE Update an existing terminology dictionary.
# @PRE dict_id exists in terminology_dictionaries table.
# @POST Dictionary metadata is updated and returned.
@staticmethod
def update_dictionary(
db: Session, dict_id: str, name: str | None = None,
@@ -112,10 +114,10 @@ class DictionaryManager:
# endregion DictionaryManager.update_dictionary
# region DictionaryManager.delete_dictionary [TYPE Function]
# @PURPOSE: Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE: dict_id exists.
# @POST: Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT: Deletes TerminologyDictionary row and all DictionaryEntry rows.
# @PURPOSE Delete a dictionary, blocked if attached to active/scheduled jobs.
# @PRE dict_id exists.
# @POST Dictionary and its entries are deleted, unless attached to ACTIVE/READY/RUNNING/SCHEDULED jobs.
# @SIDE_EFFECT Deletes TerminologyDictionary row and all DictionaryEntry rows.
@staticmethod
def delete_dictionary(db: Session, dict_id: str) -> None:
with belief_scope("DictionaryManager.delete_dictionary"):
@@ -155,9 +157,9 @@ class DictionaryManager:
# endregion DictionaryManager.delete_dictionary
# region DictionaryManager.get_dictionary [TYPE Function]
# @PURPOSE: Get a single dictionary by ID with entry count.
# @PRE: dict_id exists.
# @POST: Returns dict with dictionary + entry_count or raises ValueError.
# @PURPOSE Get a single dictionary by ID with entry count.
# @PRE dict_id exists.
# @POST Returns dict with dictionary + entry_count or raises ValueError.
@staticmethod
def get_dictionary(db: Session, dict_id: str) -> TerminologyDictionary:
dictionary = db.query(TerminologyDictionary).filter(TerminologyDictionary.id == dict_id).first()
@@ -167,9 +169,9 @@ class DictionaryManager:
# endregion DictionaryManager.get_dictionary
# region DictionaryManager.list_dictionaries [TYPE Function]
# @PURPOSE: List dictionaries with pagination and entry counts.
# @PRE: page >= 1, page_size between 1 and 100.
# @POST: Returns (list of dicts, total_count).
# @PURPOSE List dictionaries with pagination and entry counts.
# @PRE page >= 1, page_size between 1 and 100.
# @POST Returns (list of dicts, total_count).
@staticmethod
def list_dictionaries(
db: Session, page: int = 1, page_size: int = 20,
@@ -186,9 +188,9 @@ class DictionaryManager:
# endregion DictionaryManager.list_dictionaries
# region DictionaryManager.add_entry [TYPE Function]
# @PURPOSE: Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE: dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST: New DictionaryEntry row is created or raises on duplicate.
# @PURPOSE Add an entry to a dictionary with language-pair-aware uniqueness validation.
# @PRE dict_id exists. source_term, target_term are non-empty. source_language, target_language are valid BCP-47.
# @POST New DictionaryEntry row is created or raises on duplicate.
@staticmethod
def add_entry(
db: Session, dict_id: str, source_term: str, target_term: str,
@@ -237,9 +239,9 @@ class DictionaryManager:
# endregion DictionaryManager.add_entry
# region DictionaryManager.edit_entry [TYPE Function]
# @PURPOSE: Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE: entry_id exists.
# @POST: Entry fields are updated.
# @PURPOSE Edit an existing dictionary entry with language-pair-aware duplicate check.
# @PRE entry_id exists.
# @POST Entry fields are updated.
@staticmethod
def edit_entry(
db: Session, entry_id: str, source_term: str | None = None,
@@ -293,9 +295,9 @@ class DictionaryManager:
# endregion DictionaryManager.edit_entry
# region DictionaryManager.delete_entry [TYPE Function]
# @PURPOSE: Delete a single dictionary entry.
# @PRE: entry_id exists.
# @POST: Entry is deleted.
# @PURPOSE Delete a single dictionary entry.
# @PRE entry_id exists.
# @POST Entry is deleted.
@staticmethod
def delete_entry(db: Session, entry_id: str) -> None:
with belief_scope("DictionaryManager.delete_entry"):
@@ -309,9 +311,9 @@ class DictionaryManager:
# endregion DictionaryManager.delete_entry
# region DictionaryManager.clear_entries [TYPE Function]
# @PURPOSE: Delete all entries for a dictionary.
# @PRE: dict_id exists.
# @POST: All entries for the dictionary are deleted.
# @PURPOSE Delete all entries for a dictionary.
# @PRE dict_id exists.
# @POST All entries for the dictionary are deleted.
@staticmethod
def clear_entries(db: Session, dict_id: str) -> int:
with belief_scope("DictionaryManager.clear_entries"):
@@ -326,9 +328,9 @@ class DictionaryManager:
# endregion DictionaryManager.clear_entries
# region DictionaryManager.list_entries [TYPE Function]
# @PURPOSE: List entries for a dictionary with pagination.
# @PRE: dict_id exists.
# @POST: Returns (list of entries, total_count).
# @PURPOSE List entries for a dictionary with pagination.
# @PRE dict_id exists.
# @POST Returns (list of entries, total_count).
@staticmethod
def list_entries(
db: Session, dict_id: str, page: int = 1, page_size: int = 50,
@@ -351,10 +353,10 @@ class DictionaryManager:
# endregion DictionaryManager.list_entries
# region DictionaryManager.import_entries [TYPE Function]
# @PURPOSE: Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE: content is valid CSV or TSV. dict_id exists.
# @POST: Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT: Batch-inserts or updates DictionaryEntry rows.
# @PURPOSE Import entries from CSV/TSV content with duplicate detection and conflict resolution.
# @PRE content is valid CSV or TSV. dict_id exists.
# @POST Entries are created/updated/skipped per conflict mode. Returns result summary.
# @SIDE_EFFECT Batch-inserts or updates DictionaryEntry rows.
@staticmethod
def import_entries(
db: Session, dict_id: str, content: str,
@@ -504,9 +506,9 @@ class DictionaryManager:
# endregion DictionaryManager.import_entries
# region DictionaryManager.export_entries [TYPE Function]
# @PURPOSE: Export all entries as CSV string with language columns.
# @PRE: dict_id exists.
# @POST: Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
# @PURPOSE Export all entries as CSV string with language columns.
# @PRE dict_id exists.
# @POST Returns CSV string with header: source_term, target_term, source_language, target_language, context_notes, context_data, usage_notes.
@staticmethod
def export_entries(
db: Session, dict_id: str, delimiter: str = ",",
@@ -549,10 +551,10 @@ class DictionaryManager:
# endregion DictionaryManager.export_entries
# region DictionaryManager.migrate_old_entries [TYPE Function]
# @PURPOSE: Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE: db session is open.
# @POST: Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT: Updates DictionaryEntry rows in bulk.
# @PURPOSE Migrate existing single-language dictionary entries to populate source_language and target_language.
# @PRE db session is open.
# @POST Entries with "und" source_language or target_language are updated with inferred values.
# @SIDE_EFFECT Updates DictionaryEntry rows in bulk.
@staticmethod
def migrate_old_entries(db: Session) -> dict[str, int]:
with belief_scope("DictionaryManager.migrate_old_entries"):
@@ -601,12 +603,12 @@ class DictionaryManager:
# endregion DictionaryManager.migrate_old_entries
# region DictionaryManager.filter_for_batch [TYPE Function]
# @PURPOSE: Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# @PURPOSE Scan batch texts for case-insensitive, word-boundary-aware matches against all dictionaries attached to a job,
# optionally filtered by language pair. When row_context is provided, computes context-aware priority flags.
# @PRE: job_id exists and source_texts is a list of strings.
# @POST: Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# @PRE job_id exists and source_texts is a list of strings.
# @POST Returns list of matched entries with match info, sorted by priority across attached dictionaries.
# When row_context is given, each result includes a 'priority_match' boolean.
# @SIDE_EFFECT: Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
# @SIDE_EFFECT Queries TranslationJobDictionary, TerminologyDictionary, and DictionaryEntry tables.
@staticmethod
def filter_for_batch(
db: Session, source_texts: list[str], job_id: str,
@@ -738,10 +740,10 @@ class DictionaryManager:
# region DictionaryManager.submit_correction [TYPE Function]
# @PURPOSE: Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE: source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST: Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT: Creates/updates DictionaryEntry row.
# @PURPOSE Submit a term correction from a run result. Validates language match, detects conflicts.
# @PRE source_term, incorrect_target_term, corrected_target_term are non-empty. dict_id exists.
# @POST Entry created or updated; origin tracking populated. Returns action + conflict info.
# @SIDE_EFFECT Creates/updates DictionaryEntry row.
@staticmethod
def submit_correction(
db: Session,
@@ -868,10 +870,10 @@ class DictionaryManager:
# endregion DictionaryManager.submit_correction
# region DictionaryManager.submit_bulk_corrections [TYPE Function]
# @PURPOSE: Submit multiple term corrections atomically — all succeed or all fail.
# @PRE: corrections list is non-empty. dict_id exists.
# @POST: All corrections applied or none applied with conflict list.
# @SIDE_EFFECT: Creates/updates DictionaryEntry rows; commits once.
# @PURPOSE Submit multiple term corrections atomically — all succeed or all fail.
# @PRE corrections list is non-empty. dict_id exists.
# @POST All corrections applied or none applied with conflict list.
# @SIDE_EFFECT Creates/updates DictionaryEntry rows; commits once.
@staticmethod
def submit_bulk_corrections(
db: Session,
@@ -998,5 +1000,8 @@ class DictionaryManager:
# endregion DictionaryManager.submit_bulk_corrections
# #endregion DictionaryManager
# #endregion DictionaryManager
# #endregion DictionaryManagerModule

32
backend/src/plugins/translate/executor.py
View File

@@ -194,11 +194,12 @@ def _check_translation_cache(
# #endregion _check_translation_cache
# #region estimate_row_tokens [C:2] [TYPE Function] [SEMANTICS translate, token, estimation]
# #region estimate_row_tokens [C:4] [TYPE Function] [SEMANTICS translate, token, estimation]
# @BRIEF Estimate token count for a single source row including context fields.
# @PRE: source_text is a string.
# @POST: Returns estimated token count >= 1.
# @PRE source_text is a string.
# @POST Returns estimated token count >= 1.
# @RELATION DEPENDS_ON -> [_estimate_tokens_for_text]
def estimate_row_tokens(
source_text: str,
source_data: dict | None,
@@ -1745,30 +1746,47 @@ class TranslationExecutor:
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
# Reasoning-safe system prompt: always forbid chain-of-thought in output.
# Reasoning models (DeepSeek, QwQ, etc.) may leak reasoning into content
# when response_format=json_object is set. Explicit suppression in the
# system message prevents this regardless of the disable_reasoning flag.
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
"or explanation. Output ONLY valid JSON."
)
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
# Structured output — OpenRouter and Kilo also support response_format, but some
# upstream providers (e.g. StepFun) reject it. We try with response_format and
# fall back on 400 "structured_outputs is not supported".
# NOTE: Reasoning models (deepseek, qwen with reasoning, etc.) often break with
# response_format=json_object, producing reasoning text instead of JSON.
# When disable_reasoning is set, we remove response_format and rely on the
# system prompt to enforce JSON-only output.
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter/LiteLLM do NOT support disabling reasoning (returns 400)
if disable_reasoning:
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload.pop("response_format", None)
# response_format already skipped above when disable_reasoning is True
# Use caller-provided max_tokens instead of hardcoded 8192
payload["max_tokens"] = max_tokens
payload["messages"][0] = {"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."}
logger.reason(
f"LLM request url={base_url} model={payload.get('model')} "

27
backend/src/plugins/translate/metrics.py
View File

@@ -1,14 +1,15 @@
# #region TranslationMetrics [C:3] [TYPE Module] [SEMANTICS sqlalchemy, translate, metrics, job, statistics]
# @BRIEF Aggregate translation metrics from live TranslationEvent + MetricSnapshot for per-job reporting.
# @LAYER: Domain
# @RELATION DEPENDS_ON -> [TranslationEvent:Class]
# @RELATION DEPENDS_ON -> [MetricSnapshot:Class]
# @RELATION DEPENDS_ON -> [TranslationRun:Class]
# @LAYER Domain
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE: Database session is open.
# @POST: Metrics are aggregated and returned; no side effects.
# @RATIONALE: Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED: Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @RELATION DEPENDS_ON -> [TranslationSchedule:Class]
# @PRE Database session is open.
# @POST Metrics are aggregated and returned; no side effects.
# @RATIONALE Live events (<90 days) + MetricSnapshot (>=90 days) fusion for complete picture.
# @REJECTED Querying all events for all time — would be slow; MetricSnapshot provides historical summary.
# @COMPLEXITY 4
from datetime import UTC, datetime
from typing import Any
@@ -34,9 +35,9 @@ class TranslationMetrics:
self.db = db
# region get_job_metrics [TYPE Function]
# @PURPOSE: Get aggregated metrics for a specific job.
# @PRE: job_id exists.
# @POST: Returns dict with metrics from events + latest snapshot.
# @PURPOSE Get aggregated metrics for a specific job.
# @PRE job_id exists.
# @POST Returns dict with metrics from events + latest snapshot.
def get_job_metrics(self, job_id: str) -> dict[str, Any]:
with belief_scope("TranslationMetrics.get_job_metrics"):
# Run counts from TranslationRun
@@ -174,8 +175,8 @@ class TranslationMetrics:
# endregion get_job_metrics
# region get_all_metrics [TYPE Function]
# @PURPOSE: Get aggregated metrics for all jobs.
# @POST: Returns list of per-job metrics.
# @PURPOSE Get aggregated metrics for all jobs.
# @POST Returns list of per-job metrics.
def get_all_metrics(self) -> list[dict[str, Any]]:
with belief_scope("TranslationMetrics.get_all_metrics"):
job_ids = (

10
backend/src/plugins/translate/orchestrator.py
View File

@@ -238,6 +238,15 @@ class TranslationOrchestrator:
"run_id": run.id,
"error": str(e),
})
# Rollback the session if it's in a broken state (e.g. after a
# failed flush from a previous exception in executor.execute_run).
# Without this, subsequent flush()/commit() calls raise
# "This Session's transaction has been rolled back".
try:
self.db.rollback()
except Exception:
pass
run = self.db.merge(run)
run.status = "FAILED"
run.error_message = f"Translation execution failed: {e}"
run.completed_at = datetime.now(UTC)
@@ -887,6 +896,7 @@ class TranslationOrchestrator:
"id": run.id,
"job_id": run.job_id,
"status": run.status,
"trigger_type": run.trigger_type,
"started_at": run.started_at.isoformat() if run.started_at else None,
"completed_at": run.completed_at.isoformat() if run.completed_at else None,
"error_message": run.error_message,

6
backend/src/plugins/translate/plugin.py
View File

@@ -1,9 +1,9 @@
# #region TranslatePlugin [C:2] [TYPE Module] [SEMANTICS clickhouse, translate, sql, dashboard, dialect]
# @BRIEF TranslatePlugin skeleton for cross-dialect SQL and dashboard translation.
# @LAYER: Domain
# @LAYER Domain
# @RELATION INHERITS -> [PluginBase]
# @RATIONALE: Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED: Extending LLMAnalysisPlugin would conflate two distinct feature domains.
# @RATIONALE Separate plugin avoids bloating LLMAnalysisPlugin beyond fractal limit (<400 lines).
# @REJECTED Extending LLMAnalysisPlugin would conflate two distinct feature domains.
from typing import Any

33
backend/src/plugins/translate/preview.py
View File

@@ -41,7 +41,7 @@ from ._token_budget import DEFAULT_CONTEXT_WINDOW, estimate_token_budget
from .dictionary import DictionaryManager
# #region DEFAULT_EXECUTION_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for batch LLM translation execution (no context columns — faster).
# Supports both single-language and multi-language modes via {target_languages} placeholder.
DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
"Translate the following database content from {source_language} to the following language(s): {target_languages}.\n\n"
@@ -63,9 +63,9 @@ DEFAULT_EXECUTION_PROMPT_TEMPLATE: str = (
)
# #endregion DEFAULT_EXECUTION_PROMPT_TEMPLATE
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
# @BRIEF Default prompt template for LLM translation preview.
# #region DEFAULT_PREVIEW_PROMPT_TEMPLATE [TYPE Constant]
DEFAULT_PREVIEW_PROMPT_TEMPLATE: str = (
"Translate the following database content.\n\n"
"Source dialect: {source_dialect}\n"
@@ -1038,19 +1038,37 @@ class TranslationPreview:
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
# Reasoning-safe system prompt: always forbid chain-of-thought in output.
# Reasoning models (DeepSeek, QwQ, etc.) may leak reasoning into content
# when response_format=json_object is set. Explicit suppression in the
# system message prevents this regardless of the disable_reasoning flag.
system_content = (
"You are a database content translation assistant. "
"Translate the provided text accurately, preserving data semantics. "
"Respond directly with ONLY the JSON result. "
"Do NOT include any reasoning, thinking, chain-of-thought, analysis, "
"or explanation. Output ONLY valid JSON."
)
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics."},
{"role": "system", "content": system_content},
{"role": "user", "content": prompt},
],
"temperature": 0.1,
"max_tokens": max_tokens,
}
# Structured output — Kilo gateway supports response_format, but upstream providers
# (e.g. StepFun) may reject it. We try with response_format and fall back on 400.
# NOTE: Reasoning models (deepseek variants, qwen with reasoning) often break with
# response_format=json_object, producing reasoning text instead of JSON.
# When disable_reasoning is set, we skip response_format and rely on the
# system prompt to enforce JSON-only output.
if provider_type in ("openai", "openai_compatible", "kilo", "openrouter", "litellm"):
payload["response_format"] = {"type": "json_object"}
if not disable_reasoning:
payload["response_format"] = {"type": "json_object"}
# Suppress Chain of Thought reasoning to save output tokens
# NOTE: Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
@@ -1058,13 +1076,10 @@ class TranslationPreview:
# Kilo/OpenRouter/LiteLLM reject reasoning_effort — only use for native OpenAI-compatible
if provider_type not in ("kilo", "openrouter", "litellm"):
payload["reasoning_effort"] = "none"
payload.pop("response_format", None) # JSON mode triggers reasoning on some models
# response_format already skipped above when disable_reasoning is True
# Use caller-provided max_tokens instead of hardcoded 8192
# This ensures multi-language batches with large output get enough token budget.
payload["max_tokens"] = max_tokens
# Universal instruction — all models understand "respond directly without reasoning"
system_content = "You are a database content translation assistant. Translate the provided text accurately, preserving data semantics. Respond directly with ONLY the JSON result. Do NOT include any reasoning, thinking, chain-of-thought, analysis, or explanation. Output ONLY valid JSON."
payload["messages"][0] = {"role": "system", "content": system_content}
logger.reason(
f"LLM request url={base_url} model={payload.get('model')} "

7
backend/src/plugins/translate/prompt_builder.py
View File

@@ -1,9 +1,10 @@
# #region ContextAwarePromptBuilder [C:2] [TYPE Module] [SEMANTICS translate, prompt, context, dictionary]
# @BRIEF Pure-function prompt builder that enhances dictionary entries with context annotations.
# @LAYER: Domain
# @LAYER Domain
# @RELATION DEPENDS_ON -> [DictionaryEntry:Class]
# @RATIONALE: Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED: Embedding context inline in the executor would make it untestable without mocking DB.
# @RATIONALE Pure functions only — no I/O, no DB access. Separated from executor for testability.
# @REJECTED Embedding context inline in the executor would make it untestable without mocking DB.
#
# Typical workflow:
# 1. Call build_context_entries(dictionary_entries, row_context) to get annotated, prioritized entries

12
backend/src/plugins/translate/scheduler.py
View File

@@ -240,12 +240,14 @@ class TranslationScheduler:
# #endregion TranslationScheduler
# #region execute_scheduled_translation [TYPE Function]
# #region execute_scheduled_translation [C:4] [TYPE Function]
# @BRIEF APScheduler job handler — runs scheduled translation with concurrency check and new-key-only fallback.
# @PRE: schedule_id is valid.
# @POST: Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT: DB writes; LLM calls; Superset API calls.
# @RATIONALE: New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# @PRE schedule_id is valid.
# @POST Translation run created and executed if no concurrent run exists.
# @SIDE_EFFECT DB writes; LLM calls; Superset API calls.
# @RATIONALE New-key-only mode compares keys against most recent succeeded run; baseline_expired fallback does full.
# @COMPLEXITY 4
def execute_scheduled_translation(
schedule_id: str,
job_id: str,

16
backend/src/plugins/translate/service.py
View File

@@ -38,7 +38,7 @@ SUPPORTED_DIALECTS = {
}
# #region get_dialect_from_database [TYPE Function]
# #region get_dialect_from_database [C:4] [TYPE Function]
# @BRIEF Extract normalized dialect string from a Superset database record.
# @PRE: database_record is a dict from Superset API.
# @POST: Returns normalized dialect string or raises ValueError if unsupported.
@@ -84,7 +84,7 @@ def get_dialect_from_database(database_record: dict[str, Any]) -> str:
# #endregion get_dialect_from_database
# #region fetch_datasource_metadata [TYPE Function]
# #region fetch_datasource_metadata [C:4] [TYPE Function]
# @BRIEF Fetch datasource columns and database dialect from Superset.
# @PRE: dataset_id is a valid Superset dataset ID and environment has valid credentials.
# @POST: Returns (columns_list, dialect_string) or raises on failure.
@@ -141,7 +141,7 @@ def fetch_datasource_metadata(
# #endregion fetch_datasource_metadata
# #region detect_virtual_columns [TYPE Function]
# #region detect_virtual_columns [C:4] [TYPE Function]
# @BRIEF Identify virtual (calculated) columns from column metadata.
# @PRE: columns is a list of dicts with 'is_physical' key.
# @POST: Returns list of virtual column names.
@@ -519,11 +519,13 @@ def job_to_response(job: TranslationJob, dict_ids: list[str] | None = None) -> T
# #endregion job_to_response
# #region DatasourceColumnsService [TYPE Function]
# #region DatasourceColumnsService [C:4] [TYPE Function]
# @BRIEF Fetch datasource column metadata from Superset and return structured response.
# @PRE: datasource_id is a valid Superset dataset ID.
# @POST: Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT: Queries Superset API for dataset detail and database info.
# @PRE datasource_id is a valid Superset dataset ID.
# @POST Returns DatasourceColumnsResponse with column metadata and database dialect.
# @SIDE_EFFECT Queries Superset API for dataset detail and database info.
# @COMPLEXITY 4
def get_datasource_columns(
datasource_id: int,
env_id: str,

10
backend/src/plugins/translate/sql_generator.py
View File

@@ -58,7 +58,7 @@ def _normalize_timestamp_value(value: Any) -> str | None:
# #endregion _normalize_timestamp_value
# #region _quote_identifier [TYPE Function]
# #region _quote_identifier [C:4] [TYPE Function]
# @BRIEF Quote an identifier per dialect rules. PostgreSQL uses double quotes; ClickHouse uses backticks.
# @PRE: identifier is a non-empty string.
# @POST: Returns safely quoted identifier.
@@ -159,10 +159,12 @@ def generate_insert_sql(
# #endregion generate_insert_sql
# #region generate_upsert_sql [TYPE Function]
# #region generate_upsert_sql [C:4] [TYPE Function]
# @BRIEF Generate PostgreSQL dialect UPSERT SQL with ON CONFLICT DO UPDATE.
# @PRE: dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST: Returns UPSERT SQL string or raises ValueError.
# @PRE dialect is postgresql-compatible. target_table, columns, key_columns are non-empty.
# @POST Returns UPSERT SQL string or raises ValueError.
# @COMPLEXITY 4
def generate_upsert_sql(
target_schema: str | None,
target_table: str,

72
backend/src/plugins/translate/superset_executor.py
View File

@@ -1,13 +1,13 @@
# #region SupersetSqlLabExecutor [C:4] [TYPE Module] [SEMANTICS translate, superset, sqllab, execute, query]
# @BRIEF Submit SQL to Superset SQL Lab API and poll execution status.
# @LAYER: Infra
# @RELATION DEPENDS_ON -> [SupersetClient]
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE: Valid Superset environment configuration and authenticated client.
# @POST: SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT: Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE: Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED: Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
# @RELATION DEPENDS_ON -> [ConfigManager]
# @PRE Valid Superset environment configuration and authenticated client.
# @POST SQL is submitted to Superset SQL Lab; execution reference is returned.
# @SIDE_EFFECT Makes HTTP calls to Superset /api/v1/sqllab/execute/; polls status.
# @RATIONALE Direct SQL Lab API submission provides audit trail and execution monitoring within Superset.
# @REJECTED Direct database connection bypass — would skip Superset's SQL Lab audit and RBAC.
import json
import time
@@ -21,9 +21,9 @@ from ...core.superset_client import SupersetClient
# #region SupersetSqlLabExecutor [C:4] [TYPE Class]
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE: Valid environment ID and ConfigManager.
# @POST: SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
# @PRE Valid environment ID and ConfigManager.
# @POST SQL submitted to Superset; execution reference recorded.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/; polls GET for status.
class SupersetSqlLabExecutor:
def __init__(self, config_manager: ConfigManager, env_id: str):
@@ -34,10 +34,10 @@ class SupersetSqlLabExecutor:
self._database_backend: str | None = None
# region _get_client [TYPE Function]
# @PURPOSE: Lazy-initialize SupersetClient for the configured environment.
# @PRE: env_id must correspond to a valid environment config.
# @POST: Returns authenticated SupersetClient.
# @SIDE_EFFECT: Authenticates against Superset API.
# @PURPOSE Lazy-initialize SupersetClient for the configured environment.
# @PRE env_id must correspond to a valid environment config.
# @POST Returns authenticated SupersetClient.
# @SIDE_EFFECT Authenticates against Superset API.
def _get_client(self) -> SupersetClient:
with belief_scope("SupersetSqlLabExecutor._get_client"):
if self._client is not None:
@@ -53,10 +53,10 @@ class SupersetSqlLabExecutor:
# endregion _get_client
# region resolve_database_id [TYPE Function]
# @PURPOSE: Resolve the target database ID from the environment and fetch its backend engine.
# @PRE: database_name or database_id should be known.
# @POST: Returns database_id integer or raises ValueError. Also sets self._database_backend.
# @SIDE_EFFECT: Fetches databases list from Superset; optionally fetches single DB for backend.
# @PURPOSE Resolve the target database ID from the environment and fetch its backend engine.
# @PRE database_name or database_id should be known.
# @POST Returns database_id integer or raises ValueError. Also sets self._database_backend.
# @SIDE_EFFECT Fetches databases list from Superset; optionally fetches single DB for backend.
def resolve_database_id(
self,
database_name: str | None = None,
@@ -121,17 +121,17 @@ class SupersetSqlLabExecutor:
# endregion resolve_database_id
# region get_database_backend [TYPE Function]
# @PURPOSE: Return the cached database backend/engine string.
# @POST: Returns backend string or None if not yet resolved.
# @PURPOSE Return the cached database backend/engine string.
# @POST Returns backend string or None if not yet resolved.
def get_database_backend(self) -> str | None:
return self._database_backend
# endregion get_database_backend
# region execute_sql [TYPE Function]
# @PURPOSE: Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE: sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST: Returns execution result dict with query_id and status.
# @SIDE_EFFECT: Makes HTTP POST to /api/v1/sqllab/execute/.
# @PURPOSE Submit SQL to Superset SQL Lab and return execution tracking info.
# @PRE sql is valid SQL string. database_id is a valid Superset DB ID.
# @POST Returns execution result dict with query_id and status.
# @SIDE_EFFECT Makes HTTP POST to /api/v1/sqllab/execute/.
def execute_sql(
self,
sql: str,
@@ -236,10 +236,10 @@ class SupersetSqlLabExecutor:
# endregion execute_sql
# region poll_execution_status [TYPE Function]
# @PURPOSE: Poll Superset for SQL execution status until completion or timeout.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns final execution status dict.
# @SIDE_EFFECT: Makes HTTP GET requests to Superset.
# @PURPOSE Poll Superset for SQL execution status until completion or timeout.
# @PRE query_id is a valid Superset query ID.
# @POST Returns final execution status dict.
# @SIDE_EFFECT Makes HTTP GET requests to Superset.
def poll_execution_status(
self,
query_id: str,
@@ -329,10 +329,10 @@ class SupersetSqlLabExecutor:
# endregion poll_execution_status
# region execute_and_poll [TYPE Function]
# @PURPOSE: Execute SQL and wait for completion. One-shot convenience method.
# @PRE: sql is valid SQL.
# @POST: Returns final execution result.
# @SIDE_EFFECT: Makes HTTP calls to Superset API.
# @PURPOSE Execute SQL and wait for completion. One-shot convenience method.
# @PRE sql is valid SQL.
# @POST Returns final execution result.
# @SIDE_EFFECT Makes HTTP calls to Superset API.
def execute_and_poll(
self,
sql: str,
@@ -384,10 +384,10 @@ class SupersetSqlLabExecutor:
# endregion execute_and_poll
# region get_query_results [TYPE Function]
# @PURPOSE: Fetch the results of a completed query.
# @PRE: query_id is a valid Superset query ID.
# @POST: Returns query results if available.
# @SIDE_EFFECT: Makes HTTP GET to Superset.
# @PURPOSE Fetch the results of a completed query.
# @PRE query_id is a valid Superset query ID.
# @POST Returns query results if available.
# @SIDE_EFFECT Makes HTTP GET to Superset.
def get_query_results(self, query_id: str) -> dict[str, Any]:
with belief_scope("SupersetSqlLabExecutor.get_query_results"):
client = self._get_client()

4
backend/src/schemas/translate.py
View File

@@ -1,6 +1,6 @@
# #region TranslateSchemas [C:2] [TYPE Module] [SEMANTICS pydantic, translate, schema, translate-job-create]
# #region TranslateSchemas [C:3] [TYPE Module] [SEMANTICS pydantic, translate, schema, translate-job-create]
# @BRIEF Pydantic v2 schemas for translation API request/response serialization.
# @LAYER: API
# @LAYER API
# @RELATION DEPENDS_ON -> pydantic
from datetime import datetime

285
backend/src/services/git/_base.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceBase [C:3] [TYPE Module] [SEMANTICS git, repository, clone, base, mixin]
# #region GitServiceBase [C:4] [TYPE Module] [SEMANTICS git, repository, clone, base, mixin, lock, http]
# @LAYER: Infra
# @BRIEF Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), and identity configuration.
# @BRIEF Core GitService base class — initialization, path resolution, repo lifecycle (init/delete/get), identity configuration, concurrent locking, and shared HTTP client pool.
# @RELATION INHERITED_BY -> [GitService]
# @RELATION DEPENDS_ON -> [SessionLocal]
# @RELATION DEPENDS_ON -> [AppConfigRecord]
@@ -9,8 +9,12 @@
import os
import re
import shutil
import threading
from contextlib import contextmanager
from pathlib import Path
from urllib.parse import quote
import httpx
from fastapi import HTTPException
from git import Repo
from git.exc import InvalidGitRepositoryError, NoSuchPathError
@@ -21,14 +25,16 @@ from src.models.config import AppConfigRecord
from src.models.git import GitRepository
# #region GitServiceBase [C:3] [TYPE Class]
# @BRIEF Base class for GitService providing initialization, path resolution, repository lifecycle, and identity.
# #region GitServiceBase [C:4] [TYPE Class]
# @BRIEF Base class for GitService providing initialization, path resolution, repository lifecycle, identity, concurrent locking, and shared HTTP client.
# @PRE base_path is a valid string path.
# @POST GitService is initialized; base_path directory exists; shared AsyncClient ready; lock registry empty.
# @SIDE_EFFECT Creates HTTP connection pool (max_keepalive=20, max_connections=100).
class GitServiceBase:
# region GitService_init [TYPE Function]
# @PURPOSE: Initializes the GitService with a base path for repositories.
# @PARAM: base_path (str) - Root directory for all Git clones.
# region GitService_init [C:4] [TYPE Function]
# @PURPOSE: Initializes the GitService with a base path, concurrent access locks, and shared HTTP client.
# @PRE: base_path is a valid string path.
# @POST: GitService is initialized; base_path directory exists.
# @POST: GitService is initialized; base_path directory exists; _lock_registry, _http_client ready.
def __init__(self, base_path: str = "git_repos"):
with belief_scope("GitService.__init__"):
backend_root = Path(__file__).parents[3]
@@ -36,8 +42,65 @@ class GitServiceBase:
self._uses_default_base_path = base_path == "git_repos"
self.base_path = self._resolve_base_path(base_path)
self._ensure_base_path_exists()
# Fix 3: Per-dashboard reentrant lock registry for concurrent access protection
self._lock_registry: dict[int, threading.RLock] = {}
self._reg_lock = threading.Lock()
# Fix: close() race condition protection
self._closed = False
self._close_lock = threading.Lock()
# Fix 5: Shared httpx.AsyncClient with connection pooling
self._http_client = httpx.AsyncClient(
timeout=30.0,
limits=httpx.Limits(max_keepalive_connections=20, max_connections=100),
)
# endregion GitService_init
# region _get_lock [C:2] [TYPE Function] [SEMANTICS lock,concurrency]
# @BRIEF Get or create a per-dashboard reentrant lock for thread-safe GitPython access.
# @PRE: dashboard_id is a valid integer.
# @POST: Returns a threading.RLock unique to the given dashboard_id.
def _get_lock(self, dashboard_id: int) -> threading.RLock:
with self._reg_lock:
if dashboard_id not in self._lock_registry:
self._lock_registry[dashboard_id] = threading.RLock()
return self._lock_registry[dashboard_id]
# endregion _get_lock
# region _locked [C:2] [TYPE Function] [SEMANTICS lock,context,concurrency]
# @BRIEF Context manager that acquires the per-dashboard reentrant lock for the duration of the block.
# @PRE: dashboard_id is a valid integer.
# @POST: Lock is acquired on enter, released on exit.
@contextmanager
def _locked(self, dashboard_id: int):
if self._closed:
raise RuntimeError("GitService is closed")
lock = self._get_lock(dashboard_id)
lock.acquire()
try:
yield
finally:
lock.release()
# endregion _locked
# region _clone_with_auth [C:2] [TYPE Function] [SEMANTICS git,clone,auth,security]
# @BRIEF Clone repository with PAT and immediately strip credentials from origin remote URL.
# @PRE: remote_url is a valid Git URL; pat is provided when required; repo_path is writable.
# @POST: Repo cloned at repo_path; origin remote URL has no embedded PAT.
# @SIDE_EFFECT Clones remote repository to local filesystem.
def _clone_with_auth(self, remote_url: str, repo_path: str, pat: str) -> Repo:
auth_url = remote_url
if pat and "://" in remote_url:
proto, rest = remote_url.split("://", 1)
auth_url = f"{proto}://oauth2:{quote(pat, safe='')}@{rest}"
repo = Repo.clone_from(auth_url, repo_path)
# Strip PAT from origin URL to prevent credential leakage in .git/config, logs, ps aux
repo.git.remote("set-url", "origin", remote_url)
return repo
# endregion _clone_with_auth
# region _ensure_base_path_exists [TYPE Function]
# @PURPOSE: Ensure the repositories root directory exists and is a directory.
# @PRE: self.base_path is resolved to filesystem path.
@@ -194,124 +257,138 @@ class GitServiceBase:
return target_path
# endregion _get_repo_path
# region init_repo [TYPE Function]
# @PURPOSE: Initialize or clone a repository for a dashboard.
# region init_repo [C:4] [TYPE Function] [SEMANTICS git,clone,init,lock]
# @PURPOSE: Initialize or clone a repository for a dashboard with concurrent access protection.
# @PARAM: dashboard_id (int), remote_url (str), pat (str), repo_key (Optional[str]).
# @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided.
# @POST: Repository is cloned or opened at the local path.
# @POST: Repository is cloned or opened at the local path. Origin remote URL has no embedded PAT.
# @SIDE_EFFECT Clones remote repository; modifies local filesystem; protected by per-dashboard lock.
# @RETURN: Repo
def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: str | None = None) -> Repo:
with belief_scope("GitService.init_repo"):
self._ensure_base_path_exists()
repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id))
Path(repo_path).parent.mkdir(parents=True, exist_ok=True)
if pat and "://" in remote_url:
proto, rest = remote_url.split("://", 1)
auth_url = f"{proto}://oauth2:{pat}@{rest}"
else:
auth_url = remote_url
if os.path.exists(repo_path):
logger.reason(f"Opening existing repo at {repo_path}", extra={"src": "init_repo"})
try:
repo = Repo(repo_path)
except (InvalidGitRepositoryError, NoSuchPathError):
logger.reason(f"Existing path is not a Git repository, recreating: {repo_path}", extra={"src": "init_repo"})
stale_path = Path(repo_path)
if stale_path.exists():
shutil.rmtree(stale_path, ignore_errors=True)
def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: str | None = None, default_branch: str | None = None) -> Repo:
with self._locked(dashboard_id):
with belief_scope("GitService.init_repo"):
self._ensure_base_path_exists()
repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id))
Path(repo_path).parent.mkdir(parents=True, exist_ok=True)
if os.path.exists(repo_path):
logger.reason(f"Opening existing repo at {repo_path}", extra={"src": "init_repo"})
try:
repo = Repo(repo_path)
except (InvalidGitRepositoryError, NoSuchPathError):
logger.reason(f"Existing path is not a Git repository, recreating: {repo_path}", extra={"src": "init_repo"})
stale_path = Path(repo_path)
if stale_path.exists():
try:
stale_path.unlink()
except Exception:
pass
repo = Repo.clone_from(auth_url, repo_path)
shutil.rmtree(stale_path, ignore_errors=True)
if stale_path.exists():
try:
stale_path.unlink()
except Exception:
pass
repo = self._clone_with_auth(remote_url, repo_path, pat)
self._ensure_gitflow_branches(repo, dashboard_id)
return repo
logger.reason(f"Cloning {remote_url} to {repo_path}", extra={"src": "init_repo"})
repo = self._clone_with_auth(remote_url, repo_path, pat)
self._ensure_gitflow_branches(repo, dashboard_id)
return repo
logger.reason(f"Cloning {remote_url} to {repo_path}", extra={"src": "init_repo"})
repo = Repo.clone_from(auth_url, repo_path)
self._ensure_gitflow_branches(repo, dashboard_id)
return repo
# endregion init_repo
# region delete_repo [TYPE Function]
# @PURPOSE: Remove local repository and DB binding for a dashboard.
# region delete_repo [C:4] [TYPE Function] [SEMANTICS git,delete,lock]
# @PURPOSE: Remove local repository and DB binding for a dashboard with concurrent access protection.
# @PRE: dashboard_id is a valid integer.
# @POST: Local path is deleted when present and GitRepository row is removed.
# @SIDE_EFFECT Deletes filesystem directory and DB record; protected by per-dashboard lock.
def delete_repo(self, dashboard_id: int) -> None:
with belief_scope("GitService.delete_repo"):
repo_path = self._get_repo_path(dashboard_id)
removed_files = False
if os.path.exists(repo_path):
if os.path.isdir(repo_path):
shutil.rmtree(repo_path)
else:
os.remove(repo_path)
removed_files = True
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
session.delete(db_repo)
session.commit()
return
if removed_files:
return
raise HTTPException(
status_code=404,
detail=f"Repository for dashboard {dashboard_id} not found",
)
except HTTPException:
session.rollback()
raise
except Exception as e:
session.rollback()
logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to delete repository: {e!s}")
finally:
session.close()
with self._locked(dashboard_id):
with belief_scope("GitService.delete_repo"):
repo_path = self._get_repo_path(dashboard_id)
removed_files = False
if os.path.exists(repo_path):
if os.path.isdir(repo_path):
shutil.rmtree(repo_path)
else:
os.remove(repo_path)
removed_files = True
session = SessionLocal()
try:
db_repo = (
session.query(GitRepository)
.filter(GitRepository.dashboard_id == int(dashboard_id))
.first()
)
if db_repo:
session.delete(db_repo)
session.commit()
return
if removed_files:
return
raise HTTPException(
status_code=404,
detail=f"Repository for dashboard {dashboard_id} not found",
)
except HTTPException:
session.rollback()
raise
except Exception as e:
session.rollback()
logger.error(f"[delete_repo][Coherence:Failed] Failed to delete repository for dashboard {dashboard_id}: {e}")
raise HTTPException(status_code=500, detail=f"Failed to delete repository: {e!s}")
finally:
session.close()
# endregion delete_repo
# region get_repo [TYPE Function]
# @PURPOSE: Get Repo object for a dashboard.
# region get_repo [C:4] [TYPE Function] [SEMANTICS git,open,lock]
# @PURPOSE: Get Repo object for a dashboard with concurrent access protection.
# @PRE: Repository must exist on disk for the given dashboard_id.
# @POST: Returns a GitPython Repo instance for the dashboard.
# @RETURN: Repo
def get_repo(self, dashboard_id: int) -> Repo:
with belief_scope("GitService.get_repo"):
repo_path = self._get_repo_path(dashboard_id)
if not os.path.exists(repo_path):
logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist")
raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found")
try:
return Repo(repo_path)
except Exception as e:
logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}")
raise HTTPException(status_code=500, detail="Failed to open local Git repository")
with self._locked(dashboard_id):
with belief_scope("GitService.get_repo"):
repo_path = self._get_repo_path(dashboard_id)
if not os.path.exists(repo_path):
logger.error(f"[get_repo][Coherence:Failed] Repository for dashboard {dashboard_id} does not exist")
raise HTTPException(status_code=404, detail=f"Repository for dashboard {dashboard_id} not found")
try:
return Repo(repo_path)
except Exception as e:
logger.error(f"[get_repo][Coherence:Failed] Failed to open repository at {repo_path}: {e}")
raise HTTPException(status_code=500, detail="Failed to open local Git repository")
# endregion get_repo
# region configure_identity [TYPE Function]
# @PURPOSE: Configure repository-local Git committer identity for user-scoped operations.
# region configure_identity [C:4] [TYPE Function] [SEMANTICS git,identity,lock]
# @PURPOSE: Configure repository-local Git committer identity with concurrent access protection.
# @PRE: dashboard_id repository exists; git_username/git_email may be empty.
# @POST: Repository config has user.name and user.email when both identity values are provided.
# @SIDE_EFFECT Writes to repository git config; protected by per-dashboard lock.
def configure_identity(self, dashboard_id: int, git_username: str | None, git_email: str | None) -> None:
with belief_scope("GitService.configure_identity"):
normalized_username = str(git_username or "").strip()
normalized_email = str(git_email or "").strip()
if not normalized_username or not normalized_email:
return
repo = self.get_repo(dashboard_id)
try:
with repo.config_writer(config_level="repository") as config_writer:
config_writer.set_value("user", "name", normalized_username)
config_writer.set_value("user", "email", normalized_email)
logger.reason(f"Applied repository-local git identity for dashboard {dashboard_id}", extra={"src": "configure_identity"})
except Exception as e:
logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}")
raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {e!s}")
with self._locked(dashboard_id):
with belief_scope("GitService.configure_identity"):
normalized_username = str(git_username or "").strip()
normalized_email = str(git_email or "").strip()
if not normalized_username or not normalized_email:
return
repo = self.get_repo(dashboard_id)
try:
with repo.config_writer(config_level="repository") as config_writer:
config_writer.set_value("user", "name", normalized_username)
config_writer.set_value("user", "email", normalized_email)
logger.reason(f"Applied repository-local git identity for dashboard {dashboard_id}", extra={"src": "configure_identity"})
except Exception as e:
logger.error(f"[configure_identity][Coherence:Failed] Failed to configure git identity: {e}")
raise HTTPException(status_code=500, detail=f"Failed to configure git identity: {e!s}")
# endregion configure_identity
# region close [C:2] [TYPE Function] [SEMANTICS http,cleanup,shutdown]
# @BRIEF Gracefully close the shared HTTP client (connection pool).
# @PRE: _http_client is initialized.
# @POST: HTTP connections closed.
async def close(self):
with self._close_lock:
if self._closed:
return
self._closed = True
await self._http_client.aclose()
# endregion close
# #endregion GitServiceBase
# #endregion GitServiceBase

119
backend/src/services/git/_branch.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceBranchMixin [C:3] [TYPE Module] [SEMANTICS git, branch, checkout, list, namespace]
# #region GitServiceBranchMixin [C:4] [TYPE Module] [SEMANTICS git, branch, checkout, list, namespace, lock]
# @LAYER: Infra
# @BRIEF Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes.
# @BRIEF Branch and commit operations for GitService — gitflow branches, list/create/checkout branches, commit changes (all concurrent-safe via per-dashboard locks).
# @RELATION USED_BY -> [GitService]
import os
@@ -15,10 +15,11 @@ from src.core.logger import belief_scope, logger
# #region GitServiceBranchMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing branch and commit operations for GitService.
class GitServiceBranchMixin:
# region _ensure_gitflow_branches [TYPE Function]
# region _ensure_gitflow_branches [C:4] [TYPE Function] [SEMANTICS git,gitflow,branch,lock]
# @PURPOSE: Ensure standard GitFlow branches (main/dev/preprod) exist locally and on origin.
# @PRE: repo is a valid GitPython Repo instance.
# @POST: main, dev, preprod are available in local repository and pushed to origin when available.
# Active branch unchanged (no spurious checkout).
def _ensure_gitflow_branches(self, repo: Repo, dashboard_id: int) -> None:
with belief_scope("GitService._ensure_gitflow_branches"):
required_branches = ["main", "dev", "preprod"]
@@ -79,62 +80,74 @@ class GitServiceBranchMixin:
status_code=500,
detail=f"Failed to create default branch '{branch_name}' on remote: {e!s}",
)
# Fix 6: Only checkout if not already on dev
try:
repo.git.checkout("dev")
logger.reason(f"Checked out default branch dev for dashboard {dashboard_id}", extra={"src": "_ensure_gitflow_branches"})
except Exception as e:
logger.reason(f"Could not checkout dev branch for dashboard {dashboard_id}: {e}", extra={"src": "_ensure_gitflow_branches"})
current_branch = repo.active_branch.name
except Exception:
current_branch = None
if current_branch != "dev":
try:
repo.git.checkout("dev")
logger.reason(f"Checked out default branch dev for dashboard {dashboard_id}", extra={"src": "_ensure_gitflow_branches"})
except Exception as e:
logger.reason(f"Could not checkout dev branch for dashboard {dashboard_id}: {e}", extra={"src": "_ensure_gitflow_branches"})
# endregion _ensure_gitflow_branches
# region list_branches [TYPE Function]
# @PURPOSE: List all branches for a dashboard's repository.
# region list_branches [C:4] [TYPE Function] [SEMANTICS git,branch,list,lock]
# @PURPOSE: List all branches (excluding tags) for a dashboard's repository, concurrent-safe.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a list of branch metadata dictionaries.
# @POST: Returns a list of branch metadata dictionaries (no tag refs).
# @RETURN: List[dict]
def list_branches(self, dashboard_id: int) -> list[dict]:
with belief_scope("GitService.list_branches"):
repo = self.get_repo(dashboard_id)
logger.reason(f"Listing branches for {dashboard_id}. Refs: {repo.refs}", extra={"src": "list_branches"})
branches = []
for ref in repo.refs:
with self._locked(dashboard_id):
with belief_scope("GitService.list_branches"):
repo = self.get_repo(dashboard_id)
logger.reason(f"Listing branches for {dashboard_id}. Refs: {repo.refs}", extra={"src": "list_branches"})
branches = []
for ref in repo.refs:
try:
# Fix 8: Skip tag refs
ref_name = str(ref.name)
if ref_name.startswith('refs/tags/'):
continue
name = ref_name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')
if any(b['name'] == name for b in branches):
continue
branches.append({
"name": name,
"commit_hash": ref.commit.hexsha if hasattr(ref, 'commit') else "0000000",
"is_remote": ref.is_remote() if hasattr(ref, 'is_remote') else False,
"last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()
})
except Exception as e:
logger.reason(f"Skipping ref {ref}: {e}", extra={"src": "list_branches"})
try:
name = ref.name.replace('refs/heads/', '').replace('refs/remotes/origin/', '')
if any(b['name'] == name for b in branches):
continue
branches.append({
"name": name,
"commit_hash": ref.commit.hexsha if hasattr(ref, 'commit') else "0000000",
"is_remote": ref.is_remote() if hasattr(ref, 'is_remote') else False,
"last_updated": datetime.fromtimestamp(ref.commit.committed_date) if hasattr(ref, 'commit') else datetime.utcnow()
})
active_name = repo.active_branch.name
if not any(b['name'] == active_name for b in branches):
branches.append({
"name": active_name, "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
except Exception as e:
logger.reason(f"Skipping ref {ref}: {e}", extra={"src": "list_branches"})
try:
active_name = repo.active_branch.name
if not any(b['name'] == active_name for b in branches):
branches.append({
"name": active_name, "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
except Exception as e:
logger.reason(f"Could not determine active branch: {e}", extra={"src": "list_branches"})
if not branches:
branches.append({
"name": "dev", "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
return branches
logger.reason(f"Could not determine active branch: {e}", extra={"src": "list_branches"})
if not branches:
branches.append({
"name": "dev", "commit_hash": "0000000",
"is_remote": False, "last_updated": datetime.utcnow()
})
return branches
# endregion list_branches
# region create_branch [TYPE Function]
# @PURPOSE: Create a new branch from an existing one.
# region create_branch [C:4] [TYPE Function] [SEMANTICS git,branch,create,lock]
# @PURPOSE: Create a new branch from an existing one (concurrent-safe).
# @PARAM: name (str) - New branch name.
# @PARAM: from_branch (str) - Source branch.
# @PRE: Repository exists; name is valid; from_branch exists or repo is empty.
# @POST: A new branch is created in the repository.
def create_branch(self, dashboard_id: int, name: str, from_branch: str = "main"):
with belief_scope("GitService.create_branch"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.create_branch"):
repo = self.get_repo(dashboard_id)
logger.reason(f"Creating branch {name} from {from_branch}", extra={"src": "create_branch"})
if not repo.heads and not repo.remotes:
logger.reason("Repository is empty. Creating initial commit to enable branching.", extra={"src": "create_branch"})
@@ -157,26 +170,28 @@ class GitServiceBranchMixin:
raise
# endregion create_branch
# region checkout_branch [TYPE Function]
# @PURPOSE: Switch to a specific branch.
# region checkout_branch [C:4] [TYPE Function] [SEMANTICS git,branch,checkout,lock]
# @PURPOSE: Switch to a specific branch (concurrent-safe).
# @PRE: Repository exists and the specified branch name exists.
# @POST: The repository working directory is updated to the specified branch.
def checkout_branch(self, dashboard_id: int, name: str):
with belief_scope("GitService.checkout_branch"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.checkout_branch"):
repo = self.get_repo(dashboard_id)
logger.reason(f"Checking out branch {name}", extra={"src": "checkout_branch"})
repo.git.checkout(name)
# endregion checkout_branch
# region commit_changes [TYPE Function]
# @PURPOSE: Stage and commit changes.
# region commit_changes [C:4] [TYPE Function] [SEMANTICS git,commit,stage,lock]
# @PURPOSE: Stage and commit changes (concurrent-safe).
# @PARAM: message (str) - Commit message.
# @PARAM: files (List[str]) - Optional list of specific files to stage.
# @PRE: Repository exists and has changes (dirty) or files are specified.
# @POST: Changes are staged and a new commit is created.
def commit_changes(self, dashboard_id: int, message: str, files: list[str] = None):
with belief_scope("GitService.commit_changes"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.commit_changes"):
repo = self.get_repo(dashboard_id)
if not repo.is_dirty(untracked_files=True) and not files:
logger.reason(f"No changes to commit for dashboard {dashboard_id}", extra={"src": "commit_changes"})
return

42
backend/src/services/git/_gitea.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceGiteaMixin [C:3] [TYPE Module] [SEMANTICS git, gitea, api, remote, connection]
# #region GitServiceGiteaMixin [C:4] [TYPE Module] [SEMANTICS git, gitea, api, remote, connection, http_pool]
# @LAYER: Infra
# @BRIEF Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation.
# @BRIEF Gitea API operations for GitService — connection testing, repository CRUD, and pull request creation. Uses shared self._http_client for connection pooling.
# @RELATION USED_BY -> [GitService]
from typing import Any
@@ -35,24 +35,23 @@ class GitServiceGiteaMixin:
return False
pat = pat.strip()
try:
async with httpx.AsyncClient() as client:
if provider == GitProvider.GITHUB:
headers = {"Authorization": f"token {pat}"}
api_url = "https://api.github.com/user" if "github.com" in url else f"{url.rstrip('/')}/api/v3/user"
resp = await client.get(api_url, headers=headers)
elif provider == GitProvider.GITLAB:
headers = {"PRIVATE-TOKEN": pat}
api_url = f"{url.rstrip('/')}/api/v4/user"
resp = await client.get(api_url, headers=headers)
elif provider == GitProvider.GITEA:
headers = {"Authorization": f"token {pat}"}
api_url = f"{url.rstrip('/')}/api/v1/user"
resp = await client.get(api_url, headers=headers)
else:
return False
if resp.status_code != 200:
logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}")
return resp.status_code == 200
if provider == GitProvider.GITHUB:
headers = {"Authorization": f"token {pat}"}
api_url = "https://api.github.com/user" if "github.com" in url else f"{url.rstrip('/')}/api/v3/user"
resp = await self._http_client.get(api_url, headers=headers)
elif provider == GitProvider.GITLAB:
headers = {"PRIVATE-TOKEN": pat}
api_url = f"{url.rstrip('/')}/api/v4/user"
resp = await self._http_client.get(api_url, headers=headers)
elif provider == GitProvider.GITEA:
headers = {"Authorization": f"token {pat}"}
api_url = f"{url.rstrip('/')}/api/v1/user"
resp = await self._http_client.get(api_url, headers=headers)
else:
return False
if resp.status_code != 200:
logger.error(f"[test_connection][Coherence:Failed] Git connection test failed for {provider} at {api_url}. Status: {resp.status_code}")
return resp.status_code == 200
except Exception as e:
logger.error(f"[test_connection][Coherence:Failed] Error testing git connection: {e}")
return False
@@ -87,8 +86,7 @@ class GitServiceGiteaMixin:
url = f"{base_url}/api/v1{endpoint}"
headers = self._gitea_headers(pat)
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.request(method=method, url=url, headers=headers, json=payload)
response = await self._http_client.request(method=method, url=url, headers=headers, json=payload)
except Exception as e:
logger.error(f"[gitea_request][Coherence:Failed] Network error: {e}")
raise HTTPException(status_code=503, detail=f"Gitea API is unavailable: {e!s}")

147
backend/src/services/git/_merge.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceMergeMixin [C:3] [TYPE Module] [SEMANTICS git, merge, branch, conflict, resolution]
# #region GitServiceMergeMixin [C:4] [TYPE Module] [SEMANTICS git, merge, branch, conflict, resolution, lock]
# @LAYER: Infra
# @BRIEF Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote.
# @BRIEF Merge operations for GitService — conflict detection, resolution, abort, continue, and direct promote (all concurrent-safe via per-dashboard locks).
# @RELATION USED_BY -> [GitService]
import os
@@ -87,11 +87,12 @@ class GitServiceMergeMixin:
}
# endregion _build_unfinished_merge_payload
# region get_merge_status [TYPE Function]
# @PURPOSE: Get current merge status for a dashboard repository.
# region get_merge_status [C:4] [TYPE Function] [SEMANTICS git,merge,status,lock]
# @PURPOSE: Get current merge status for a dashboard repository (concurrent-safe).
def get_merge_status(self, dashboard_id: int) -> dict[str, Any]:
with belief_scope("GitService.get_merge_status"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.get_merge_status"):
repo = self.get_repo(dashboard_id)
merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD")
if not os.path.exists(merge_head_path):
current_branch = "unknown"
@@ -120,11 +121,12 @@ class GitServiceMergeMixin:
}
# endregion get_merge_status
# region get_merge_conflicts [TYPE Function]
# @PURPOSE: List all files with conflicts and their contents.
# region get_merge_conflicts [C:4] [TYPE Function] [SEMANTICS git,conflict,list,lock]
# @PURPOSE: List all files with conflicts and their contents (concurrent-safe).
def get_merge_conflicts(self, dashboard_id: int) -> list[dict[str, Any]]:
with belief_scope("GitService.get_merge_conflicts"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.get_merge_conflicts"):
repo = self.get_repo(dashboard_id)
conflicts = []
unmerged = repo.index.unmerged_blobs()
for file_path, stages in unmerged.items():
@@ -143,11 +145,12 @@ class GitServiceMergeMixin:
return sorted(conflicts, key=lambda item: item["file_path"])
# endregion get_merge_conflicts
# region resolve_merge_conflicts [TYPE Function]
# @PURPOSE: Resolve conflicts using specified strategy.
# region resolve_merge_conflicts [C:4] [TYPE Function] [SEMANTICS git,conflict,resolve,lock]
# @PURPOSE: Resolve conflicts using specified strategy (concurrent-safe).
def resolve_merge_conflicts(self, dashboard_id: int, resolutions: list[dict[str, Any]]) -> list[str]:
with belief_scope("GitService.resolve_merge_conflicts"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.resolve_merge_conflicts"):
repo = self.get_repo(dashboard_id)
resolved_files: list[str] = []
repo_root = os.path.abspath(str(repo.working_tree_dir or ""))
if not repo_root:
@@ -176,11 +179,12 @@ class GitServiceMergeMixin:
return resolved_files
# endregion resolve_merge_conflicts
# region abort_merge [TYPE Function]
# @PURPOSE: Abort ongoing merge.
# region abort_merge [C:4] [TYPE Function] [SEMANTICS git,merge,abort,lock]
# @PURPOSE: Abort ongoing merge (concurrent-safe).
def abort_merge(self, dashboard_id: int) -> dict[str, Any]:
with belief_scope("GitService.abort_merge"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.abort_merge"):
repo = self.get_repo(dashboard_id)
try:
repo.git.merge("--abort")
except GitCommandError as e:
@@ -192,11 +196,12 @@ class GitServiceMergeMixin:
return {"status": "aborted"}
# endregion abort_merge
# region continue_merge [TYPE Function]
# @PURPOSE: Finalize merge after conflict resolution.
# region continue_merge [C:4] [TYPE Function] [SEMANTICS git,merge,continue,lock]
# @PURPOSE: Finalize merge after conflict resolution (concurrent-safe).
def continue_merge(self, dashboard_id: int, message: str | None = None) -> dict[str, Any]:
with belief_scope("GitService.continue_merge"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.continue_merge"):
repo = self.get_repo(dashboard_id)
unmerged_files = self._get_unmerged_file_paths(repo)
if unmerged_files:
raise HTTPException(
@@ -227,51 +232,69 @@ class GitServiceMergeMixin:
return {"status": "committed", "commit_hash": commit_hash}
# endregion continue_merge
# region promote_direct_merge [TYPE Function]
# @PURPOSE: Perform direct merge between branches in local repo and push target branch.
# region promote_direct_merge [C:4] [TYPE Function] [SEMANTICS git,merge,promote,branch,isolation]
# @PURPOSE: Perform direct merge between branches with branch isolation — original branch restored on error.
# @PRE: Repository exists and both branches are valid.
# @POST: Target branch contains merged changes from source branch.
# @POST: Target branch contains merged changes from source branch. Active branch restored to original.
# Merge survives locally even if push fails (partial success).
# @SIDE_EFFECT: Changes local branch state during merge; restores original branch in finally block.
# @RETURN: Dict[str, Any]
def promote_direct_merge(self, dashboard_id: int, from_branch: str, to_branch: str) -> dict[str, Any]:
with belief_scope("GitService.promote_direct_merge"):
if not from_branch or not to_branch:
raise HTTPException(status_code=400, detail="from_branch and to_branch are required")
repo = self.get_repo(dashboard_id)
source = from_branch.strip()
target = to_branch.strip()
if source == target:
raise HTTPException(status_code=400, detail="from_branch and to_branch must be different")
try:
origin = repo.remote(name="origin")
except ValueError:
raise HTTPException(status_code=400, detail="Remote 'origin' not configured")
try:
origin.fetch()
if source not in [head.name for head in repo.heads]:
if f"origin/{source}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", source, f"origin/{source}")
else:
raise HTTPException(status_code=404, detail=f"Source branch '{source}' not found")
if target in [head.name for head in repo.heads]:
repo.git.checkout(target)
elif f"origin/{target}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", target, f"origin/{target}")
else:
raise HTTPException(status_code=404, detail=f"Target branch '{target}' not found")
with self._locked(dashboard_id):
with belief_scope("GitService.promote_direct_merge"):
if not from_branch or not to_branch:
raise HTTPException(status_code=400, detail="from_branch and to_branch are required")
repo = self.get_repo(dashboard_id)
source = from_branch.strip()
target = to_branch.strip()
if source == target:
raise HTTPException(status_code=400, detail="from_branch and to_branch must be different")
try:
origin.pull(target)
origin = repo.remote(name="origin")
except ValueError:
raise HTTPException(status_code=400, detail="Remote 'origin' not configured")
# Remember original branch for restoration in finally
original_branch = None
try:
original_branch = repo.active_branch.name
except Exception:
pass
repo.git.merge(source, "--no-ff", "-m", f"chore(flow): promote {source} -> {target}")
origin.push(refspec=f"{target}:{target}")
except HTTPException:
raise
except Exception as e:
message = str(e)
if "CONFLICT" in message.upper():
raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}")
raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}")
return {"mode": "direct", "from_branch": source, "to_branch": target, "status": "merged"}
original_branch = None
try:
origin.fetch()
if source not in [head.name for head in repo.heads]:
if f"origin/{source}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", source, f"origin/{source}")
else:
raise HTTPException(status_code=404, detail=f"Source branch '{source}' not found")
if target in [head.name for head in repo.heads]:
repo.git.checkout(target)
elif f"origin/{target}" in [ref.name for ref in repo.refs]:
repo.git.checkout("-b", target, f"origin/{target}")
else:
raise HTTPException(status_code=404, detail=f"Target branch '{target}' not found")
try:
origin.pull(target)
except Exception:
pass
repo.git.merge(source, "--no-ff", "-m", f"chore(flow): promote {source} -> {target}")
origin.push(refspec=f"{target}:{target}")
except HTTPException:
raise
except Exception as e:
message = str(e)
if "CONFLICT" in message.upper():
raise HTTPException(status_code=409, detail=f"Merge conflict during direct promote: {message}")
raise HTTPException(status_code=500, detail=f"Direct promote failed: {message}")
finally:
# Restore original branch even if merge/push partially failed
if original_branch:
try:
repo.git.checkout(original_branch)
except Exception:
pass
return {"mode": "direct", "from_branch": source, "to_branch": target, "status": "merged"}
# endregion promote_direct_merge
# #endregion GitServiceMergeMixin
# #endregion GitServiceMergeMixin

16
backend/src/services/git/_remote_providers.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceRemoteMixin [C:3] [TYPE Module] [SEMANTICS git, provider, github, remote, url]
# #region GitServiceRemoteMixin [C:4] [TYPE Module] [SEMANTICS git, provider, github, remote, url, http_pool]
# @LAYER: Infra
# @BRIEF GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation.
# @BRIEF GitHub and GitLab provider operations for GitService — repository creation and PR/MR creation. Uses shared self._http_client for connection pooling.
# @RELATION USED_BY -> [GitService]
from typing import Any
@@ -38,8 +38,7 @@ class GitServiceGithubMixin:
if default_branch:
payload["default_branch"] = default_branch
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
response = await self._http_client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {e!s}")
if response.status_code >= 400:
@@ -78,8 +77,7 @@ class GitServiceGithubMixin:
"body": description or "", "draft": bool(draft),
}
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
response = await self._http_client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitHub API is unavailable: {e!s}")
if response.status_code >= 400:
@@ -118,8 +116,7 @@ class GitServiceGitlabMixin:
if default_branch:
payload["default_branch"] = default_branch
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
response = await self._http_client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {e!s}")
if response.status_code >= 400:
@@ -162,8 +159,7 @@ class GitServiceGitlabMixin:
"description": description or "", "remove_source_branch": bool(remove_source_branch),
}
try:
async with httpx.AsyncClient(timeout=20.0) as client:
response = await client.post(api_url, headers=headers, json=payload)
response = await self._http_client.post(api_url, headers=headers, json=payload)
except Exception as e:
raise HTTPException(status_code=503, detail=f"GitLab API is unavailable: {e!s}")
if response.status_code >= 400:

31
backend/src/services/git/_status.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceStatusMixin [C:3] [TYPE Module] [SEMANTICS git, status, diff, log, history]
# #region GitServiceStatusMixin [C:4] [TYPE Module] [SEMANTICS git, status, diff, log, history, lock]
# @LAYER: Infra
# @BRIEF Status, diff, and commit history operations for GitService using git status --porcelain to avoid --cached flag issues.
# @BRIEF Status, diff, and commit history operations for GitService (all concurrent-safe via per-dashboard locks).
# @RELATION USED_BY -> [GitService]
from datetime import datetime
@@ -50,14 +50,15 @@ class GitServiceStatusMixin:
return staged, modified, untracked
# endregion _parse_status_porcelain
# region get_status [TYPE Function]
# @PURPOSE: Get current repository status (dirty files, untracked, etc.)
# region get_status [C:4] [TYPE Function] [SEMANTICS git,status,lock]
# @PURPOSE: Get current repository status (concurrent-safe).
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a dictionary representing the Git status.
# @RETURN: dict
def get_status(self, dashboard_id: int) -> dict:
with belief_scope("GitService.get_status"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.get_status"):
repo = self.get_repo(dashboard_id)
has_commits = False
try:
repo.head.commit
@@ -110,16 +111,17 @@ class GitServiceStatusMixin:
}
# endregion get_status
# region get_diff [TYPE Function]
# @PURPOSE: Generate diff for a file or the whole repository.
# region get_diff [C:4] [TYPE Function] [SEMANTICS git,diff,lock]
# @PURPOSE: Generate diff for a file or the whole repository (concurrent-safe).
# @PARAM: file_path (str) - Optional specific file.
# @PARAM: staged (bool) - Whether to show staged changes.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns the diff text as a string.
# @RETURN: str
def get_diff(self, dashboard_id: int, file_path: str = None, staged: bool = False) -> str:
with belief_scope("GitService.get_diff"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.get_diff"):
repo = self.get_repo(dashboard_id)
diff_args = []
if staged:
diff_args.append("--staged")
@@ -128,15 +130,16 @@ class GitServiceStatusMixin:
return repo.git.diff(*diff_args)
# endregion get_diff
# region get_commit_history [TYPE Function]
# @PURPOSE: Retrieve commit history for a repository.
# region get_commit_history [C:4] [TYPE Function] [SEMANTICS git,history,lock]
# @PURPOSE: Retrieve commit history for a repository (concurrent-safe).
# @PARAM: limit (int) - Max number of commits to return.
# @PRE: Repository for dashboard_id exists.
# @POST: Returns a list of dictionaries for each commit in history.
# @RETURN: List[dict]
def get_commit_history(self, dashboard_id: int, limit: int = 50) -> list[dict]:
with belief_scope("GitService.get_commit_history"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.get_commit_history"):
repo = self.get_repo(dashboard_id)
commits = []
try:
if not repo.heads and not repo.remotes:

22
backend/src/services/git/_sync.py
View File

@@ -1,6 +1,6 @@
# #region GitServiceSyncMixin [C:3] [TYPE Module] [SEMANTICS git, sync, push, pull, remote]
# #region GitServiceSyncMixin [C:4] [TYPE Module] [SEMANTICS git, sync, push, pull, remote, lock]
# @LAYER: Infra
# @BRIEF Push and pull operations for GitService with origin host auto-alignment.
# @BRIEF Push and pull operations for GitService with origin host auto-alignment (concurrent-safe).
# @RELATION USED_BY -> [GitService]
# @RELATION DEPENDS_ON -> [GitServiceUrlMixin]
@@ -17,13 +17,14 @@ from src.models.git import GitRepository, GitServerConfig
# #region GitServiceSyncMixin [C:3] [TYPE Class]
# @BRIEF Mixin providing push and pull operations with origin host alignment.
class GitServiceSyncMixin:
# region push_changes [TYPE Function]
# @PURPOSE: Push local commits to remote.
# region push_changes [C:4] [TYPE Function] [SEMANTICS git,push,lock]
# @PURPOSE: Push local commits to remote (concurrent-safe).
# @PRE: Repository exists and has an 'origin' remote.
# @POST: Local branch commits are pushed to origin.
def push_changes(self, dashboard_id: int):
with belief_scope("GitService.push_changes"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.push_changes"):
repo = self.get_repo(dashboard_id)
if not repo.heads:
logger.warning(f"[push_changes][Coherence:Failed] No local branches to push for dashboard {dashboard_id}")
return
@@ -110,13 +111,14 @@ class GitServiceSyncMixin:
raise HTTPException(status_code=500, detail=f"Git push failed: {e!s}")
# endregion push_changes
# region pull_changes [TYPE Function]
# @PURPOSE: Pull changes from remote.
# region pull_changes [C:4] [TYPE Function] [SEMANTICS git,pull,lock]
# @PURPOSE: Pull changes from remote (concurrent-safe).
# @PRE: Repository exists and has an 'origin' remote.
# @POST: Changes from origin are pulled and merged into the active branch.
def pull_changes(self, dashboard_id: int):
with belief_scope("GitService.pull_changes"):
repo = self.get_repo(dashboard_id)
with self._locked(dashboard_id):
with belief_scope("GitService.pull_changes"):
repo = self.get_repo(dashboard_id)
merge_head_path = os.path.join(repo.git_dir, "MERGE_HEAD")
if os.path.exists(merge_head_path):
payload = self._build_unfinished_merge_payload(repo)