chore: accumulate uncommitted workspace changes

This commit is contained in:
2026-08-26 17:03:22 +03:00
parent c4ccfcc9fd
commit 0415a2ed7d
183 changed files with 3015 additions and 2620 deletions

View File

@@ -31,7 +31,6 @@ indexing:
- '*.html'
- 'coverage_html_frontend/'
- 'coverage/'
- agent/
- '*,cover'
- docker/
source_dirs:
@@ -42,9 +41,16 @@ indexing:
- backend/tests
- frontend/src
- frontend/tests
- agent/src
- agent/tests
- shared/src
doc_dirs:
- docs
- specs
- .agents
- .agents/agents
- .agents/skills
- .agents/commands
- .opencode
- .specify
- .opencode/agents
@@ -246,7 +252,7 @@ tags:
multiline: false
is_reference: true
description: 'Graph dependency edge. Links contracts. Recommended on any function/module with external dependencies.'
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES, USES, CONTAINS, BELONGS_TO, ASSOCIATED_WITH]
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES]
PRE:
type: string
multiline: true
@@ -258,12 +264,12 @@ tags:
RATIONALE:
type: string
multiline: true
description: 'Architectural decision rationale. WHY this implementation was chosen. Decision Memory — prevents regression loops.'
description: 'Architectural decision rationale. WHY this implementation was chosen. Optional. Omit rather than invent. Synthetic text is a defect (INV_9).'
decision_memory: true
REJECTED:
type: string
multiline: true
description: 'Rejected alternative and disqualification reason. WHAT was tried and WHY it failed. Decision Memory — active guardrail against re-implementation.'
description: 'Rejected alternative and disqualification reason. Optional. Omit rather than invent a generic alternative. Synthetic text is a defect (INV_9).'
decision_memory: true
DATA_CONTRACT:
type: string
@@ -438,13 +444,13 @@ belief_runtime:
# #endregion AxiomConfig.BeliefRuntime
# #region AxiomConfig.ComplexityRules [C:2] [TYPE Block] [SEMANTICS config,complexity,rules]
# @BRIEF Per-tier tag requirements from GRACE-Poly SSOT. All tags allowed everywhere,
# but C4+ require formal contract annotations.
# @BRIEF Advisory typical tags from GRACE-Poly SSOT. Empty required lists:
# missing tags are not violations (INV_9). Do not fill tags to silence audits.
complexity_rules:
"4":
required: [PRE, POST, SIDE_EFFECT, DATA_CONTRACT]
required: []
"5":
required: [PRE, POST, SIDE_EFFECT, DATA_CONTRACT, INVARIANT]
required: []
# #endregion AxiomConfig.ComplexityRules

1
.gitignore vendored
View File

@@ -103,6 +103,7 @@ e2e_*.png
#generated doxygen
docs/api/html
docs/api/nav/
docs/api/build/
superset-tools.bundle

View File

@@ -163,20 +163,37 @@ CLI-бинарём `doc-gen` из Rust-проекта `axiom-mcp` (соседн
# 1. Собрать бинарник (один раз)
cargo build --release --bin doc-gen --manifest-path ../axiom-mcp/Cargo.toml
# 2. Переиндексировать workspace и собрать Doxygen HTML
# 2. Фрактальный граф: модули → функции (карты + Doxygen HTML)
make docs-nav
# эквивалент:
../axiom-mcp/target/release/doc-gen \
--workspace-root /home/busya/dev/ss-tools \
--html /tmp/ss-tools-doxygen
# 3. Только навигационный граф (без doxygen)
../axiom-mcp/target/release/doc-gen \
--workspace-root /home/busya/dev/ss-tools \
--nav /tmp/ss-tools-nav
--nav docs/api/nav \
--html docs/api/html
```
Как ходить по графу:
1. **Модули**`docs/api/nav/root.map` (или HTML mainpage). Не читать функции с корня.
2. **Функции**`docs/api/nav/<Module>.map` секция `@FUNCTIONS`, либо Doxygen group → `\ingroup` страница функции.
Примечания:
- `--workspace-root` передавать **абсолютным** путём (относительный путь ломает
проверку `safe_join` при записи в DuckDB).
- Результат doxygen: `.dox`-страницы, `Doxyfile` и каталог `html/` внутри `--html`-директории.
- Остальные опции: `doc-gen --help` (формат, per-file вывод, `--filter`, `--group-cap`, `--body-lines`).
- `make docs-doxygen` — отдельный XML/HTML extract из исходных комментариев в `docs/api/build`.
- Навигационный граф агента — `doc-gen --nav/--html`, не плоский список `axiom_*.html`.
- Остальные опции: `doc-gen --help` (`--filter`, `--group-cap`, `--body-lines`).
## Agent prompts and skills
`.agents/skills/` is the canonical source for semantic skills. The Kilo runtime
loads the generated copy from `.kilo/skills/`; after changing a skill, run:
```bash
./scripts/sync-skills.sh
```
Do not edit `.kilo/skills/` directly. Agent prompts follow the same source/runtime
split: canonical prompts are in `.agents/agents/` and the Kilo-loaded copies are
in `.kilo/agents/`.

View File

@@ -20,7 +20,7 @@ TIMEOUT_SLOW := 600
.PHONY: help test test-unit test-frontend test-related test-integration test-e2e test-all
.PHONY: coverage coverage-backend coverage-frontend
.PHONY: lint lint-backend lint-frontend
.PHONY: docs-doxygen docs-doxygen-check
.PHONY: docs-doxygen docs-doxygen-check docs-nav
# ── Help ───────────────────────────────────────────────────
help: ## Show this help message
@@ -88,7 +88,21 @@ lint-frontend: ## Frontend eslint
@cd $(FRONTEND) && npx eslint .
# ── Documentation ───────────────────────────────────────────
docs-doxygen: ## Generate Doxygen XML and HTML documentation
DOC_GEN ?= $(ROOT)/../axiom-mcp/target/release/doc-gen
docs-nav: ## Fractal module→function navigation graph (doc-gen --nav + --html)
@if [ ! -x "$(DOC_GEN)" ]; then \
echo " ▶ Building doc-gen..."; \
cargo build --release --bin doc-gen --manifest-path "$(ROOT)/../axiom-mcp/Cargo.toml"; \
fi
@echo " ▶ Navigation graph (modules → functions)..."
@LD_LIBRARY_PATH="$(ROOT)/../axiom-mcp/target/release/deps:$$LD_LIBRARY_PATH" \
"$(DOC_GEN)" --workspace-root "$(ROOT)" --nav "$(ROOT)/docs/api/nav" --html "$(ROOT)/docs/api/html"
@test -f "$(ROOT)/docs/api/nav/root.map"
@echo " ✅ Nav maps: docs/api/nav/root.map"
@echo " ✅ HTML: docs/api/html/index.html"
docs-doxygen: ## Generate Doxygen XML and HTML documentation from source comments
@if ! command -v doxygen >/dev/null 2>&1; then \
echo " ✗ doxygen not found."; \
echo " Debian/Ubuntu: sudo apt-get install -y doxygen"; \

View File

@@ -155,9 +155,6 @@ def _guard_tool_permission(tool_name: str) -> None:
# @BRIEF Fixed-delay retry wrapper for read-only tool HTTP calls on transient errors.
# @PRE Tool is read-only (risk_level="safe"). Error is 5xx or httpx.ConnectError.
# @POST Retries once with fixed 1s asyncio.sleep. On success: returns response. On exhaust: raises with retry_exhausted=True.
# @TEST_EDGE: first_attempt_502 -> Auto-retries once, succeeds.
# @TEST_EDGE: both_attempts_502 -> Raises error with retry_exhausted=True.
# @TEST_EDGE: write_tool_502 -> No retry, raises immediately.
# @RATIONALE 1 retry with 1s delay catches ~80% of transient failures. Read-only only.
# @REJECTED Exponential backoff — single retry cannot be exponential.
# @REJECTED Retry all tools — write operations not idempotent.
@@ -237,7 +234,6 @@ def _summarise_response(text: str, limit: int = 4000) -> str:
# @ingroup AgentChat
# @BRIEF Configurable timeout wrapper (default 30s). Write tools: retryable=false.
# @POST Returns tool result within timeout. On TimeoutError: yields tool_timeout SSE.
# @TEST_EDGE: complete_under_timeout→normal, read_exceed→retryable timeout, write_exceed→retryable=false
async def _execute_with_timeout(tool_name: str, tool_fn, is_write: bool = False, timeout_s: int = 30):
"""Execute tool with configurable timeout. Write tools: no retry on timeout.
@@ -365,7 +361,6 @@ class SearchDashboardsInput(BaseModel):
# #region AgentChat.Tools.SearchDashboards [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,search,dashboard]
# @ingroup AgentChat
# @BRIEF Search and list dashboards by name, with optional environment filter.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns formatted dashboard list string.
# @POST Response surfaces effective profile filter metadata so hidden dashboards are never reported as absent.
# @RATIONALE Agent search must see the full environment catalog: without page_context=other the backend
@@ -460,7 +455,6 @@ class HealthSummaryInput(BaseModel):
# #region AgentChat.Tools.HealthSummary [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,health,summary]
# @ingroup AgentChat
# @BRIEF Get system health summary — dashboard validation status, recent failures.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns health summary text (trimmed to 2000 chars).
# @SIDE_EFFECT HTTP GET to FastAPI /api/health/summary.
@tool(args_schema=HealthSummaryInput)
@@ -485,7 +479,6 @@ async def get_health_summary(env_id: str | None = None) -> str:
# #region AgentChat.Tools.ListEnvironments [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,environment,list]
# @ingroup AgentChat
# @BRIEF List configured deployment environments.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns JSON string of environments (sensitive fields redacted).
# @SIDE_EFFECT HTTP GET to FastAPI /api/settings/environments.
@tool
@@ -503,7 +496,6 @@ async def list_environments() -> str:
# #region AgentChat.Tools.TaskStatus [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,task,status]
# @ingroup AgentChat
# @BRIEF Check the status of a background task by its task_id.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns task status text from FastAPI.
# @SIDE_EFFECT HTTP GET to FastAPI /api/tasks/{task_id}.
@tool
@@ -535,7 +527,6 @@ async def show_capabilities() -> str:
# #region AgentChat.Tools.ListLlmProviders [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,llm,providers]
# @ingroup AgentChat
# @BRIEF List configured LLM providers.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns provider list text.
# @SIDE_EFFECT HTTP GET to FastAPI /api/llm/providers.
@tool
@@ -553,7 +544,6 @@ async def list_llm_providers() -> str:
# #region AgentChat.Tools.LlmStatus [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,llm,status]
# @ingroup AgentChat
# @BRIEF Check whether the LLM runtime is configured and usable.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns LLM status text.
# @SIDE_EFFECT HTTP GET to FastAPI /api/llm/status.
@tool
@@ -579,7 +569,6 @@ class RunBackupInput(BaseModel):
# #region AgentChat.Tools.RunBackup [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,backup,operation]
# @ingroup AgentChat
# @BRIEF Run a Superset backup for an environment, optionally scoped to dashboard IDs.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns task creation result (201).
# @SIDE_EFFECT HTTP POST to FastAPI /api/tasks (superset-backup plugin).
@tool(args_schema=RunBackupInput)
@@ -617,7 +606,6 @@ class ExecuteMigrationInput(BaseModel):
# #region AgentChat.Tools.ExecuteMigration [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,migration,operation]
# @ingroup AgentChat
# @BRIEF Execute dashboard migration between two environments.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns migration result text.
# @SIDE_EFFECT HTTP POST to FastAPI /api/migration/execute.
@tool(args_schema=ExecuteMigrationInput)
@@ -667,7 +655,6 @@ class CreateBranchInput(GitDashboardInput):
# #region AgentChat.Tools.CreateBranch [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,branch]
# @ingroup AgentChat
# @BRIEF Create a branch in a dashboard Git repository.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns branch creation result.
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/branches.
@tool(args_schema=CreateBranchInput)
@@ -704,7 +691,6 @@ class CommitChangesInput(GitDashboardInput):
# #region AgentChat.Tools.CommitChanges [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,commit]
# @ingroup AgentChat
# @BRIEF Stage and commit changes in a dashboard Git repository.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns commit result.
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/commit.
@tool(args_schema=CommitChangesInput)
@@ -740,7 +726,6 @@ class DeployDashboardInput(GitDashboardInput):
# #region AgentChat.Tools.DeployDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,deploy]
# @ingroup AgentChat
# @BRIEF Deploy a dashboard from Git to a target environment.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns deployment result.
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/deploy.
@tool(args_schema=DeployDashboardInput)
@@ -892,7 +877,6 @@ async def run_llm_validation(
# #region AgentChat.Tools.ListMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,list]
# @ingroup AgentChat
# @BRIEF List active and completed maintenance events.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns maintenance events text.
# @SIDE_EFFECT HTTP GET to FastAPI /api/maintenance/events.
@tool
@@ -920,7 +904,6 @@ class StartMaintenanceInput(BaseModel):
# #region AgentChat.Tools.StartMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,start]
# @ingroup AgentChat
# @BRIEF Start a maintenance event and apply banners to affected dashboards.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns maintenance start result (200/202).
# @SIDE_EFFECT HTTP POST to FastAPI /api/maintenance/start.
@tool(args_schema=StartMaintenanceInput)
@@ -962,7 +945,6 @@ class EndMaintenanceInput(BaseModel):
# #region AgentChat.Tools.EndMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,end]
# @ingroup AgentChat
# @BRIEF End one maintenance event, or end all active events when end_all is true.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns maintenance end result (202).
# @SIDE_EFFECT HTTP POST to FastAPI /api/maintenance/{id}/end or /api/maintenance/end-all.
@tool(args_schema=EndMaintenanceInput)
@@ -1139,7 +1121,6 @@ class AuditPermissionsInput(BaseModel):
# #region AgentChat.Tools.SupersetAudit [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,audit,permissions]
# @ingroup AgentChat
# @BRIEF Audit access rights: user x dashboard x dataset x RLS permission matrix.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns audit report with per-user dashboard/dataset access and RLS regions.
# @SIDE_EFFECT HTTP GET to FastAPI /api/agent/superset/audit/permissions.
@tool(args_schema=AuditPermissionsInput)
@@ -1176,7 +1157,6 @@ class CreateSupersetDashboardInput(BaseModel):
# #region AgentChat.Tools.SupersetCreateDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dashboard,create]
# @ingroup AgentChat
# @BRIEF Create a new dashboard in Superset.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns created dashboard dict.
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/dashboards.
@tool(args_schema=CreateSupersetDashboardInput)
@@ -1215,7 +1195,6 @@ class CopySupersetDashboardInput(BaseModel):
# #region AgentChat.Tools.SupersetCopyDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dashboard,copy]
# @ingroup AgentChat
# @BRIEF Deep-copy a Superset dashboard including all charts.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns copy result dict.
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/dashboards/{id}/copy.
@tool(args_schema=CopySupersetDashboardInput)
@@ -1250,7 +1229,6 @@ class CreateSupersetDatasetInput(BaseModel):
# #region AgentChat.Tools.SupersetCreateDataset [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dataset,create]
# @ingroup AgentChat
# @BRIEF Create a new dataset in Superset.
# @PRE User authenticated via dual-identity JWT.
# @POST Returns created dataset dict.
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/datasets.
@tool(args_schema=CreateSupersetDatasetInput)

View File

@@ -43,9 +43,6 @@ router = APIRouter(prefix="/api/auth", tags=["auth"])
# @SIDE_EFFECT DB read for user verification; writes security event LOGIN.
# @SIDE_EFFECT Molecular CoT: REASON on entry, REFLECT on success, EXPLORE on failure.
# @RELATION CALLS -> [Services.Auth.Service]
# @TEST_EDGE: invalid_credentials -> 401
# @TEST_EDGE: locked_account -> 423
# @TEST_EDGE: missing_fields -> 422
@router.post("/login", response_model=Token)
async def login_for_access_token(
@@ -186,8 +183,6 @@ async def get_session_policy(
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.GetCurrentUser]
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.TouchSessionActivity]
# @RELATION DEPENDS_ON -> [Services.Auth.Service]
# @TEST_EDGE: idle_expired -> 401 SESSION_IDLE_EXPIRED
# @TEST_EDGE: renewed -> 200 with new access_token and same sid
@router.post("/session/activity", response_model=SessionActivityResponse)
async def post_session_activity(
@@ -256,7 +251,6 @@ async def post_session_activity(
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.GetCurrentUser]
# @RELATION CALLS -> [Auth.Jwt.BlacklistToken]
# @RELATION CALLS -> [Services.Auth.Service]
# @TEST_EDGE: already_expired_token -> 200 (idempotent)
@router.post("/logout")
async def logout(
@@ -321,8 +315,6 @@ async def login_adfs(request: starlette.requests.Request):
# @SIDE_EFFECT DB write for user provisioning; writes security event LOGIN_ADFS.
# @SIDE_EFFECT Molecular CoT: REASON/REFLECT/EXPLORE markers.
# @RELATION CALLS -> [Services.Auth.Service]
# @TEST_EDGE: adfs_timeout -> 504
# @TEST_EDGE: invalid_state -> 401
@router.get("/callback/adfs", name="auth_callback_adfs")
async def auth_callback_adfs(

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting.Package [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing,package]
# #region Api.DashboardTesting.Package [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing,package]
# @defgroup Api Dashboard testing API route package — core + structure-diff + structure-snapshot + verification-runs.
# @LAYER API
# @BRIEF Combined router from submodules: core (existing), structure (feature-037),
@@ -50,4 +50,4 @@ router.include_router(scenario_automation_router)
# Re-export for backward-compatible imports
__all__ = ["router"]
#endregion Api.DashboardTesting.Package
# #endregion Api.DashboardTesting.Package

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing]
# #region Api.DashboardTesting [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing]
# @defgroup Api Dashboard testing API routes — inspection, execution, comparison, baselines, candidates.
# @LAYER API
# @RELATION DEPENDS_ON -> [BaselineEngine.QueryModel.Inspect]

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting.Inheritance [C:4] [TYPE Module] [SEMANTICS baseline,api,inheritance,plan,execute]
# #region Api.DashboardTesting.Inheritance [C:4] [TYPE Module] [SEMANTICS baseline,api,inheritance,plan,execute]
# @defgroup Api Baseline inheritance API endpoints — plan and execute.
# @LAYER API
# @RELATION DEPENDS_ON -> [BaselineEngine.Inheritance.PlanInheritance]
@@ -174,4 +174,4 @@ async def inheritance_execute(
) from err
# #endregion Api.DashboardTesting.Inheritance.ExecuteEndpoint
#endregion Api.DashboardTesting.Inheritance
# #endregion Api.DashboardTesting.Inheritance

View File

@@ -107,7 +107,6 @@ class ScenarioDispositionRequest(BaseModel):
# @BRIEF Deterministically compile a dashboard goal into a ScenarioGraph.
# @PRE Caller has dashboard-testing WRITE permission.
# @POST Returns ScenarioResponse (graph + validation) or 422 for invalid canonical inputs.
# @TEST_EDGE invalid_inputs -> 422 VALIDATION_ERROR.
@router.post("/scenarios/compile", status_code=status.HTTP_200_OK)
def api_compile_scenario(
req: ScenarioCompileRequest,
@@ -250,7 +249,6 @@ def api_draft_pack(
# @ingroup Api
# @BRIEF Execute a screenshot capture step via the 036 Evidence bridge.
# @POST Returns artifact refs (original + masked when mask_selectors present).
# @TEST_EDGE invalid_profile -> 422.
@router.post("/scenarios/{scenario_id}/capture", status_code=status.HTTP_200_OK)
def api_capture_screenshot(
scenario_id: str,

View File

@@ -313,8 +313,6 @@ def api_decide_human(run_id: str, body: HumanDecisionRequest, db=_DB, current_us
# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch]
# @INVARIANT Approval CAS can expose an approved run as queued, but only the separate server
# dispatcher may claim and execute its initial frontier.
# @TEST_EDGE approve -> queued; deny -> blocked; already_decided -> 409; missing_gate -> 404;
# missing_run_prod_scope -> 403; invalid_decision -> 422.
@runs_router.post("/{run_id}/approval/decision")
def api_decide_approval_gate(run_id: str, body: ApprovalDecisionRequest, db=_DB, current_user=_USER, _=_RUN_PROD_PERMISSION):
gate = (

View File

@@ -111,7 +111,6 @@ def api_list_scenarios(
# @BRIEF Atomically register a validated draft pack as a candidate revision.
# @PRE Caller has dashboard:testing WRITE; handles identify server-owned artifacts.
# @POST Returns scenario/revision ids with materialization_status=materialized; failures roll back both rows.
# @TEST_EDGE digest_mismatch -> 409; missing_runner_plan -> 409; access_denied -> 409.
@router.post("", response_model=ScenarioCreateResponse, status_code=status.HTTP_201_CREATED)
def api_create_scenario(
body: ScenarioCreateRequest,
@@ -505,7 +504,6 @@ def api_revalidate_scenario(
# @BRIEF Load a scenario detail by id (operationId scenarioRegistry.detail; fixes getScenarioDraft 404).
# @PRE Caller has dashboard:testing READ.
# @POST Returns ScenarioDetailResponse; 404 NOT_FOUND when the scenario is not in the registry.
# @TEST_EDGE not_found -> 404 with {"code": "NOT_FOUND"} envelope.
@router.get("/{scenario_id}", response_model=ScenarioDetailResponse)
def api_get_scenario(
scenario_id: str,

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting.StructureDiff [C:3] [TYPE Module] [SEMANTICS baseline,api,structure-diff,feature-037]
# #region Api.DashboardTesting.StructureDiff [C:3] [TYPE Module] [SEMANTICS baseline,api,structure-diff,feature-037]
# @defgroup Api Structure diff API route — POST /dashboard-testing/structure-diff.
# @LAYER API
# @RELATION DEPENDS_ON -> [BaselineEngine.StructureDiff.Service]
@@ -51,4 +51,4 @@ async def compute_structure_diff_endpoint(
# #endregion Api.DashboardTesting.ComputeStructureDiff
#endregion Api.DashboardTesting.StructureDiff
# #endregion Api.DashboardTesting.StructureDiff

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting.StructureSnapshot [C:4] [TYPE Module] [SEMANTICS baseline,api,structure-snapshot,release-bound,capture,diff]
# #region Api.DashboardTesting.StructureSnapshot [C:4] [TYPE Module] [SEMANTICS baseline,api,structure-snapshot,release-bound,capture,diff]
# @defgroup Api Release-bound structure snapshot API routes — capture + diff with identity validation.
# @LAYER API
# @RELATION DEPENDS_ON -> [BaselineEngine.StructureSnapshot.Service]
@@ -45,6 +45,9 @@ _EXECUTE_PERMISSION = Depends(has_permission("dashboard:testing", "EXECUTE"))
_DB_SESSION = Depends(get_db)
# #region Api.DashboardTesting.StructureSnapshot.ResolveEnvById [C:2] [TYPE Function]
# @ingroup Api
# @BRIEF Resolve a configured Environment by id — raises 404 if unknown.
async def _resolve_env_by_id(environment_id: str) -> Environment:
"""Resolve a configured Environment by id — raises 404 if unknown."""
env = get_config_manager().get_environment(environment_id)
@@ -54,6 +57,7 @@ async def _resolve_env_by_id(environment_id: str) -> Environment:
detail=f"Environment '{environment_id}' not found.",
)
return env
# #endregion Api.DashboardTesting.StructureSnapshot.ResolveEnvById
# #region Api.DashboardTesting.CaptureReleaseSnapshot [C:5] [TYPE Function] [SEMANTICS baseline,api,capture,release-bound,provenance]
@@ -254,4 +258,4 @@ async def diff_release_snapshots_endpoint(
) from err
# #endregion Api.DashboardTesting.DiffReleaseSnapshots
#endregion Api.DashboardTesting.StructureSnapshot
# #endregion Api.DashboardTesting.StructureSnapshot

View File

@@ -1,4 +1,4 @@
#region Api.DashboardTesting.VerificationRuns [C:3] [TYPE Module] [SEMANTICS baseline,api,verification-runs,feature-037]
# #region Api.DashboardTesting.VerificationRuns [C:3] [TYPE Module] [SEMANTICS baseline,api,verification-runs,feature-037]
# @defgroup Api Verification runs API route — POST create + GET history/detail (037 T081).
# @LAYER API
# @RELATION DEPENDS_ON -> [BaselineEngine.Verification.Service]
@@ -71,8 +71,6 @@ async def create_verification_run_endpoint(
# @BRIEF List verification runs chronologically, optionally filtered by dashboard_id/environment_id (037 T081).
# @POST Returns VerificationRun[] ordered by created_at desc; matches frontend getVerificationHistory().
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
# @TEST_EDGE dashboard_filter -> only runs for that dashboard returned.
# @TEST_EDGE env_filter -> only runs for that environment returned.
@router.get("/verification/history", response_model=list[VerificationRun])
def list_verification_history(
dashboard_id: int | None = None,
@@ -96,7 +94,6 @@ def list_verification_history(
# @BRIEF Return a single verification run by id (037 T081).
# @POST Returns VerificationRun; 404 when the run does not exist.
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
# @TEST_EDGE missing_run -> 404.
@router.get("/verification/{run_id}", response_model=VerificationRun)
def get_verification_run(
run_id: str,
@@ -109,4 +106,4 @@ def get_verification_run(
return _record_to_response(record)
# #endregion Api.DashboardTesting.VerificationDetail
#endregion Api.DashboardTesting.VerificationRuns
# #endregion Api.DashboardTesting.VerificationRuns

View File

@@ -14,25 +14,6 @@
# @SIDE_EFFECT Performs external calls to Superset API and potentially Git providers.
# @DATA_CONTRACT Input(env_id, filters) -> Output(DashboardsResponse)
#
# @TEST_CONTRACT DashboardsAPI -> {
# required_fields: {env_id: string, page: integer, page_size: integer},
# optional_fields: {search: string},
# invariants: ["Pagination must be valid", "Environment must exist"]
# }
#
# @TEST_FIXTURE dashboard_list_happy -> {
# "env_id": "prod",
# "expected_count": 1,
# "dashboards": [{"id": 1, "title": "Main Revenue"}]
# }
#
# @TEST_EDGE pagination_zero_page -> {"env_id": "prod", "page": 0, "status": 400}
# @TEST_EDGE pagination_oversize -> {"env_id": "prod", "page_size": 101, "status": 400}
# @TEST_EDGE missing_env -> {"env_id": "ghost", "status": 404}
# @TEST_EDGE empty_dashboards -> {"env_id": "empty_env", "expected_total": 0}
# @TEST_EDGE external_superset_failure -> {"env_id": "bad_conn", "status": 503}
#
# @TEST_INVARIANT metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards]
from ._action_routes import *
from ._detail_routes import *

View File

@@ -11,10 +11,6 @@
# @RATIONALE Centralizes secret inventory and recovery after ENCRYPTION_KEY change.
# Without this, operators must manually trace decrypt failures across
# scattered API responses.
# @TEST_EDGE: empty_payload — POST /recover with empty items list → returns failed/failed
# @TEST_EDGE: provider_not_found — recovery for non-existent provider ID → skipped
# @TEST_EDGE: connection_not_found — recovery for non-existent connection ID → skipped
# @TEST_EDGE: invalid_type — recovery item with unknown type → skipped
import hashlib

View File

@@ -127,7 +127,7 @@ def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig:
# #endregion Api.Helpers.GetGitConfigOr404
# #region Api.Helpers.FindDashboardIdBySlug [C:2] [TYPE Function]
# #region Api.Helpers.GitHelpers.FindDashboardIdBySlug [C:2] [TYPE Function]
# @BRIEF Resolve dashboard numeric ID by slug in a specific environment.
async def _find_dashboard_id_by_slug(
client: SupersetClient,
@@ -158,10 +158,10 @@ async def _find_dashboard_id_by_slug(
return None
# #endregion Api.Helpers.FindDashboardIdBySlug
# #endregion Api.Helpers.GitHelpers.FindDashboardIdBySlug
# #region Api.Helpers.ResolveDashboardIdFromRef [C:2] [TYPE Function]
# #region Api.Helpers.GitHelpers.ResolveDashboardIdFromRef [C:2] [TYPE Function]
# @BRIEF Resolve dashboard ID from slug-or-id reference for Git routes.
async def _resolve_dashboard_id_from_ref(
dashboard_ref: str,
@@ -192,10 +192,10 @@ async def _resolve_dashboard_id_from_ref(
return dashboard_id
# #endregion Api.Helpers.ResolveDashboardIdFromRef
# #endregion Api.Helpers.GitHelpers.ResolveDashboardIdFromRef
# #region Api.Helpers.FindDashboardIdBySlugAsync [C:2] [TYPE Function]
# #region Api.Helpers.GitHelpers.FindDashboardIdBySlugAsync [C:2] [TYPE Function]
# @BRIEF Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes.
async def _find_dashboard_id_by_slug_async(
client: "AsyncSupersetClient",
@@ -226,10 +226,10 @@ async def _find_dashboard_id_by_slug_async(
return None
# #endregion Api.Helpers.FindDashboardIdBySlugAsync
# #endregion Api.Helpers.GitHelpers.FindDashboardIdBySlugAsync
# #region Api.Helpers.ResolveDashboardIdFromRefAsync [C:2] [TYPE Function]
# #region Api.Helpers.GitHelpers.ResolveDashboardIdFromRefAsync [C:2] [TYPE Function]
# @BRIEF Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes.
async def _resolve_dashboard_id_from_ref_async(
dashboard_ref: str,
@@ -266,7 +266,7 @@ async def _resolve_dashboard_id_from_ref_async(
await client.aclose()
# #endregion Api.Helpers.ResolveDashboardIdFromRefAsync
# #endregion Api.Helpers.GitHelpers.ResolveDashboardIdFromRefAsync
# #region Api.Helpers.ResolveRepoKeyFromRef [C:2] [TYPE Function]

View File

@@ -14,13 +14,6 @@
# @POST Migration tasks are enqueued or dry-run results are computed and returned.
# @SIDE_EFFECT Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls.
# @DATA_CONTRACT [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary]
# @TEST_CONTRACT [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary]
# @TEST_SCENARIO [invalid_environment] -> [HTTP_400_or_404]
# @TEST_SCENARIO [valid_execution] -> [success_payload_with_required_fields]
# @TEST_EDGE [missing_field] ->[HTTP_400]
# @TEST_EDGE [invalid_type] ->[validation_error]
# @TEST_EDGE [external_fail] ->[HTTP_500]
# @TEST_INVARIANT [EnvironmentValidationBeforeAction] -> VERIFIED_BY: [invalid_environment, valid_execution]
# @RATIONALE Separates API concerns (routing, permission checks, request/response serialization) from core migration business logic, enabling independent evolution of HTTP contract and domain logic.
# @REJECTED Embedding API logic directly in the core migration layer was rejected — it would couple HTTP concerns with business logic, making both layers harder to test, version, and maintain independently.

View File

@@ -84,19 +84,6 @@ def _parse_csv_enum_list(raw: str | None, enum_cls, field_name: str) -> list:
# @RELATION DEPENDS_ON -> [Models.Report.ReportQuery]
# @RELATION DEPENDS_ON -> [Services.ReportService.ReportsService]
#
# @TEST_CONTRACT ListReportsApi ->
# {
# required_fields: {page: int, page_size: int, sort_by: str, sort_order: str},
# optional_fields: {task_types: str, statuses: str, search: str},
# invariants: [
# "Returns ReportCollection on success",
# "Raises HTTPException 400 for invalid query parameters"
# ]
# }
# @TEST_FIXTURE valid_list_request -> {"page": 1, "page_size": 20}
# @TEST_EDGE invalid_task_type_filter -> raises HTTPException(400)
# @TEST_EDGE malformed_query -> raises HTTPException(400)
# @TEST_INVARIANT consistent_list_payload -> verifies: [valid_list_request]
@router.get("", response_model=ReportCollection)
async def list_reports(
page: int = Query(1, ge=1),

View File

@@ -231,13 +231,6 @@ async def get_task(
# @POST Returns a list of log entries or raises 404.
# @RELATION CALLS -> [Core.Manager.TaskManager]
# @RELATION DEPENDS_ON -> [Core.Models.LogFilter]
# @TEST_CONTRACT TaskLogQueryInput -> List[LogEntry]
# @TEST_SCENARIO existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination.
# @TEST_FIXTURE valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json
# @TEST_EDGE missing_task -> Unknown task_id returns 404 Task not found.
# @TEST_EDGE invalid_level_type -> Non-string/invalid level query rejected by validation or yields empty result.
# @TEST_EDGE pagination_bounds -> offset=0 and limit=1000 remain within API bounds and do not overflow.
# @TEST_INVARIANT logs_only_for_existing_task -> VERIFIED_BY: [existing_task_logs_filtered, missing_task]
@router.get("/{task_id}/logs")
async def get_task_logs(
task_id: str,

View File

@@ -490,14 +490,25 @@ _POLLING_EXACT_PATHS = frozenset({
"/api/agent/llm-config",
"/api/auth/session/activity",
"/api/settings/consolidated",
"/api/environments",
"/api/auth/me",
"/api/auth/session",
"/api/settings/features",
"/api/settings/allowed-languages",
"/api/profile/preferences",
"/api/dashboards",
"/api/maintenance/dashboard-banners",
"/api/maintenance/events",
"/api/validation-tasks/status/batch",
"/api/git/repositories/status/batch",
})
def _is_suppressed_request(request: Request) -> bool:
"""Return True for high-frequency polling requests (framing suppressed)."""
path = request.url.path
return path in _POLLING_EXACT_PATHS or (
request.method == "GET" and path.startswith("/api/tasks")
return request.method == "GET" and (
path in _POLLING_EXACT_PATHS or path.startswith("/api/tasks")
)
@@ -884,22 +895,6 @@ def _set_websocket_trace_id(websocket: WebSocket) -> str:
# — sends unnecessary data over the network when the server can discard early. Polling
# /api/tasks for status changes was rejected — introduces latency and load.
#
# @TEST_CONTRACT WebSocketLogStreamApi ->
# {
# required_fields: {websocket: WebSocket, task_id: str},
# optional_fields: {source: str, level: str},
# invariants: [
# "Accepts the WebSocket connection",
# "Applies source and level filters correctly to streamed logs",
# "Cleans up subscriptions on disconnect"
# ]
# }
# @TEST_FIXTURE valid_ws_connection -> {"task_id": "test_1", "source": "plugin"}
# @TEST_EDGE task_not_found_ws -> closes connection or sends error
# @TEST_EDGE empty_task_logs -> waits for new logs
# @TEST_INVARIANT consistent_streaming -> verifies: [valid_ws_connection]
# @TEST_EDGE ws_auth_missing_token -> connection rejected with 4001
# @TEST_EDGE ws_auth_invalid_token -> connection rejected with 4001
@app.websocket("/ws/logs/{task_id}")
async def websocket_endpoint(websocket: WebSocket, task_id: str, source: str = None, level: str = None):
"""

View File

@@ -17,7 +17,7 @@ import asyncio
from typing import Any
# #region Core.AsyncJobRunner [C:4] [TYPE Class] [SEMANTICS async,runner,scheduler]
# #region Core.AsyncJobRunner.Class [C:4] [TYPE Class] [SEMANTICS async,runner,scheduler]
# @ingroup Core
# @BRIEF Centralized async/sync bridge for dispatching coroutines from sync threads.
# @PRE Event loop is captured at construction time.
@@ -126,5 +126,5 @@ class AsyncJobRunner:
error=traceback.format_exc())
asyncio.run_coroutine_threadsafe(_guarded(), self.loop)
# #endregion Core.AsyncJobRunner.Dispatch
# #endregion Core.AsyncJobRunner
# #endregion Core.AsyncJobRunner.Class
# #endregion Core.AsyncJobRunner

View File

@@ -129,7 +129,6 @@ def _prune_blacklist(db: "Session") -> None:
# @SIDE_EFFECT Writes to token_blacklist table; prunes expired entries.
# @RELATION DEPENDS_ON -> [Models.Auth.TokenBlacklist]
# @RELATION CALLS -> [Auth.Jwt._HashToken]
# @TEST_EDGE: already_expired -> skips blacklisting
def blacklist_token(token: str, db: "Session") -> None:
from sqlalchemy.orm import Session # noqa: F811
from ...models.auth import TokenBlacklist

View File

@@ -40,7 +40,7 @@ from .rate_limiter import invalidate_rate_limiter_config
_AUTH_POLICY_FIELDS = ("auth_max_attempts", "auth_attempt_window", "auth_ban_duration")
# #region Core.ConfigManager [C:5] [TYPE Class]
# #region Core.ConfigManager.Class [C:5] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle.
# @PRE Database is accessible and AppConfigRecord schema is loaded.
@@ -638,5 +638,5 @@ class ConfigManager:
self.save()
return True
# #endregion Core.ConfigManager.DeleteEnvironment
# #endregion Core.ConfigManager
# #endregion Core.ConfigManager.Class
# #endregion Core.ConfigManager

View File

@@ -75,7 +75,7 @@ def _validate_name_uniqueness(
# #endregion Core.ConnectionService.ValidateNameUniqueness
# #region Core.ConnectionService [C:3] [TYPE Class] [SEMANTICS settings,connections,service]
# #region Core.ConnectionService.Class [C:3] [TYPE Class] [SEMANTICS settings,connections,service]
# @ingroup Core
# @BRIEF Service class wrapping all DatabaseConnection CRUD + test operations.
class ConnectionService:
@@ -522,5 +522,5 @@ class ConnectionService:
# #endregion Core.ConnectionService.ValidateConnectionRefs
# #endregion Core.ConnectionService
# #endregion Core.ConnectionService.Class
# #endregion Core.ConnectionService

View File

@@ -76,17 +76,6 @@ def _require_fernet_key() -> bytes:
# run — cached at module level via get_encryption_manager() singleton to eliminate
# redundant env-var reads and key validation.
#
# @TEST_CONTRACT EncryptionManagerModel ->
# {
# required_fields: {},
# invariants: [
# "encrypted data can be decrypted back to the original string"
# ]
# }
# @TEST_FIXTURE basic_encryption_cycle -> {"data": "my_secret_key"}
# @TEST_EDGE decrypt_invalid_data -> raises Exception
# @TEST_EDGE empty_string_encryption -> {"data": ""}
# @TEST_INVARIANT symmetric_encryption -> verifies: [basic_encryption_cycle, empty_string_encryption]
class EncryptionManager:
# region Core.Encryption.EncryptionManagerInit [TYPE Function]
# @BRIEF: Initialize the encryption manager with a Fernet key.

View File

@@ -10,8 +10,6 @@
# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs.
# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
#
# @INVARIANT sync_environment must handle remote API failures gracefully.
# @RATIONALE Centralizes UUID-to-integer ID resolution for Superset resources because the Superset API uses different ID schemes across endpoints (UUIDs for import/export, integer IDs for CRUD operations), enabling cross-environment migration with consistent resource references.
# @REJECTED BackgroundScheduler — was never started; replaced by AsyncJobRunner for async/sync bridge.
@@ -36,20 +34,6 @@ from src.models.mapping import Environment, ResourceMapping, ResourceType
# @SIDE_EFFECT Performs database writes during sync cycles.
# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService]
#
# @TEST_CONTRACT IdMappingServiceModel ->
# {
# required_fields: {db_session: Session},
# invariants: [
# "sync_environment correctly creates or updates ResourceMapping records",
# "get_remote_id returns an integer or None",
# "get_remote_ids_batch returns a dictionary of valid UUIDs to integers"
# ]
# }
# @TEST_FIXTURE valid_mapping_service -> {"db_session": "MockSession()"}
# @TEST_EDGE sync_api_failure -> handles exception gracefully
# @TEST_EDGE get_remote_id_not_found -> returns None
# @TEST_EDGE get_batch_empty_list -> returns empty dict
# @TEST_INVARIANT resilient_fetching -> verifies: [sync_api_failure]
class IdMappingService:
# #region Core.MappingService.Init [C:2] [TYPE Function]
# @BRIEF: Initializes the mapping service.

View File

@@ -14,14 +14,6 @@
# @POST Risk scoring output preserves item list and provides bounded score with derived level.
# @SIDE_EFFECT Emits diagnostic logs and performs read-only metadata requests via Superset client.
# @DATA_CONTRACT Module[build_risks, score_risks]
# @TEST_CONTRACT [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
# @TEST_SCENARIO [overwrite_update_objects] -> [confirmation overwrite_existing item is emitted for each update diff item without increasing risk score]
# @TEST_SCENARIO [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
# @TEST_SCENARIO [owner_mismatch_dashboard] -> [low owner_mismatch risk is emitted]
# @TEST_EDGE [missing_field] -> [object without uuid is ignored by indexer]
# @TEST_EDGE [invalid_type] -> [non-list owners input normalizes to empty identifiers]
# @TEST_EDGE [external_fail] -> [target_client get_databases exception propagates to caller]
# @TEST_INVARIANT [score_upper_bound_100] -> VERIFIED_BY: [severity_weight_aggregation]
# @UX_STATE [Idle] -> [N/A backend domain module]
# @UX_FEEDBACK [N/A] -> [No direct UI side effects in this module]
# @UX_RECOVERY [N/A] -> [Caller-level retry/recovery]

View File

@@ -16,7 +16,7 @@ from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.Audit.SupersetAuditMixin [C:4] [TYPE Class]
# #region Core.Audit.SupersetAuditMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing comprehensive permissions audit across users, dashboards, datasets, and RLS.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -354,5 +354,5 @@ class SupersetAuditMixin:
return found
# #endregion SupersetAudit.FindDatasourcePermissions
# #endregion Core.Audit.SupersetAuditMixin
# #endregion Core.Audit.SupersetAuditMixin.Class
# #endregion Core.Audit.SupersetAuditMixin

View File

@@ -23,7 +23,7 @@ from ..utils.async_network import AsyncAPIClient
from ..utils.network import SupersetAPIError
app_logger = cast(Any, app_logger)
# #region Core.Base.SupersetClientBase [C:4] [TYPE Class]
# #region Core.Base.SupersetClientBase.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Base class providing Superset client initialization, auth, pagination, and import/export plumbing.
# @RELATION DEPENDS_ON -> [Core.ConfigModels]
@@ -300,5 +300,5 @@ class SupersetClientBase:
async def aclose(self) -> None:
await self.client.aclose()
# #endregion Core.Base.Aclose
# #endregion Core.Base.SupersetClientBase
# #endregion Core.Base.SupersetClientBase.Class
# #endregion Core.Base.SupersetClientBase

View File

@@ -1,4 +1,4 @@
#region SupersetClient.ChartData.Execute [C:4] [TYPE Module] [SEMANTICS baseline,superset,chart-data,async]
# #region SupersetClient.ChartData.Execute [C:4] [TYPE Module] [SEMANTICS baseline,superset,chart-data,async]
# @defgroup Core Superset chart-data POST /api/v1/chart/data mixin.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -30,13 +30,13 @@ class ChartDataResponse:
source_response_hash: str = field(repr=False)
# #endregion SupersetClient.ChartData.ResponseDTO
# @region SupersetClient.ChartData.SupersetChartDataMixin [C:4] [TYPE Class]
# #region SupersetClient.ChartData.SupersetChartDataMixin [C:4] [TYPE Class]
# @defgroup Core Chart data execution mixin for SupersetClient.
# @SIDE_EFFECT Async POST to Superset /api/v1/chart/data.
class SupersetChartDataMixin:
"""Mixin for executing Superset chart-data queries."""
# @region SupersetClient.ChartData.ExecuteQueryRaw [C:4] [TYPE Function] [SEMANTICS chart-data,raw,response,hash]
# #region SupersetClient.ChartData.ExecuteQueryRaw [C:4] [TYPE Function] [SEMANTICS chart-data,raw,response,hash]
# @ingroup Core
# @BRIEF Execute chart-data query with raw_response=True; return ChartDataResponse with raw bytes + hash.
# @PRE Saved chart exists and is accessible.
@@ -114,9 +114,9 @@ class SupersetChartDataMixin:
except Exception as e:
app_logger.explore("Chart data raw execution failed", extra={"chart_id": chart_id}, error=str(e))
raise SupersetAPIError(f"Chart data raw execution failed: {e}") from e
# @endregion SupersetClient.ChartData.ExecuteQueryRaw
# #endregion SupersetClient.ChartData.ExecuteQueryRaw
# @region SupersetClient.ChartData.ExecuteQuery [C:4] [TYPE Function]
# #region SupersetClient.ChartData.ExecuteQuery [C:4] [TYPE Function]
# @ingroup Core
# @BRIEF Backward-compatible wrapper — execute chart-data query, return only the parsed dict.
# @PRE Saved chart exists and is accessible.
@@ -150,9 +150,9 @@ class SupersetChartDataMixin:
row_limit=row_limit,
)
return response.parsed
# @endregion SupersetClient.ChartData.ExecuteQuery
# #endregion SupersetClient.ChartData.ExecuteQuery
# @region SupersetClient.ChartData.BuildAdhocFilters [C:2] [TYPE Function] [SEMANTICS superse,chart-data,filters]
# #region SupersetClient.ChartData.BuildAdhocFilters [C:2] [TYPE Function] [SEMANTICS superse,chart-data,filters]
# @ingroup Core
# @BRIEF Normalize filter dicts into Superset adhoc filter format, preserving TEMPORAL_RANGE.
# @PRE Input filters use subject/comparator/expressionType keys.
@@ -191,8 +191,8 @@ class SupersetChartDataMixin:
entry["to"] = to_val
adhoc_filters.append(entry)
return adhoc_filters
# @endregion SupersetClient.ChartData.BuildAdhocFilters
# #endregion SupersetClient.ChartData.BuildAdhocFilters
# @endregion SupersetClient.ChartData.SupersetChartDataMixin
# #endregion SupersetClient.ChartData.SupersetChartDataMixin
#endregion SupersetClient.ChartData.Execute
# #endregion SupersetClient.ChartData.Execute

View File

@@ -10,7 +10,7 @@ from typing import Any, cast
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.Charts.SupersetChartsMixin [C:3] [TYPE Class]
# #region Core.Charts.SupersetChartsMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing all chart-related Superset API operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -74,5 +74,5 @@ class SupersetChartsMixin:
walk(payload)
return found
# #endregion Core.Charts.SupersetClientExtractChartIdsFromLayout
# #endregion Core.Charts.SupersetChartsMixin
# #endregion Core.Charts.SupersetChartsMixin.Class
# #endregion Core.Charts.SupersetChartsMixin

View File

@@ -16,7 +16,7 @@ import httpx
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.DashboardsCrud.SupersetDashboardsCrudMixin [C:3] [TYPE Class]
# #region Core.DashboardsCrud.SupersetDashboardsCrudMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing dashboard detail resolution, export, import, and delete operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -430,5 +430,5 @@ class SupersetDashboardsCrudMixin:
raise
# #endregion Core.DashboardsCrud.SupersetClientGetDashboard
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin.Class
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin

View File

@@ -10,7 +10,7 @@ from typing import Any, cast
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.DashboardsFilters.SupersetDashboardsFiltersMixin [C:3] [TYPE Class]
# #region Core.DashboardsFilters.SupersetDashboardsFiltersMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -236,5 +236,5 @@ class SupersetDashboardsFiltersMixin:
)
return result
# #endregion Core.DashboardsFilters.SupersetClientParseDashboardUrlForFilters
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin.Class
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin

View File

@@ -11,7 +11,7 @@ from typing import Any, cast
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.DashboardsList.SupersetDashboardsListMixin [C:3] [TYPE Class]
# #region Core.DashboardsList.SupersetDashboardsListMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing dashboard listing and summary projection operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -205,5 +205,5 @@ class SupersetDashboardsListMixin:
})
return total_count, result
# #endregion Core.DashboardsList.SupersetClientGetDashboardsSummaryPage
# #endregion Core.DashboardsList.SupersetDashboardsListMixin
# #endregion Core.DashboardsList.SupersetDashboardsListMixin.Class
# #endregion Core.DashboardsList.SupersetDashboardsListMixin

View File

@@ -26,7 +26,7 @@ from ._layout_utils import (
app_logger = cast(Any, app_logger)
# #region Core.DashboardsWrite.SupersetDashboardsWriteMixin [C:4] [TYPE Class]
# #region Core.DashboardsWrite.SupersetDashboardsWriteMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing markdown chart CRUD and dashboard layout manipulation for maintenance banners.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -532,5 +532,5 @@ class SupersetDashboardsWriteMixin:
raise
# #endregion Core.DashboardsWrite.UnpublishDashboard
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin.Class
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin

View File

@@ -10,7 +10,7 @@ from typing import Any, cast
from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.Databases.SupersetDatabasesMixin [C:3] [TYPE Class]
# #region Core.Databases.SupersetDatabasesMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing all database-related Superset API operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -462,5 +462,5 @@ class SupersetDatabasesMixin:
raise
# #endregion Core.Databases.SupersetClientGetAvailableEngines
# #endregion Core.Databases.SupersetDatabasesMixin
# #endregion Core.Databases.SupersetDatabasesMixin.Class
# #endregion Core.Databases.SupersetDatabasesMixin

View File

@@ -13,7 +13,7 @@ from ..logger import belief_scope, logger as app_logger
from ..utils.network import SupersetAPIError
# #region Core.DatasetsPreview.SupersetDatasetsPreviewMixin [C:4] [TYPE Class]
# #region Core.DatasetsPreview.SupersetDatasetsPreviewMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing dataset preview compilation and query context building.
# @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API.
@@ -217,5 +217,5 @@ class SupersetDatasetsPreviewMixin:
)
return payload
# #endregion Core.DatasetsPreview.SupersetClientBuildDatasetPreviewQueryContext
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin.Class
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin

View File

@@ -12,7 +12,7 @@ from ..logger import belief_scope, logger as app_logger
from ..utils.network import SupersetAPIError
# #region Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class]
# #region Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -181,5 +181,5 @@ class SupersetDatasetsPreviewFiltersMixin:
f"(diagnostics={response_diagnostics!r})"
)
# #endregion Core.DatasetsPreviewFilters.SupersetClientExtractCompiledSqlFromPreviewResponse
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin.Class
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin

View File

@@ -11,7 +11,7 @@ from ..logger import belief_scope, logger as app_logger
app_logger = cast(Any, app_logger)
# #region Core.SavedQueries.SupersetSavedQueriesMixin [C:3] [TYPE Class]
# #region Core.SavedQueries.SupersetSavedQueriesMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing saved query CRUD operations.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -156,5 +156,5 @@ class SupersetSavedQueriesMixin:
raise
# #endregion SupersetSavedQueries.Delete
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin.Class
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin

View File

@@ -14,7 +14,7 @@ from .safety import is_dangerous_sql
app_logger = cast(Any, app_logger)
# #region Core.SqlLab.SupersetSqlLabMixin [C:4] [TYPE Class]
# #region Core.SqlLab.SupersetSqlLabMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing SQL Lab execution, formatting, results retrieval, cost estimation, CSV export, and query history.
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
@@ -261,5 +261,5 @@ class SupersetSqlLabMixin:
raise
# #endregion SupersetSqlLab.StopQuery
# #endregion Core.SqlLab.SupersetSqlLabMixin
# #endregion Core.SqlLab.SupersetSqlLabMixin.Class
# #endregion Core.SqlLab.SupersetSqlLabMixin

View File

@@ -38,19 +38,6 @@ from .task_logger import TaskLogger
# plugin signatures. A global singleton logger was rejected — it cannot scope log
# entries to individual task executions when multiple tasks run concurrently.
#
# @TEST_CONTRACT TaskContextContract ->
# {
# required_fields: {task_id: str, add_log_fn: Callable, params: dict},
# optional_fields: {default_source: str},
# invariants: [
# "task_id matches initialized logger's task_id",
# "logger is a valid TaskLogger instance"
# ]
# }
# @TEST_FIXTURE valid_context -> {"task_id": "123", "add_log_fn": lambda *args: None, "params": {"k": "v"}, "default_source": "plugin"}
# @TEST_EDGE missing_task_id -> raises TypeError
# @TEST_EDGE missing_add_log_fn -> raises TypeError
# @TEST_INVARIANT logger_initialized -> verifies: [valid_context]
class TaskContext:
"""
Execution context provided to plugins during task execution.

View File

@@ -15,17 +15,6 @@
# @RELATION DEPENDS_ON -> [Core.Lifecycle.JobLifecycle]
# @RELATION DEPENDS_ON -> [Core.EventBus]
# @INVARIANT Task IDs are unique.
# @TEST_CONTRACT TaskManagerRuntime -> {
# required_fields: {plugin_loader: PluginLoader},
# optional_fields: {},
# invariants: ["Must use belief_scope for logging"]
# }
# @TEST_FIXTURE valid_module -> {"manager_initialized": true}
# @TEST_EDGE missing_required_field -> {"plugin_loader": null}
# @TEST_EDGE empty_response -> {"tasks": []}
# @TEST_EDGE invalid_type -> {"plugin_loader": "string_instead_of_object"}
# @TEST_EDGE external_failure -> {"db_unavailable": true}
# @TEST_INVARIANT logger_compliance -> verifies: [valid_module]
# @RATIONALE Decomposed from 708-line monolithic module into four focused modules (TaskGraph,
# EventBus, JobLifecycle, and this facade) to satisfy INV_7. TaskManager now delegates
# to sub-services while preserving the public API contract.
@@ -73,7 +62,7 @@ class TaskManager:
JobLifecycle (state machine) into a single TaskManager interface.
"""
# #region Core.Manager.Init [C:4] [TYPE Function] [C:5]
# #region Core.Manager.Init [C:4] [TYPE Function]
# @BRIEF Initialize sub-services, create add_log callback, start background flusher.
# @PRE plugin_loader is initialized.
# @POST TaskManager is ready to accept tasks.
@@ -168,7 +157,7 @@ class TaskManager:
# ── Task CRUD delegates to TaskGraph ──
# #region Core.Manager.GetTask [C:2] [TYPE Function] [C:2]
# #region Core.Manager.GetTask [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Retrieves a task by its ID.
def get_task(self, task_id: str) -> Task | None:
@@ -181,7 +170,7 @@ class TaskManager:
return self.graph.get_all_tasks()
# #endregion Core.Manager.GetAllTasks
# #region Core.Manager.GetTasks [C:3] [TYPE Function] [C:3]
# #region Core.Manager.GetTasks [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
def get_tasks(
@@ -196,14 +185,14 @@ class TaskManager:
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only, search)
# #endregion Core.Manager.GetTasks
# #region Core.Manager.LoadPersistedTasks [C:2] [TYPE Function] [C:2]
# #region Core.Manager.LoadPersistedTasks [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Load persisted tasks using persistence service.
def load_persisted_tasks(self) -> None:
self.graph.load_persisted_tasks(limit=100)
# #endregion Core.Manager.LoadPersistedTasks
# #region Core.Manager.ClearTasks [C:4] [TYPE Function] [C:4]
# #region Core.Manager.ClearTasks [C:4] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Clears tasks based on status filter (also deletes associated logs).
# @SIDE_EFFECT Removes tasks from registry and persistence; cancels futures.
@@ -235,7 +224,7 @@ class TaskManager:
# ── Log delegates to EventBus ──
# #region Core.Manager.GetTaskLogs [C:3] [TYPE Function] [C:3]
# #region Core.Manager.GetTaskLogs [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Retrieves logs for a specific task (from memory or persistence).
def get_task_logs(
@@ -249,14 +238,14 @@ class TaskManager:
)
# #endregion Core.Manager.GetTaskLogs
# #region Core.Manager.GetTaskLogStats [C:2] [TYPE Function] [C:2]
# #region Core.Manager.GetTaskLogStats [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Get statistics about logs for a task.
def get_task_log_stats(self, task_id: str) -> LogStats:
return self.event_bus.get_task_log_stats(task_id)
# #endregion Core.Manager.GetTaskLogStats
# #region Core.Manager.GetTaskLogSources [C:2] [TYPE Function] [C:2]
# #region Core.Manager.GetTaskLogSources [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Get unique sources for a task's logs.
def get_task_log_sources(self, task_id: str) -> list[str]:
@@ -265,14 +254,14 @@ class TaskManager:
# ── Subscription delegates to EventBus ──
# #region Core.Manager.SubscribeLogs [C:2] [TYPE Function] [C:2]
# #region Core.Manager.SubscribeLogs [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Subscribes to real-time logs for a task.
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
return await self.event_bus.subscribe_logs(task_id)
# #endregion Core.Manager.SubscribeLogs
# #region Core.Manager.UnsubscribeLogs [C:2] [TYPE Function] [C:2]
# #region Core.Manager.UnsubscribeLogs [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Unsubscribes from real-time logs for a task.
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
@@ -281,28 +270,28 @@ class TaskManager:
# ── Status subscribers ──
# #region Core.Manager.SubscribeStatus [C:2] [TYPE Function] [C:2]
# #region Core.Manager.SubscribeStatus [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Subscribes to real-time status updates for a task.
async def subscribe_status(self, task_id: str) -> asyncio.Queue:
return await self.event_bus.subscribe_status(task_id)
# #endregion Core.Manager.SubscribeStatus
# #region Core.Manager.UnsubscribeStatus [C:2] [TYPE Function] [C:2]
# #region Core.Manager.UnsubscribeStatus [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Unsubscribes from status updates for a task.
def unsubscribe_status(self, task_id: str, queue: asyncio.Queue):
self.event_bus.unsubscribe_status(task_id, queue)
# #endregion Core.Manager.UnsubscribeStatus
# #region Core.Manager.SubscribeTaskEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.SubscribeTaskEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Subscribes to global task events (all task status changes).
async def subscribe_task_events(self) -> asyncio.Queue:
return await self.event_bus.subscribe_task_events()
# #endregion Core.Manager.SubscribeTaskEvents
# #region Core.Manager.UnsubscribeTaskEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.UnsubscribeTaskEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Unsubscribes from global task events.
def unsubscribe_task_events(self, queue: asyncio.Queue):
@@ -311,7 +300,7 @@ class TaskManager:
# ── Lifecycle delegates to JobLifecycle ──
# #region Core.Manager.CreateTask [C:4] [TYPE Function] [C:4]
# #region Core.Manager.CreateTask [C:4] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Creates and queues a new task for execution.
async def create_task(
@@ -330,7 +319,7 @@ class TaskManager:
return task
# #endregion Core.Manager.CreateTask
# #region Core.Manager.RunTask [C:4] [TYPE Function] [C:4]
# #region Core.Manager.RunTask [C:4] [TYPE Function]
# @BRIEF Internal method to execute a task with TaskContext support (delegates to lifecycle).
# Tracks the asyncio.Task for management/cancellation.
async def _run_task(self, task_id: str):
@@ -344,7 +333,7 @@ class TaskManager:
self._async_tasks.pop(task_id, None)
# #endregion Core.Manager.RunTask
# #region Core.Manager.CancelTask [C:3] [TYPE Function] [C:3]
# #region Core.Manager.CancelTask [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Cancel a running task by ID.
# @PRE Task must be currently tracked as running.
@@ -358,28 +347,28 @@ class TaskManager:
return True
# #endregion Core.Manager.CancelTask
# #region Core.Manager.ResolveTask [C:3] [TYPE Function] [C:3]
# #region Core.Manager.ResolveTask [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Resumes a task that is awaiting mapping.
async def resolve_task(self, task_id: str, resolution_params: dict[str, Any]):
await self.lifecycle.resolve_task(task_id, resolution_params)
# #endregion Core.Manager.ResolveTask
# #region Core.Manager.WaitForResolution [C:3] [TYPE Function] [C:3]
# #region Core.Manager.WaitForResolution [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Pauses execution and waits for a resolution signal.
async def wait_for_resolution(self, task_id: str):
await self.lifecycle.wait_for_resolution(task_id)
# #endregion Core.Manager.WaitForResolution
# #region Core.Manager.WaitForInput [C:3] [TYPE Function] [C:3]
# #region Core.Manager.WaitForInput [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Pauses execution and waits for user input.
async def wait_for_input(self, task_id: str):
await self.lifecycle.wait_for_input(task_id)
# #endregion Core.Manager.WaitForInput
# #region Core.Manager.AwaitInput [C:3] [TYPE Function] [C:3]
# #region Core.Manager.AwaitInput [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Transition a task to AWAITING_INPUT state with input request.
async def await_input(self, task_id: str, input_request: dict[str, Any]) -> None:
@@ -389,7 +378,7 @@ class TaskManager:
)
# #endregion Core.Manager.AwaitInput
# #region Core.Manager.ResumeTaskWithPassword [C:3] [TYPE Function] [C:3]
# #region Core.Manager.ResumeTaskWithPassword [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Resume a task that is awaiting input with provided passwords.
async def resume_task_with_password(
@@ -407,7 +396,7 @@ class TaskManager:
)
# #endregion Core.Manager.ResumeTaskWithPassword
# #region Core.Manager.RetryTask [C:3] [TYPE Function] [C:3]
# #region Core.Manager.RetryTask [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Retry a failed task by resetting state and re-queuing execution.
async def retry_task(self, task_id: str) -> Task:
@@ -426,21 +415,21 @@ class TaskManager:
# ── Maintenance event delegates to EventBus ──
# #region Core.Manager.SubscribeMaintenanceEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.SubscribeMaintenanceEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Subscribes to global maintenance events.
async def subscribe_maintenance_events(self) -> asyncio.Queue:
return await self.event_bus.subscribe_maintenance_events()
# #endregion Core.Manager.SubscribeMaintenanceEvents
# #region Core.Manager.UnsubscribeMaintenanceEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.UnsubscribeMaintenanceEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Unsubscribes from global maintenance events.
def unsubscribe_maintenance_events(self, queue: asyncio.Queue):
self.event_bus.unsubscribe_maintenance_events(queue)
# #endregion Core.Manager.UnsubscribeMaintenanceEvents
# #region Core.Manager.BroadcastMaintenanceEvent [C:2] [TYPE Function] [C:2]
# #region Core.Manager.BroadcastMaintenanceEvent [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Broadcast a maintenance event to all subscribers.
async def broadcast_maintenance_event(self, event: dict):
@@ -449,14 +438,14 @@ class TaskManager:
# ── Dataset event delegates to JobLifecycle ──
# #region Core.Manager.SubscribeDatasetEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.SubscribeDatasetEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Subscribe to dataset.updated events for an environment.
async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue:
return await self.lifecycle.subscribe_dataset_events(env_id)
# #endregion Core.Manager.SubscribeDatasetEvents
# #region Core.Manager.UnsubscribeDatasetEvents [C:2] [TYPE Function] [C:2]
# #region Core.Manager.UnsubscribeDatasetEvents [C:2] [TYPE Function]
# @ingroup TaskManager
# @BRIEF Unsubscribe from dataset.updated events.
def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue):

View File

@@ -45,19 +45,6 @@ from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus
# @RELATION DEPENDS_ON -> [Core.Graph.TaskGraph]
# @INVARIANT Persistence must handle potentially missing task fields natively.
#
# @TEST_CONTRACT TaskPersistenceContract ->
# {
# required_fields: {},
# invariants: [
# "persist_task creates or updates a record",
# "load_tasks retrieves valid Task instances",
# "delete_tasks correctly removes records from the database"
# ]
# }
# @TEST_FIXTURE valid_task_persistence -> {"task_id": "123", "status": "PENDING"}
# @TEST_EDGE persist_invalid_task_type -> raises Exception
# @TEST_EDGE load_corrupt_json_params -> handled gracefully
# @TEST_INVARIANT accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]
class TaskPersistenceService:
# #region Core.Persistence.JsonLoadIfNeeded [TYPE Function] [C:1]
# @BRIEF: Safely load JSON strings from DB if necessary
@@ -137,7 +124,7 @@ class TaskPersistenceService:
return str(env.id)
return None
# #endregion Core.Persistence.ResolveEnvironmentId
# #region Core.Persistence.Init [C:2] [TYPE Function] [C:3]
# #region Core.Persistence.TaskPersistenceService.__init__ [C:2] [TYPE Function]
# @BRIEF: Initializes the persistence service.
# @PRE None.
# @POST Service is ready.
@@ -145,8 +132,8 @@ class TaskPersistenceService:
with belief_scope("TaskPersistenceService.__init__"):
# Use the unified SessionLocal from database.py.
pass
# #endregion Core.Persistence.Init
# #region Core.Persistence.PersistTask [C:3] [TYPE Function] [C:3]
# #endregion Core.Persistence.TaskPersistenceService.__init__
# #region Core.Persistence.PersistTask [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Persists or updates a single task in the database.
# @PRE isinstance(task, Task)
@@ -222,7 +209,7 @@ class TaskPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.PersistTask
# #region Core.Persistence.PersistTasks [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.PersistTasks [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Persists multiple tasks.
# @PRE isinstance(tasks, list)
@@ -234,7 +221,7 @@ class TaskPersistenceService:
for task in tasks:
self.persist_task(task)
# #endregion Core.Persistence.PersistTasks
# #region Core.Persistence.LoadTasks [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.LoadTasks [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Loads tasks from the database.
# @PRE limit is an integer.
@@ -300,7 +287,7 @@ class TaskPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.LoadTasks
# #region Core.Persistence.DeleteTasks [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.DeleteTasks [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Deletes specific tasks from the database.
# @PRE task_ids is a list of strings.
@@ -345,31 +332,19 @@ class TaskPersistenceService:
# efficient SQL filtering/aggregation. File-per-task logging was rejected — loses
# cross-task query capability and complicates lifecycle cleanup.
#
# @TEST_CONTRACT TaskLogPersistenceContract ->
# {
# required_fields: {},
# invariants: [
# "add_logs efficiently saves logs to the database",
# "get_logs retrieves properly filtered LogEntry objects"
# ]
# }
# @TEST_FIXTURE valid_log_batch -> {"task_id": "123", "logs": [{"level": "INFO", "message": "msg"}]}
# @TEST_EDGE empty_log_list -> no-op behavior
# @TEST_EDGE add_logs_db_error -> rollback and log error
# @TEST_INVARIANT accurate_log_aggregation -> verifies: [valid_log_batch]
class TaskLogPersistenceService:
"""
Service for persisting and querying task logs.
Supports batch inserts, filtering, and statistics.
"""
# #region Core.Persistence.Init [C:2] [TYPE Function] [C:3]
# #region Core.Persistence.TaskLogPersistenceService.__init__ [C:2] [TYPE Function]
# @BRIEF: Initializes the TaskLogPersistenceService
# @PRE config is provided or defaults are used
# @POST Service is ready for log persistence
def __init__(self, config=None):
pass
# #endregion Core.Persistence.Init
# #region Core.Persistence.AddLogs [C:3] [TYPE Function] [C:3]
# #endregion Core.Persistence.TaskLogPersistenceService.__init__
# #region Core.Persistence.AddLogs [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Batch insert log entries for a task.
# @PRE logs is a list of LogEntry objects.
@@ -425,7 +400,7 @@ class TaskLogPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.AddLogs
# #region Core.Persistence.GetLogs [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.GetLogs [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Query logs for a task with filtering and pagination.
# @PRE task_id is a valid task ID.
@@ -480,7 +455,7 @@ class TaskLogPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.GetLogs
# #region Core.Persistence.GetLogStats [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.GetLogStats [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Get statistics about logs for a task.
# @PRE task_id is a valid task ID.
@@ -525,7 +500,7 @@ class TaskLogPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.GetLogStats
# #region Core.Persistence.GetSources [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.GetSources [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Get unique sources for a task's logs.
# @PRE task_id is a valid task ID.
@@ -604,7 +579,7 @@ class TaskLogPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.IterLogsForExport
# #region Core.Persistence.DeleteLogsForTask [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.DeleteLogsForTask [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Delete all logs for a specific task.
# @PRE task_id is a valid task ID.
@@ -628,7 +603,7 @@ class TaskLogPersistenceService:
finally:
session.close()
# #endregion Core.Persistence.DeleteLogsForTask
# #region Core.Persistence.DeleteLogsForTasks [C:3] [TYPE Function] [C:3]
# #region Core.Persistence.DeleteLogsForTasks [C:3] [TYPE Function]
# @ingroup TaskManager
# @BRIEF: Delete all logs for multiple tasks.
# @PRE task_ids is a list of task IDs.

View File

@@ -23,19 +23,6 @@ from ..logger import logger as main_cot_logger # noqa: TID252
# @INVARIANT All log calls include the task_id and source.
# @UX_STATE Idle -> Logging -> (system records log)
#
# @TEST_CONTRACT TaskLoggerContract ->
# {
# required_fields: {task_id: str, add_log_fn: Callable},
# optional_fields: {source: str},
# invariants: [
# "All specific log methods (info, error) delegate to _log",
# "with_source creates a new logger with the same task_id"
# ]
# }
# @TEST_FIXTURE valid_task_logger -> {"task_id": "test_123", "add_log_fn": lambda *args: None, "source": "test_plugin"}
# @TEST_EDGE missing_task_id -> raises TypeError
# @TEST_EDGE invalid_add_log_fn -> raises TypeError
# @TEST_INVARIANT consistent_delegation -> verifies: [valid_task_logger]
class TaskLogger:
"""
A dedicated logger for tasks that automatically tags logs with source attribution.

View File

@@ -23,7 +23,7 @@ from ..logger import belief_scope, logger as app_logger
# #region Core.Network.SupersetAPIError [C:1] [TYPE Class]
# @BRIEF Base exception for all Superset API related errors.
class SupersetAPIError(Exception):
# #region Core.Network.Init [TYPE Function] [C:1]
# #region Core.Network.SupersetAPIError.__init__ [TYPE Function] [C:1]
# @BRIEF: Initializes the exception with a message and context.
# @PRE message is a string, context is a dict.
# @POST Exception is initialized with context.
@@ -31,48 +31,48 @@ class SupersetAPIError(Exception):
with belief_scope("SupersetAPIError.__init__"):
self.context = context
super().__init__(f"[API_FAILURE] {message} | Context: {self.context}")
# #endregion Core.Network.Init
# #endregion Core.Network.SupersetAPIError.__init__
# #endregion Core.Network.SupersetAPIError
# #region Core.Network.AuthenticationError [C:1] [TYPE Class]
# @BRIEF Exception raised when authentication fails.
class AuthenticationError(SupersetAPIError):
# #region Core.Network.Init [TYPE Function] [C:1]
# #region Core.Network.AuthenticationError.__init__ [TYPE Function] [C:1]
# @BRIEF: Initializes the authentication error.
# @PRE message is a string, context is a dict.
# @POST AuthenticationError is initialized.
def __init__(self, message: str = "Authentication failed", **context: Any):
with belief_scope("AuthenticationError.__init__"):
super().__init__(message, type="authentication", **context)
# #endregion Core.Network.Init
# #endregion Core.Network.AuthenticationError.__init__
# #endregion Core.Network.AuthenticationError
# #region Core.Network.PermissionDeniedError [TYPE Class]
# @defgroup Core Module group.
# @ingroup Core
# @BRIEF Exception raised when access is denied.
class PermissionDeniedError(AuthenticationError):
# #region Core.Network.Init [TYPE Function]
# #region Core.Network.PermissionDeniedError.__init__ [TYPE Function]
# @BRIEF: Initializes the permission denied error.
# @PRE message is a string, context is a dict.
# @POST PermissionDeniedError is initialized.
def __init__(self, message: str = "Permission denied", **context: Any):
with belief_scope("PermissionDeniedError.__init__"):
super().__init__(message, **context)
# #endregion Core.Network.Init
# #endregion Core.Network.PermissionDeniedError.__init__
# #endregion Core.Network.PermissionDeniedError
# #region Core.Network.DashboardNotFoundError [TYPE Class]
# @defgroup Core Module group.
# @ingroup Core
# @BRIEF Exception raised when a dashboard cannot be found.
class DashboardNotFoundError(SupersetAPIError):
# #region Core.Network.Init [TYPE Function]
# #region Core.Network.DashboardNotFoundError.__init__ [TYPE Function]
# @BRIEF: Initializes the not found error with resource ID.
# @PRE resource_id is provided.
# @POST DashboardNotFoundError is initialized.
def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any):
with belief_scope("DashboardNotFoundError.__init__"):
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
# #endregion Core.Network.Init
# #endregion Core.Network.DashboardNotFoundError.__init__
# #endregion Core.Network.DashboardNotFoundError
# #region Core.Network.NetworkError [TYPE Class]
# @defgroup Core Module group.
# @ingroup Core
# @BRIEF Exception raised when a network level error occurs.
class NetworkError(Exception):
# #region NetworkError.__init__ [TYPE Function]
@@ -87,7 +87,7 @@ class NetworkError(Exception):
# #endregion NetworkError.__init__
# #endregion Core.Network.NetworkError
# #region Core.Network.SupersetAuthCache [TYPE Class]
# @defgroup Core Module group.
# @ingroup Core
# @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials.
# @PRE base_url and username are stable strings.
# @POST Cached entries expire automatically by TTL and can be reused across requests.

View File

@@ -39,7 +39,7 @@ class SupersetParsedContext:
partial_recovery: bool = False
dataset_payload: dict[str, Any] | None = None
# #endregion Core.Base.SupersetParsedContext
# #region Core.Base.SupersetContextExtractorBase [C:4] [TYPE Class]
# #region Core.Base.SupersetContextExtractorBase.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
# @RELATION DEPENDS_ON -> [Core.ConfigModels.Environment]
@@ -191,4 +191,4 @@ class SupersetContextExtractorBase:
query_state[key] = decoded_value
return query_state
# #endregion SupersetContextExtractorBase._decode_query_state
# #endregion Core.Base.SupersetContextExtractorBase
# #endregion Core.Base.SupersetContextExtractorBase.Class

View File

@@ -14,7 +14,7 @@ from ...logger import logger as app_logger
app_logger = cast(Any, app_logger)
# #endregion Core.Filters.FiltersImports
# #region Core.Filters.SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
# #region Core.Filters.SupersetContextFiltersExtractMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor.
class SupersetContextFiltersExtractMixin:
@@ -236,4 +236,4 @@ class SupersetContextFiltersExtractMixin:
)
return imported_filters
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
# #endregion Core.Filters.SupersetContextFiltersExtractMixin
# #endregion Core.Filters.SupersetContextFiltersExtractMixin.Class

View File

@@ -16,7 +16,7 @@ from ._base import SupersetParsedContext
logger = cast(Any, logger)
# #endregion Core.Parsing.ParsingImports
# #region Core.Parsing.SupersetContextParsingMixin [C:4] [TYPE Class]
# #region Core.Parsing.SupersetContextParsingMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor.
class SupersetContextParsingMixin:
@@ -356,4 +356,4 @@ class SupersetContextParsingMixin:
unresolved_references.append("dashboard_dataset_binding_missing")
return None, unresolved_references
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
# #endregion Core.Parsing.SupersetContextParsingMixin
# #endregion Core.Parsing.SupersetContextParsingMixin.Class

View File

@@ -17,7 +17,7 @@ from ._base import SupersetParsedContext
logger = cast(Any, logger)
# #endregion Core.Recovery.RecoveryImports
# #region Core.Recovery.SupersetContextRecoveryMixin [C:4] [TYPE Class]
# #region Core.Recovery.SupersetContextRecoveryMixin.Class [C:4] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor.
class SupersetContextRecoveryMixin:
@@ -286,4 +286,4 @@ class SupersetContextRecoveryMixin:
"notes": str(payload.get("notes") or default_note),
}
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
# #endregion Core.Recovery.SupersetContextRecoveryMixin
# #endregion Core.Recovery.SupersetContextRecoveryMixin.Class

View File

@@ -14,7 +14,7 @@ from ...logger import belief_scope, logger
logger = cast(Any, logger)
# #endregion Core.Templates.TemplatesImports
# #region Core.Templates.SupersetContextTemplatesMixin [C:3] [TYPE Class]
# #region Core.Templates.SupersetContextTemplatesMixin.Class [C:3] [TYPE Class]
# @defgroup Core Module group.
# @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor.
class SupersetContextTemplatesMixin:
@@ -227,4 +227,4 @@ class SupersetContextTemplatesMixin:
except ValueError:
return normalized_literal
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
# #endregion Core.Templates.SupersetContextTemplatesMixin
# #endregion Core.Templates.SupersetContextTemplatesMixin.Class

View File

@@ -101,7 +101,6 @@ class MigrationJob(Base):
# #region Models.Mapping.ResourceMapping [C:3] [TYPE Class]
# @defgroup Models Module group.
# @BRIEF Maps a universal UUID for a resource to its actual ID on a specific environment.
# @TEST_DATA: resource_mapping_record -> {'environment_id': 'prod-env-1', 'resource_type': 'chart', 'uuid': '123e4567-e89b-12d3-a456-426614174000', 'remote_integer_id': '42'}
# @RELATION DEPENDS_ON -> Models.Mapping.MappingModels
class ResourceMapping(Base):
__tablename__ = "resource_mappings"

View File

@@ -53,18 +53,6 @@ class ReportStatus(str, Enum):
# @INVARIANT The properties accurately describe error state.
# @BRIEF Error and recovery context for failed/partial reports.
#
# @TEST_CONTRACT ErrorContextModel ->
# {
# required_fields: {
# message: str
# },
# optional_fields: {
# code: str,
# next_actions: list[str]
# }
# }
# @TEST_FIXTURE basic_error -> {"message": "Connection timeout", "code": "ERR_504", "next_actions": ["retry"]}
# @TEST_EDGE missing_message -> {"code": "ERR_504"}
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
class ErrorContext(BaseModel):
code: str | None = None
@@ -80,35 +68,6 @@ class ErrorContext(BaseModel):
# @INVARIANT Must represent canonical task record attributes.
# @BRIEF Canonical normalized report envelope for one task execution.
#
# @TEST_CONTRACT TaskReportModel ->
# {
# required_fields: {
# report_id: str,
# task_id: str,
# task_type: TaskType,
# status: ReportStatus,
# updated_at: datetime,
# summary: str
# },
# invariants: [
# "report_id is a non-empty string",
# "task_id is a non-empty string",
# "summary is a non-empty string"
# ]
# }
# @TEST_FIXTURE valid_task_report ->
# {
# report_id: "rep-123",
# task_id: "task-456",
# task_type: "migration",
# status: "success",
# updated_at: "2026-02-26T12:00:00Z",
# summary: "Migration completed successfully"
# }
# @TEST_EDGE empty_report_id -> {"report_id": " ", "task_id": "task-456", "task_type": "migration", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}
# @TEST_EDGE empty_summary -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "migration", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": ""}
# @TEST_EDGE invalid_task_type -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "invalid_type", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}
# @TEST_INVARIANT non_empty_validators -> verifies: [empty_report_id, empty_summary]
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
class TaskReport(BaseModel):
report_id: str
@@ -139,24 +98,6 @@ class TaskReport(BaseModel):
# @INVARIANT Time and pagination queries are mutually consistent.
# @BRIEF Query object for server-side report filtering, sorting, and pagination.
#
# @TEST_CONTRACT ReportQueryModel ->
# {
# optional_fields: {
# page: int, page_size: int, task_types: list[TaskType], statuses: list[ReportStatus],
# time_from: datetime, time_to: datetime, search: str, sort_by: str, sort_order: str
# },
# invariants: [
# "page >= 1", "1 <= page_size <= 100",
# "sort_by in {'updated_at', 'status', 'task_type'}",
# "sort_order in {'asc', 'desc'}",
# "time_from <= time_to if both exist"
# ]
# }
# @TEST_FIXTURE valid_query -> {"page": 1, "page_size":20, "sort_by": "updated_at", "sort_order": "desc"}
# @TEST_EDGE invalid_page_size_large -> {"page_size": 150}
# @TEST_EDGE invalid_sort_by -> {"sort_by": "unknown_field"}
# @TEST_EDGE invalid_time_range -> {"time_from": "2026-02-26T12:00:00Z", "time_to": "2026-02-25T12:00:00Z"}
# @TEST_INVARIANT attribute_constraints_enforced -> verifies: [invalid_page_size_large, invalid_sort_by, invalid_time_range]
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
class ReportQuery(BaseModel):
page: int = Field(default=1, ge=1)
@@ -199,15 +140,6 @@ class ReportQuery(BaseModel):
# @INVARIANT Represents paginated data correctly.
# @BRIEF Paginated collection of normalized task reports.
#
# @TEST_CONTRACT ReportCollectionModel ->
# {
# required_fields: {
# items: list[TaskReport], total: int, page: int, page_size: int, has_next: bool, applied_filters: ReportQuery
# },
# invariants: ["total >= 0", "page >= 1", "page_size >= 1"]
# }
# @TEST_FIXTURE empty_collection -> {"items": [], "total": 0, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}}
# @TEST_EDGE negative_total -> {"items": [], "total": -5, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}}
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
class ReportCollection(BaseModel):
items: list[TaskReport]
@@ -226,13 +158,6 @@ class ReportCollection(BaseModel):
# @INVARIANT Incorporates a report and logs correctly.
# @BRIEF Detailed report representation including diagnostics and recovery actions.
#
# @TEST_CONTRACT ReportDetailViewModel ->
# {
# required_fields: {report: TaskReport},
# optional_fields: {timeline: list[dict], diagnostics: dict, next_actions: list[str]}
# }
# @TEST_FIXTURE valid_detail -> {"report": {"report_id": "rep-1", "task_id": "task-1", "task_type": "backup", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}}
# @TEST_EDGE missing_report -> {}
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
class ReportDetailView(BaseModel):
report: TaskReport
@@ -249,7 +174,6 @@ class ReportDetailView(BaseModel):
# @BRIEF Aggregated task counts by type and status for the summary dashboard.
# @DATA_CONTRACT Output: TaskSummary → frontend interface TaskSummary
# @RELATION DEPENDS_ON -> [Models.Report.TaskReport]
# @TEST_CONTRACT TaskSummaryModel -> { invariants: ["all counts >= 0", "total == sum of all status counts across all types"] }
class StatusCounts(BaseModel):
"""Counts per status for a single task type."""
pending: int = Field(default=0, ge=0)

View File

@@ -39,60 +39,6 @@ class TaskRecord(Base):
# @RELATION DEPENDS_ON -> Models.Task.TaskRecord
# @INVARIANT Each log entry belongs to exactly one task.
#
# @TEST_CONTRACT TaskLogCreate ->
# {
# required_fields: {
# task_id: str,
# timestamp: datetime,
# level: str,
# source: str,
# message: str
# },
# optional_fields: {
# metadata_json: str,
# id: int
# },
# invariants: [
# "task_id matches an existing TaskRecord.id"
# ]
# }
#
# @TEST_FIXTURE basic_info_log ->
# {
# task_id: "00000000-0000-0000-0000-000000000000",
# timestamp: "2026-02-26T12:00:00Z",
# level: "INFO",
# source: "system",
# message: "Task initialization complete"
# }
#
# @TEST_EDGE missing_required_field ->
# {
# timestamp: "2026-02-26T12:00:00Z",
# level: "ERROR",
# source: "system",
# message: "Missing task_id"
# }
#
# @TEST_EDGE invalid_type ->
# {
# task_id: "00000000-0000-0000-0000-000000000000",
# timestamp: "2026-02-26T12:00:00Z",
# level: 500,
# source: "system",
# message: "Integer level"
# }
#
# @TEST_EDGE empty_message ->
# {
# task_id: "00000000-0000-0000-0000-000000000000",
# timestamp: "2026-02-26T12:00:00Z",
# level: "DEBUG",
# source: "system",
# message: ""
# }
#
# @TEST_INVARIANT exact_one_task_association -> verifies: [basic_info_log, missing_required_field]
class TaskLogRecord(Base):
__tablename__ = "task_logs"

View File

@@ -0,0 +1,16 @@
# #region Plugin.Service.Constants [C:1] [TYPE Module] [SEMANTICS llm,screenshot,timeouts]
# @defgroup LLMAnalysis Module group.
# @BRIEF Named timeout constants for LLM analysis and screenshot services.
# @LAYER Plugin
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants. Zero remaining numeric timeout literals.
# @REJECTED Keeping inline numeric timeout literals was rejected — they create a maintenance hazard where any timeout adjustment requires grep-and-fix across 1700+ lines; named constants centralize tuning and make timeout configuration auditable.
# Timeout constants (milliseconds unless noted)
PLAYWRIGHT_NAVIGATION_TIMEOUT_MS = 30000
PLAYWRIGHT_WAIT_TIMEOUT_MS = 10000
PLAYWRIGHT_SHORT_TIMEOUT_MS = 5000
HTTP_REQUEST_TIMEOUT_MS = 60000
SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
# #endregion Plugin.Service.Constants

View File

@@ -0,0 +1,215 @@
# #region Plugin.Service.DatasetHealthModule [C:3] [TYPE Module] [SEMANTICS llm,dataset,health,superset]
# @defgroup LLMAnalysis Module group.
# @BRIEF Dataset accessibility and chart-data health checks via Superset API.
# @LAYER Plugin
import asyncio
import json
from typing import Any
# #region Plugin.Service.DatasetHealthChecker [C:3] [TYPE Class]
# @defgroup LLMAnalysis Module group.
# @BRIEF Checks dataset accessibility and KXD connectivity via Superset API.
# @LAYER Service
# @RELATION CALLS -> [Core.Init.SupersetClient]
# @INVARIANT Every unique dataset referenced by dashboard charts is checked.
# @RATIONALE Without dataset health checking, silent KXD errors (connection refused, timeout)
# are invisible to the LLM validation. Screenshot captures visual state but doesn't
# verify that data actually arrived (vs. cache).
class DatasetHealthChecker:
# #region DatasetHealthChecker.__init__ [C:2] [TYPE Function]
# @BRIEF Initialize with a SupersetClient-compatible instance.
# @PRE client is a SupersetClient (sync, wrapped via asyncio.to_thread) or AsyncSupersetClient.
# @POST self.client is ready for health checks.
def __init__(self, client: Any):
self.client = client
# #endregion DatasetHealthChecker.__init__
# #region DatasetHealthChecker._call_sync [C:2] [TYPE Function]
# @BRIEF Wrap a sync client method call in asyncio.to_thread for async compat.
# @PRE method is a callable on self.client.
# @POST Returns the result of method(*args, **kwargs) executed in a thread.
@staticmethod
async def _call_sync(method, *args: Any, **kwargs: Any) -> Any:
"""Call a potentially sync method in a thread, or await if already async."""
if asyncio.iscoroutinefunction(method):
return await method(*args, **kwargs)
return await asyncio.to_thread(method, *args, **kwargs)
# #endregion DatasetHealthChecker._call_sync
# #region DatasetHealthChecker.check_dataset_health [C:3] [TYPE Function]
# @BRIEF Fetch dataset metadata and verify level 1-2 accessibility.
# @PRE dataset_id is a valid Superset dataset ID.
# @POST Returns dict with level 1-2 health fields.
# @SIDE_EFFECT Calls GET /api/v1/dataset/{id} via client.get_dataset
async def check_dataset_health(self, dataset_id: int) -> dict:
"""
Check a single dataset's accessibility (levels 1-2 per FR-044).
Level 1: metadata_accessible — HTTP 200 from GET /api/v1/dataset/{id}
Level 2: datasource_resolvable — database info available
Returns dict with:
dataset_id, dataset_name, database_name, backend, kind,
metadata_accessible (bool), error (str|None)
"""
try:
dataset = await self._call_sync(self.client.get_dataset, dataset_id)
# The response from Superset may have a 'result' wrapper or be flat
result_data = dataset.get("result", dataset) if isinstance(dataset, dict) else {}
# Extract database info
database = result_data.get("database", {}) or {}
result = {
"dataset_id": dataset_id,
"dataset_name": result_data.get("table_name", f"dataset_{dataset_id}"),
"database_name": database.get("database_name", "unknown"),
"backend": database.get("backend", "unknown"),
"kind": result_data.get("kind", "physical"),
"metadata_accessible": True,
"error": None,
}
return result
except Exception as e:
return {
"dataset_id": dataset_id,
"dataset_name": f"dataset_{dataset_id}",
"database_name": "unknown",
"backend": "unknown",
"kind": "unknown",
"metadata_accessible": False,
"error": str(e),
}
# #endregion DatasetHealthChecker.check_dataset_health
# #region DatasetHealthChecker.check_chart_data [C:3] [TYPE Function]
# @BRIEF Execute chart data query (level 3-4 per FR-044).
# @PRE chart_id is valid, form_data is constructed from chart params.
# @POST Returns dict with execution result.
# @SIDE_EFFECT Calls POST /api/v1/chart/data via client.network.request
async def check_chart_data(self, chart_id: int, form_data: dict) -> dict:
"""
Execute a chart query to verify data returns.
Level 3: query_executable — POST /api/v1/chart/data succeeds
Level 4: data_returned — row_count > 0 or no error
Returns dict with:
chart_id, executed (bool), duration_ms (int|None),
row_count (int|None), error (str|None)
"""
import time
start = time.time()
try:
# Use the client's network layer for the chart data POST.
# For sync SupersetClient: network.request(...) is synchronous.
# We wrap it via asyncio.to_thread if it's a sync method.
payload = json.dumps(form_data)
headers = {"Content-Type": "application/json"}
network_request = self.client.network.request
result = await self._call_sync(
network_request,
"POST",
"/chart/data",
data=payload,
headers=headers,
)
duration_ms = int((time.time() - start) * 1000)
# Normalize response — may have 'result' wrapper
rows = []
if isinstance(result, dict):
rows = result.get("result", []) or []
elif isinstance(result, list):
rows = result
return {
"chart_id": chart_id,
"executed": True,
"duration_ms": duration_ms,
"row_count": len(rows),
"error": None,
}
except Exception as e:
duration_ms = int((time.time() - start) * 1000)
return {
"chart_id": chart_id,
"executed": False,
"duration_ms": duration_ms,
"row_count": None,
"error": str(e),
}
# #endregion DatasetHealthChecker.check_chart_data
# #region DatasetHealthChecker.check_dashboard_datasets [C:3] [TYPE Function]
# @BRIEF For every unique dataset in dashboard charts, check health.
# @PRE chart_list has chart dicts with slice_id and datasource_id.
# @POST Returns dict with datasets and optional chart_data lists.
async def check_dashboard_datasets(
self,
chart_list: list[dict],
execute_chart_data: bool = False,
) -> dict:
"""
Check all unique datasets referenced by dashboard charts.
Args:
chart_list: list of chart dicts with at least
{'slice_id', 'datasource_id', 'viz_type', 'params'}
execute_chart_data: if True, also execute chart queries (level 3-4)
Returns:
{datasets: [...], chart_data: [...]}
"""
# Collect unique datasource_ids
unique_ds_ids: set[int] = set()
for chart in chart_list:
ds_id = chart.get("datasource_id")
if ds_id is not None:
unique_ds_ids.add(int(ds_id))
# Check each dataset
dataset_results: list[dict] = []
for ds_id in sorted(unique_ds_ids):
result = await self.check_dataset_health(ds_id)
# Map affected charts
affected_charts = [{"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")} for c in chart_list if c.get("datasource_id") == ds_id]
result["affected_charts"] = affected_charts
dataset_results.append(result)
# Optionally execute chart data
chart_data_results: list[dict] = []
if execute_chart_data:
for chart in chart_list:
chart_id = chart.get("slice_id")
params = chart.get("params", {})
if isinstance(params, str):
params = json.loads(params)
form_data = {
"slice_id": chart_id,
"viz_type": chart.get("viz_type", "table"),
"datasource_id": chart.get("datasource_id"),
"datasource_type": chart.get("datasource_type", "table"),
"granularity_sqla": params.get("granularity_sqla"),
"time_range": params.get("time_range", "Last 30 days"),
"metrics": params.get("metrics", []),
"groupby": params.get("groupby", []),
"adhoc_filters": params.get("adhoc_filters", []),
}
result = await self.check_chart_data(chart_id, form_data)
chart_data_results.append(result)
return {
"datasets": dataset_results,
"chart_data": chart_data_results,
}
# #endregion DatasetHealthChecker.check_dashboard_datasets
# #endregion Plugin.Service.DatasetHealthChecker
# #endregion Plugin.Service.DatasetHealthModule

View File

@@ -0,0 +1,351 @@
# #region Plugin.Service.LLMClient.AnalysisModule [C:4] [TYPE Module] [SEMANTICS llm,analysis,multimodal,batch]
# @defgroup LLMAnalysis Module group.
# @BRIEF Dashboard analysis paths over the LLM transport: multimodal images and text batch (mixin).
# @LAYER Plugin
import asyncio
import base64
import io
from typing import Any
from PIL import Image
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
from ...core.logger import belief_scope, logger
from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt
from .exceptions import ProviderAuthenticationFailure, ProviderConfigurationFailure
from ._llm_client_core import _should_retry
# #region Plugin.Service.LLMClientAnalysisMixin [C:4] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: image optimization, chunk merging, multimodal and text-batch dashboard analysis.
class LLMClientAnalysisMixin:
# #region LLMClient.analyze_dashboard [C:4] [TYPE Function]
# @BRIEF Sends dashboard data (screenshot + logs) to LLM for health analysis.
# @PRE screenshot_path exists, logs is a list of strings.
# @POST Returns a structured analysis dictionary (status, summary, issues).
# @SIDE_EFFECT Reads screenshot file and calls external LLM API.
# @RATIONALE Delegates to analyze_dashboard_multimodal for single-screenshot
# backward compatibility. Keeps the same contract for v1 consumers.
async def analyze_dashboard(
self,
screenshot_path: str,
logs: list[str],
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
) -> dict[str, Any]:
# Delegate to multimodal variant for backward compat with v1 consumers.
return await self.analyze_dashboard_multimodal(
screenshot_paths=[screenshot_path],
logs=logs,
prompt_template=prompt_template,
)
# #endregion LLMClient.analyze_dashboard
# #region LLMClient._reduce_image_quality [TYPE Function] [C:2]
# @BRIEF Open, resize, and compress a screenshot image for LLM consumption.
# @PRE path points to an existing image file.
# @POST Returns (base64_str, byte_size) tuple.
@staticmethod
def _reduce_image_quality(
path: str,
max_width: int = 1024,
image_quality: int = 60,
) -> tuple[str, int]:
"""
Open, resize, compress, and base64-encode an image.
Returns (base64_str, byte_size).
"""
with Image.open(path) as img:
if img.mode in ("RGBA", "P"):
img = img.convert("RGB")
if img.width > max_width or img.height > 2048:
scale = min(max_width / img.width, 2048 / img.height)
if scale < 1.0:
new_width = int(img.width * scale)
new_height = int(img.height * scale)
img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
buffer = io.BytesIO()
img.save(buffer, format="JPEG", quality=image_quality, optimize=True)
raw = buffer.getvalue()
return base64.b64encode(raw).decode("utf-8"), len(raw)
# #endregion LLMClient._reduce_image_quality
# #region LLMClient._estimate_payload_size [TYPE Function] [C:2]
# @BRIEF Estimate LLM payload size in tokens before sending.
# @POST Returns {estimated_tokens, exceeds_limit, pct_of_limit} dict.
# @RATIONALE FR-056: if >80% of model context window, trigger quality reduction.
@staticmethod
def _estimate_payload_size(
image_paths: list[str],
text_length: int,
model_context: int = 128000,
) -> dict[str, Any]:
"""
Estimate token usage for multimodal payload.
Rough heuristic: 1 image token ~ 258 tokens (GPT-4o), text ~4 chars/token.
Returns {estimated_tokens, exceeds_limit, pct_of_limit}
"""
image_tokens = len(image_paths) * 258 * 5 # rough upper bound for compressed images
text_tokens = text_length // 4
total_tokens = image_tokens + text_tokens
exceeds_limit = total_tokens > (model_context * 0.8)
return {
"estimated_tokens": total_tokens,
"exceeds_limit": exceeds_limit,
"pct_of_limit": round(total_tokens / model_context * 100, 1),
}
# #endregion LLMClient._estimate_payload_size
# #region LLMClient._deduplicate_issues [TYPE Function] [C:2]
# @BRIEF Deduplicate issues by (severity, message, location) while preserving order.
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
seen: set[tuple[str, str, str]] = set()
result: list[dict] = []
for issue in issues:
key = (issue.get("severity", ""), issue.get("message", ""), issue.get("location", "") or "")
if key not in seen:
seen.add(key)
result.append(issue)
return result
# #endregion LLMClient._deduplicate_issues
# #region LLMClient._optimize_images [TYPE Function] [C:2]
# @BRIEF Convert screenshot paths to base64 at given quality, with fallback to raw read.
def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]:
encoded: list[str] = []
for path in paths:
try:
b64, _ = self._reduce_image_quality(path, max_width, quality)
encoded.append(b64)
except Exception as e:
logger.explore("Image optimization failed, falling back to raw read", payload={"path": path}, error=str(e))
with open(path, "rb") as f:
raw = f.read()
b64 = base64.b64encode(raw).decode("utf-8")
encoded.append(b64)
return encoded
# #endregion LLMClient._optimize_images
# #region LLMClient._merge_chunk_results [TYPE Function] [C:2]
# @BRIEF Merge multiple chunk analyses into one. Takes the worst status,
# concatenates summaries, and deduplicates issues.
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
# @POST Returns a single merged dict with chunk_count.
def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]:
STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3}
worst_status = "UNKNOWN"
all_summaries: list[str] = []
all_issues: list[dict] = []
for i, chunk in enumerate(chunks):
s = chunk.get("status", "UNKNOWN")
if STATUS_ORDER.get(s, 3) < STATUS_ORDER.get(worst_status, 3):
worst_status = s
all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}")
all_issues.extend(chunk.get("issues", []))
merged: dict[str, Any] = {
"status": worst_status,
"summary": " | ".join(all_summaries),
"issues": self._deduplicate_issues(all_issues),
"chunk_count": len(chunks),
}
return merged
# #endregion LLMClient._merge_chunk_results
# #region LLMClient._call_llm_for_images [TYPE Function] [C:2]
# @BRIEF Send a single chunk of images to the LLM and return parsed result.
async def _call_llm_for_images(self, encoded_images: list[str], prompt: str) -> dict[str, Any]:
content: list[dict] = [{"type": "text", "text": prompt}]
for b64_img in encoded_images:
content.append(
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
}
)
messages = [{"role": "user", "content": content}]
return await self.get_json_completion(messages)
# #endregion LLMClient._call_llm_for_images
# #region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
# @BRIEF Path A: send screenshots + logs to multimodal LLM, with chunking support.
# @PRE screenshot_paths is a non-empty list of paths.
# tab_labels, if provided, must have the same length as screenshot_paths.
# @POST Returns dict {status, summary, issues} with optional chunk_count.
# @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks).
# @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits.
# Quality reduction is skipped when chunking — each chunk fits the limit by definition.
# Results are merged via _merge_chunk_results.
async def analyze_dashboard_multimodal(
self,
screenshot_paths: list[str],
logs: list[str],
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
max_width: int = 1024,
image_quality: int = 60,
max_images: int | None = None,
tab_labels: list[str] | None = None,
) -> dict[str, Any]:
with belief_scope("analyze_dashboard_multimodal"):
if not screenshot_paths:
raise ValueError("screenshot_paths must be a non-empty list")
# 1. Optimize all images at requested quality
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality)
log_text = "\n".join(logs)
tab_list_text = "\n".join(f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or [])) or "Screenshots are in order."
prompt = render_prompt(
prompt_template,
{
"logs": log_text,
"tab_list": tab_list_text,
"total_chunks": str(len(encoded_images)),
},
)
# 2. Determine chunking
# Default to 8 images per chunk as a safe fallback when max_images is 0 or None
# (0 means probe failed — e.g. Kilo gateway doesn't support OpenAI image format)
DEFAULT_CHUNK_SIZE = 8
effective_max = max_images if (max_images is not None and max_images > 0) else DEFAULT_CHUNK_SIZE
n_total = len(encoded_images)
chunk_size = effective_max if effective_max < n_total else n_total
is_chunking = chunk_size < n_total
if is_chunking:
logger.reason(
"Chunking images for multimodal analysis",
payload={"total_images": n_total, "chunk_count": (n_total + chunk_size - 1) // chunk_size, "chunk_size": chunk_size},
)
# Skip quality reduction: each chunk has ≤ max_images images,
# well within the context window at normal quality.
else:
# Single batch: estimate payload and reduce quality if needed
estimate = self._estimate_payload_size(screenshot_paths, len(prompt) + len(log_text))
if estimate["exceeds_limit"] and image_quality > 30:
logger.reason(
"Reducing image quality to fit context window",
payload={"pct_of_limit": estimate["pct_of_limit"], "new_quality": 30},
)
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30)
# 3. Split into chunks
chunks: list[list[str]] = [encoded_images[i : i + chunk_size] for i in range(0, n_total, chunk_size)]
# 4. Call LLM — parallel for multiple chunks, single for one
try:
if len(chunks) == 1:
result = await self._call_llm_for_images(chunks[0], prompt)
else:
tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks]
chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
valid_results: list[dict] = []
for i, cr in enumerate(chunk_results):
if isinstance(cr, Exception):
# Re-raise permanent provider errors (auth, config) — they apply to all chunks
normalized = self._normalize_provider_error(cr)
if not normalized.retryable:
logger.explore(
"Multimodal analysis chunk failed with permanent provider error, aborting",
payload={"chunk_index": i + 1, "error_type": type(normalized).__name__},
error=str(cr),
)
raise normalized from cr
# Transient chunk failures can be merged as UNKNOWN
logger.explore("Multimodal analysis chunk failed (transient)", payload={"chunk_index": i + 1, "total_chunks": len(chunks)}, error=str(cr))
valid_results.append(
{
"status": "UNKNOWN",
"summary": f"Chunk {i + 1} failed: {cr!s}",
"issues": [],
}
)
else:
valid_results.append(cr)
result = self._merge_chunk_results(valid_results)
except (ProviderAuthenticationFailure, ProviderConfigurationFailure):
# Permanent provider errors propagate to TaskManager — no UNKNOWN swallow
raise
except Exception as e:
# Transient errors and unexpected failures → UNKNOWN (existing behavior)
normalized = self._normalize_provider_error(e)
if not normalized.retryable:
raise normalized from e
logger.explore("Failed to get multimodal analysis from LLM", payload={}, error=str(e))
return {
"status": "UNKNOWN",
"summary": f"Failed to get response from LLM: {e!s}",
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}],
}
return result
# #endregion LLMClient.analyze_dashboard_multimodal
# #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
# @BRIEF Path B batch: multiple dashboards in a single text-only LLM call.
# @PRE payloads is a non-empty list of {dashboard_id, topology, dataset_health, log_text} dicts.
# @POST Returns dict {dashboards: [{dashboard_id, status, summary, issues}]}.
# Missing/parse-error dashboard_id -> marked UNKNOWN individually.
# @RATIONALE Text-only batch avoids image token costs. Uses per-dashboard sections
# with explicit JSON response contract. Fallback ensures partial results survive
# single-dashboard parse failures.
@retry(
stop=stop_after_attempt(5),
wait=wait_exponential(multiplier=2, min=5, max=60),
retry=retry_if_exception(_should_retry),
reraise=True,
)
async def analyze_dashboard_text_batch(
self,
payloads: list[dict],
prompt_template: str,
) -> dict[str, Any]:
"""
Batch analyze multiple dashboards in one LLM call.
payloads: list of dicts with keys:
- dashboard_id (str)
- topology (str) — dashboard structure
- dataset_health (str) — health results
- log_text (str) — execution logs
Returns dict like {dashboards: [{dashboard_id, status, summary, issues}]}
"""
if not payloads:
return {"dashboards": []}
# 1. Build per-dashboard sections
sections = []
for i, p in enumerate(payloads):
did = p.get("dashboard_id", "UNKNOWN")
top = p.get("topology", "")
first_line = top.split("\n")[0] if top else "(no topology)"
section = f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n{top}\n\nDataset health:\n{p.get("dataset_health", "")}\n\nLogs:\n{p.get("log_text", "")}'
sections.append(section)
full_prompt = prompt_template.replace("{total_dashboards}", str(len(payloads)))
full_prompt += '\n\nRespond with a JSON object containing EACH dashboard\'s results:\n{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n'
full_prompt += "\n---\n".join(sections)
messages = [{"role": "user", "content": full_prompt}]
return await self.get_json_completion(messages)
# #endregion LLMClient.analyze_dashboard_text_batch
# #endregion Plugin.Service.LLMClientAnalysisMixin
# #endregion Plugin.Service.LLMClient.AnalysisModule

View File

@@ -0,0 +1,391 @@
# #region Plugin.Service.LLMClient.CoreModule [C:4] [TYPE Module] [SEMANTICS llm,client,provider,openai,retry]
# @defgroup LLMAnalysis Module group.
# @BRIEF Core LLM provider transport: client init, SSL, error normalization, JSON completion (mixin).
# @LAYER Plugin
import asyncio
import json
import os
import ssl
import httpx
from openai import AsyncOpenAI, AuthenticationError as OpenAIAuthenticationError, RateLimitError
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
from ...core.logger import belief_scope, logger
from .exceptions import (
ProviderAuthenticationFailure,
ProviderFailure,
ProviderRateLimitFailure,
ProviderTransportFailure,
)
from .models import LLMProviderType
from typing import Any
from ._constants import LLM_HTTP_TIMEOUT_S
# #region Plugin.Service.ShouldRetry [C:2] [TYPE Function] [SEMANTICS llm,retry,policy]
# @BRIEF Custom retry predicate for Tenacity — excludes non-recoverable errors from LLM retry loops.
# @PRE exception is an Exception raised during LLM API call.
# @POST Returns True if the error is retryable (transport, rate limit), False for permanent errors (auth, config).
# @RELATION CALLED_BY -> [Plugin.Service.LLMClient.get_json_completion]
# @RELATION CALLED_BY -> [Plugin.Service.LLMClient.analyze_dashboard_text_batch]
# @RATIONALE Extracted to module level because it is referenced by @retry decorators in two separate methods
# (get_json_completion and analyze_dashboard_text_batch). A nested function would be fragile —
# moving either method would break the other's retry configuration.
def _should_retry(exception: Exception) -> bool:
"""Custom retry predicate that excludes non-recoverable errors."""
# Typed provider failures use their native retryability
if isinstance(exception, ProviderFailure):
return exception.retryable
# Don't retry on OpenAIAuthenticationError
if isinstance(exception, OpenAIAuthenticationError):
return False
# Don't retry on null content / model errors — retrying won't help
msg = str(exception).lower()
if "null content" in msg or "none" in msg:
return False
# Retry on rate limit errors
if isinstance(exception, RateLimitError):
return True
# For other exceptions, limit retries
return True
# #endregion Plugin.Service.ShouldRetry
# #region Plugin.Service.LLMClientCoreMixin [C:4] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: provider client construction, SSL policy, typed error normalization, JSON-mode completion.
class LLMClientCoreMixin:
# #region LLMClient.__init__ [C:2] [TYPE Function]
# @BRIEF Initializes the LLMClient with provider settings.
# @PRE api_key, base_url, and default_model are non-empty strings.
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
self.provider_type = provider_type
normalized_key = (api_key or "").strip()
if normalized_key.lower().startswith("bearer "):
normalized_key = normalized_key[7:].strip()
self.api_key = normalized_key
self.base_url = base_url
self.default_model = default_model
# DEBUG: Log initialization parameters (without exposing full API key)
logger.reason(
"Initializing LLM client",
payload={
"provider_type": str(provider_type),
"base_url": base_url,
"default_model": default_model,
"api_key_present": bool(self.api_key),
"api_key_length": len(self.api_key) if self.api_key else 0,
},
)
# Some OpenAI-compatible gateways are strict about auth header naming.
default_headers = {"Authorization": f"Bearer {self.api_key}"}
if self.provider_type == LLMProviderType.OPENROUTER:
default_headers["HTTP-Referer"] = os.getenv("OPENROUTER_SITE_URL", "").strip() or os.getenv("APP_BASE_URL", "").strip()
default_headers["X-Title"] = os.getenv("OPENROUTER_APP_NAME", "").strip() or ""
if self.provider_type == LLMProviderType.KILO:
default_headers["Authentication"] = f"Bearer {self.api_key}"
default_headers["X-API-Key"] = self.api_key
# LiteLLM proxy uses standard OpenAI-compatible Bearer auth — no special headers needed.
# It routes to upstream providers transparently, and the default Authorization header
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
ssl_verify = self._get_ssl_verify()
from ...core.ssl import describe_context
ssl_desc = describe_context(ssl_verify)
logger.reason("LLM client SSL verification configured", payload={"ssl_verify": ssl_desc})
http_client = httpx.AsyncClient(
headers=default_headers,
timeout=LLM_HTTP_TIMEOUT_S,
verify=ssl_verify,
)
self.client = AsyncOpenAI(
api_key=self.api_key,
base_url=base_url,
default_headers=default_headers,
http_client=http_client,
)
# #endregion LLMClient.__init__
# #region LLMClient._get_ssl_verify [C:3] [TYPE Function]
# @BRIEF Resolve SSL verification flag from environment.
# @POST Returns SSLContext with system CA dir (never False — centralized SSL).
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
# построения цепочки (verify code 20). capath с хеш-симлинками работает
# корректно (verify code 0). Оба пути — cafile и capath — указывают на
# один и тот же набор сертификатов, но capath правильно обрабатывает
# цепочку Root → Policy → Issuing.
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
# путь в verify=, требует SSLContext.
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
@staticmethod
def _get_ssl_verify() -> ssl.SSLContext | bool:
from ...core.ssl import system_ssl_context
return system_ssl_context()
# #endregion LLMClient._get_ssl_verify
# #region LLMClient._format_connection_error [C:2] [TYPE Function]
# @BRIEF Format exception chain for diagnostics, extracting httpx cause details.
# @POST Returns a human-readable string with the full error chain.
@staticmethod
def _format_connection_error(exc: Exception) -> str:
parts = [f"{type(exc).__name__}: {exc!s}"]
cause = exc.__cause__ or exc.__context__
while cause:
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
cause = cause.__cause__ or cause.__context__
return "\n".join(parts)
# #endregion LLMClient._format_connection_error
# #region LLMClient._supports_json_response_format [C:3] [TYPE Function]
# @BRIEF Detect whether provider/model is likely compatible with response_format=json_object.
# @PRE Client initialized with base_url and default_model.
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
def _supports_json_response_format(self) -> bool:
model = (self.default_model or "").lower()
# Free-tier models from ANY gateway often reject json_object mode
# (Nvidia NeMo free via OpenRouter or Kilo, stepfun free, etc.)
if ":free" in model:
return False
# stepfun models (even non-free) don't support json_object mode
if "stepfun/" in model or model.startswith("step-"):
return False
return True
# #endregion LLMClient._supports_json_response_format
# #region LLMClient._normalize_provider_error [C:3] [TYPE Function]
# @BRIEF Normalize OpenAI/HTTP exceptions into typed ProviderFailure hierarchy.
# @PRE exc is a raw exception from the OpenAI SDK or httpx client.
# @POST Returns a ProviderFailure subclass that expresses retryability.
# @SIDE_EFFECT Logs the normalization decision.
# @RATIONALE OpenAI SDK exceptions (OpenAIAuthenticationError, etc.) carry status codes and
# messages that map to typed provider errors. httpx transport errors map to
# ProviderTransportFailure. Generic exceptions with "401" patterns are caught
# via string fallback for non-OpenAI gateways.
@staticmethod
def _normalize_provider_error(exc: Exception, provider_id: str | None = None) -> ProviderFailure:
# Already a typed failure — pass through
if isinstance(exc, ProviderFailure):
return exc
# OpenAI SDK auth error
if isinstance(exc, OpenAIAuthenticationError):
return ProviderAuthenticationFailure(
str(exc), provider_id=provider_id, status_code=401, original=exc,
)
# OpenAI rate limit
if isinstance(exc, RateLimitError):
return ProviderRateLimitFailure(
str(exc), provider_id=provider_id, status_code=429, original=exc,
)
# httpx transport errors (connection, timeout, DNS)
if isinstance(exc, httpx.TimeoutException):
return ProviderTransportFailure(
str(exc), provider_id=provider_id, original=exc,
)
if isinstance(exc, httpx.ConnectError):
return ProviderTransportFailure(
str(exc), provider_id=provider_id, original=exc,
)
# HTTP status code via status_code attribute (OpenAI-like SDKs)
status_code = getattr(exc, "status_code", None) or getattr(exc, "status", None)
if status_code is not None:
status_code = int(status_code) # type: ignore[arg-type]
if status_code in (401, 403):
return ProviderAuthenticationFailure(
str(exc), provider_id=provider_id, status_code=status_code, original=exc,
)
if status_code == 429:
return ProviderRateLimitFailure(
str(exc), provider_id=provider_id, status_code=429, original=exc,
)
if status_code >= 500:
return ProviderTransportFailure(
str(exc), provider_id=provider_id, status_code=status_code, original=exc,
)
# String fallback for non-OpenAI gateways that embed an auth status in generic errors.
msg = str(exc).lower()
if "401" in msg or "403" in msg or "authentication" in msg or "unauthorized" in msg:
return ProviderAuthenticationFailure(
str(exc),
provider_id=provider_id,
status_code=401 if "401" in msg else 403 if "403" in msg else None,
original=exc,
)
if "429" in msg or "rate limit" in msg:
return ProviderRateLimitFailure(
str(exc), provider_id=provider_id, original=exc,
)
# Default: assume transport-level failure (retryable)
return ProviderTransportFailure(
str(exc), provider_id=provider_id, original=exc,
)
# #endregion LLMClient._normalize_provider_error
# #region LLMClient.get_json_completion [C:4] [TYPE Function]
# @BRIEF Helper to handle LLM calls with JSON mode and fallback parsing.
# @PRE messages is a list of valid message dictionaries.
# @POST Returns a parsed JSON dictionary.
# @SIDE_EFFECT Calls external LLM API.
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), retry=retry_if_exception(_should_retry), reraise=True)
async def get_json_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
with belief_scope("get_json_completion"):
response = None
try:
use_json_mode = self._supports_json_response_format()
try:
logger.reason(
"Attempting LLM call",
payload={
"model": self.default_model,
"json_mode": "on" if use_json_mode else "off",
"base_url": self.base_url,
"message_count": len(messages),
"api_key_present": bool(self.api_key and len(self.api_key) > 0),
},
)
if use_json_mode:
response = await self.client.chat.completions.create(model=self.default_model, messages=messages, response_format={"type": "json_object"})
else:
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
except Exception as e:
if use_json_mode and ("JSON mode is not enabled" in str(e) or "json_object is not supported" in str(e).lower() or "response_format" in str(e).lower() or "400" in str(e)):
logger.explore("JSON mode failed or not supported, falling back to plain text", payload={"model": self.default_model}, error=str(e))
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
else:
raise e
logger.reflect("LLM API response received", payload={"response_summary": str(response)[:200]})
except RateLimitError as e:
logger.explore("Rate limit hit on LLM call, retrying with backoff", payload={}, error=str(e))
# Extract retry_delay from error metadata if available
retry_delay = 5.0 # Default fallback
try:
# Based on logs, the raw response is in e.body or e.response.json()
# The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper
# Let's try to find the 'retryDelay' in the error message or response
import re
# Try to find "retryDelay": "XXs" in the string representation of the error
error_str = str(e)
match = re.search(r'"retryDelay":\s*"(\d+)s"', error_str)
if match:
retry_delay = float(match.group(1))
else:
# Try to parse from response if it's a standard OpenAI-like error with body
if hasattr(e, "body") and isinstance(e.body, dict):
# Some providers put it in details
details = e.body.get("error", {}).get("details", [])
for detail in details:
if detail.get("@type") == "type.googleapis.com/google.rpc.RetryInfo":
delay_str = detail.get("retryDelay", "5s")
retry_delay = float(delay_str.rstrip("s"))
break
except Exception as parse_e:
logger.explore("Failed to parse retry delay from error response", payload={}, error=str(parse_e))
# Add a small safety margin (0.5s) as requested
wait_time = retry_delay + 0.5
logger.reason("Waiting before LLM retry", payload={"wait_time_seconds": wait_time})
await asyncio.sleep(wait_time)
raise
except Exception as e:
# Normalize into typed provider exception chain
provider_exc = self._normalize_provider_error(e, provider_id=self.default_model)
logger.explore(
"LLM call failed, normalized as",
payload={"type": type(provider_exc).__name__, "status_code": provider_exc.status_code},
error=str(provider_exc),
)
raise provider_exc from e
if not response or not hasattr(response, "choices") or not response.choices:
raise RuntimeError(f"Invalid LLM response: {response}")
content = response.choices[0].message.content
logger.reflect("Raw LLM response content received for parsing", payload={"content_length": len(content) if content else 0})
# LLM returned null content — likely content filter or rate limit
if content is None:
raise RuntimeError("LLM returned null content (content filter or rate limit)")
try:
return json.loads(content)
except json.JSONDecodeError:
logger.explore("Failed to parse JSON directly, attempting to extract from code blocks", payload={}, error="JSONDecodeError on first parse attempt")
if "```json" in content:
json_str = content.split("```json")[1].split("```")[0].strip()
return json.loads(json_str)
elif "```" in content:
json_str = content.split("```")[1].split("```")[0].strip()
return json.loads(json_str)
else:
raise
# #endregion LLMClient.get_json_completion
# #region LLMClient.test_runtime_connection [C:3] [TYPE Function]
# @BRIEF Validate provider credentials using the same chat completions transport as runtime analysis.
# @PRE Client is initialized with provider credentials and default_model.
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
# @SIDE_EFFECT Calls external LLM API.
async def test_runtime_connection(self) -> dict[str, Any]:
with belief_scope("test_runtime_connection"):
messages = [
{
"role": "user",
"content": 'Return exactly this JSON object and nothing else: {"ok": true}',
}
]
return await self.get_json_completion(messages)
# #endregion LLMClient.test_runtime_connection
# #region LLMClient.fetch_models [C:3] [TYPE Function]
# @BRIEF Fetch available models from the provider's API.
# @PRE Client is initialized with provider credentials.
# @POST Returns a list of model ID strings.
# @SIDE_EFFECT Calls external LLM API /v1/models endpoint.
async def fetch_models(self) -> list[str]:
with belief_scope("LLMClient.fetch_models"):
try:
response = await self.client.models.list()
model_ids = [m.id for m in response.data]
model_ids.sort()
logger.reason(
"Fetched available models from provider",
payload={"model_count": len(model_ids), "base_url": self.base_url},
)
return model_ids
except Exception as e:
logger.explore(
"Failed to fetch models from provider",
payload={"base_url": self.base_url, "formatted_error": self._format_connection_error(e)},
error=str(e),
)
raise
# #endregion LLMClient.fetch_models
# #endregion Plugin.Service.LLMClientCoreMixin
# #endregion Plugin.Service.LLMClient.CoreModule

View File

@@ -0,0 +1,60 @@
# #region Plugin.Service.RedactionModule [C:2] [TYPE Module] [SEMANTICS llm,redaction,pii,security]
# @defgroup LLMAnalysis Module group.
# @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses.
# @LAYER Plugin
import re
# #region Plugin.Service.RedactionService [C:2] [TYPE Module]
# @defgroup LLMAnalysis Module group.
# @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses.
# @LAYER Service
# @RATIONALE FR-029: sensitive data must be filtered BEFORE external LLM send and BEFORE persistence.
class RedactionService:
"""Redacts PII, credentials, and sensitive data."""
# Common patterns to redact
PATTERNS = [
(r"password[=:]\s*\S+", "password=***"),
(r"secret[=:]\s*\S+", "secret=***"),
(r"token[=:]\s*\S+", "token=***"),
(r"api_key[=:]\s*\S+", "api_key=***"),
(r"apikey[=:]\s*\S+", "apikey=***"),
(r"Authorization:\s*\S+", "Authorization: ***"),
(r"Bearer\s+\S+\.\S+\.\S+", "Bearer ***"),
(r"[A-Za-z0-9+/=]{40,}", "***"), # base64 or long tokens
(r"[\w.+-]+@[\w-]+\.[\w.-]+", "***@***"), # emails
]
# #region RedactionService.redact_logs [TYPE Function] [C:2]
# @BRIEF Redact PII/credentials from log lines.
# @PRE logs is a list of strings.
# @POST Returns redacted list preserving structure.
@staticmethod
def redact_logs(logs: list[str]) -> list[str]:
"""Redact PII/credentials from log lines."""
redacted = []
for line in logs:
for pattern, replacement in RedactionService.PATTERNS:
line = re.sub(pattern, replacement, line, flags=re.IGNORECASE)
redacted.append(line)
return redacted
# #endregion RedactionService.redact_logs
# #region RedactionService.redact_raw_response [TYPE Function] [C:2]
# @BRIEF Redact sensitive data from LLM raw response.
# @PRE raw is a string.
# @POST Returns redacted string.
@staticmethod
def redact_raw_response(raw: str) -> str:
"""Redact sensitive data from LLM raw response."""
for pattern, replacement in RedactionService.PATTERNS:
raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE)
return raw
# #endregion RedactionService.redact_raw_response
# #endregion Plugin.Service.RedactionService
# #endregion Plugin.Service.RedactionModule

View File

@@ -0,0 +1,39 @@
# #region Plugin.Service.Screenshot [C:4] [TYPE Module] [SEMANTICS llm,screenshot,facade]
# @defgroup LLMAnalysis Module group.
# @BRIEF Facade assembling ScreenshotService from login/wait/session/capture/media mixins.
# @LAYER Plugin
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.LoginModule]
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.WaitModule]
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.SessionModule]
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.CaptureModule]
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.MediaModule]
from ...core.config_models import Environment
from ._screenshot_capture import ScreenshotCaptureMixin
from ._screenshot_login import ScreenshotLoginMixin
from ._screenshot_media import ScreenshotMediaMixin
from ._screenshot_session import ScreenshotSessionMixin
from ._screenshot_wait import ScreenshotWaitMixin
__all__ = ["ScreenshotService"]
# #region Plugin.Service.ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright]
# @defgroup LLMAnalysis Module group.
# @BRIEF Handles capturing screenshots of Superset dashboards.
# @SIDE_EFFECT Launches Playwright browser; captures screenshots to disk.
class ScreenshotService(
ScreenshotLoginMixin,
ScreenshotWaitMixin,
ScreenshotSessionMixin,
ScreenshotCaptureMixin,
ScreenshotMediaMixin,
):
# #region ScreenshotService.__init__ [C:1] [TYPE Function] [SEMANTICS init]
# @BRIEF Initializes the ScreenshotService with environment configuration.
# @PRE env is a valid Environment object.
def __init__(self, env: Environment):
self.env = env
# #endregion ScreenshotService.__init__
# #endregion Plugin.Service.ScreenshotService
# #endregion Plugin.Service.Screenshot

View File

@@ -0,0 +1,256 @@
# #region Plugin.Service.Screenshot.CaptureModule [C:4] [TYPE Module] [SEMANTICS llm,screenshot,capture,tabs]
# @defgroup LLMAnalysis Module group.
# @BRIEF Per-tab CDP screenshot capture and orchestration for ScreenshotService (mixin).
# @LAYER Plugin
import base64
import os
import re
from playwright.async_api import async_playwright
from ...core.logger import belief_scope, logger
from ._constants import PLAYWRIGHT_WAIT_TIMEOUT_MS
# #region Plugin.Service.ScreenshotCaptureMixin [C:4] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: browser session launch delegation, per-tab chunked capture, PNG->JPEG/WebP pipeline.
class ScreenshotCaptureMixin:
# #region ScreenshotService.capture_dashboard_chunks [C:4] [TYPE Function]
# @BRIEF Capture per-tab screenshots: login → navigate → switch tabs → per-tab CDP screenshots.
# @PRE dashboard_id is valid, browser available.
# @POST Returns list of {tab_name, path} dicts — one per tab.
# @SIDE_EFFECT Launches browser, logs in, switches tabs, captures screenshots.
# @RATIONALE Multi-chunk: one screenshot per tab instead of one full-page.
# All screenshots written to output_dir; CDP fallback to Playwright full_page.
# @REJECTED Single full-page screenshot rejected for v2 — per-tab captures give LLM
# better visibility into individual tab content, especially for dashboards with
# many tabs where the full-page capture may miss lazy-loaded tab content.
async def capture_dashboard_chunks(
self,
dashboard_id: str,
output_dir: str,
parsed_context: dict | None = None,
) -> list[dict]:
"""Capture per-tab screenshots instead of one full-page.
Args:
dashboard_id: Superset dashboard ID
output_dir: Directory to save screenshots
parsed_context: Optional parsed context with activeTabs, native_filters from URL parse
Returns:
list of {tab_name, path} — one per tab
"""
import time as _time
timestamp = int(_time.time())
os.makedirs(output_dir, exist_ok=True)
async with async_playwright() as p:
browser, context, page = await self._launch_and_login(p, dashboard_id, parsed_context)
try:
results: list[dict] = []
processed_tabs: set[str] = set()
async def _capture_tabs(depth: int = 0) -> None:
if depth > 3:
return
tab_selectors = [
".ant-tabs-nav-list .ant-tabs-tab",
".dashboard-component-tabs .ant-tabs-tab",
'[data-test="dashboard-component-tabs"] .ant-tabs-tab',
]
found_tabs = []
for selector in tab_selectors:
found_tabs = await page.locator(selector).all()
if found_tabs:
break
if not found_tabs:
return
logger.reason("Found tabs at current depth", payload={"tab_count": len(found_tabs), "depth": depth})
for i, tab in enumerate(found_tabs):
try:
tab_text = (await tab.inner_text()).strip()
tab_id = f"{depth}_{i}_{tab_text}"
if tab_id in processed_tabs:
continue
if not await tab.is_visible():
continue
processed_tabs.add(tab_id)
logger.reason("Switching to tab", payload={"tab_text": tab_text, "depth": depth, "index": i})
is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "")
if not is_active:
await tab.click()
try:
await page.wait_for_function(
"""() => {
const activeTab = document.querySelector('.ant-tabs-tab-active');
if (!activeTab) return true;
const tabPane = activeTab.closest('.ant-tabs')?.querySelector('.ant-tabs-content-holder');
if (!tabPane) return true;
const charts = tabPane.querySelectorAll('canvas, svg');
return charts.length > 0;
}""",
timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS,
)
except Exception:
logger.explore(
"Content verification timed out after tab switch",
payload={"tab_text": tab_text},
error="Verification wait timed out",
)
# Wait for charts to stabilize
await self._wait_for_charts_stabilized(page)
# Resize viewport to 1920x1200 for consistent screenshots
await page.set_viewport_size({"width": 1920, "height": 1200})
await self._wait_for_resize_rendered(page, {})
# CDP screenshot with fallback
safe_tab = re.sub(r"[^\w\-_ ]", "", tab_text).strip().replace(" ", "_")[:40]
if not safe_tab:
safe_tab = f"tab_{depth}_{i}"
tab_filename = f"{dashboard_id}_{safe_tab}_{timestamp}_d{depth}.png"
tab_path = os.path.join(output_dir, tab_filename)
try:
cdp = await page.context.new_cdp_session(page)
screenshot_data = await cdp.send(
"Page.captureScreenshot",
{
"format": "png",
"fromSurface": True,
"captureBeyondViewport": True,
},
)
image_data = base64.b64decode(screenshot_data["data"])
with open(tab_path, "wb") as f:
f.write(image_data)
except Exception as cdp_err:
logger.explore(
"CDP screenshot failed, falling back to Playwright full_page",
payload={"tab_text": tab_text},
error=str(cdp_err),
)
await page.screenshot(path=tab_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
logger.reason("Saved screenshot for tab", payload={"tab_path": tab_path, "tab_text": tab_text})
results.append({"tab_name": tab_text, "path": tab_path})
# Recurse into nested tabs
await _capture_tabs(depth + 1)
except Exception as tab_e:
logger.explore(
"Failed to process tab",
payload={"tab_index": i, "depth": depth},
error=str(tab_e),
)
# Return to first tab
try:
first_tab = found_tabs[0]
if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""):
await first_tab.click()
except Exception:
pass
await _capture_tabs()
# If no tabs found, capture the whole page as a single chunk
if not results:
logger.reason("No tabs found, capturing full-page as single chunk")
await self._wait_for_charts_stabilized(page)
await page.set_viewport_size({"width": 1920, "height": 1200})
tab_path = os.path.join(output_dir, f"{dashboard_id}_full_{timestamp}.png")
try:
cdp = await page.context.new_cdp_session(page)
screenshot_data = await cdp.send(
"Page.captureScreenshot",
{
"format": "png",
"fromSurface": True,
"captureBeyondViewport": True,
},
)
image_data = base64.b64decode(screenshot_data["data"])
with open(tab_path, "wb") as f:
f.write(image_data)
except Exception as cdp_err:
logger.explore("CDP full-page fallback failed, using Playwright full_page", payload={}, error=str(cdp_err))
await page.screenshot(path=tab_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
results.append({"tab_name": "full", "path": tab_path})
return results
finally:
await browser.close()
# #endregion ScreenshotService.capture_dashboard_chunks
# #region ScreenshotService.capture_dashboard [C:4] [TYPE Function]
# @BRIEF Captures multi-chunk screenshots, converts for LLM, archives to WebP.
# @PRE dashboard_id is a valid string, output_path is a writable path.
# @POST Returns list of {original, webp_path} dicts from WebP archive.
# Empty list on complete failure.
# @SIDE_EFFECT Launches browser, performs UI login, writes PNG/JPEG/WebP files;
# deletes intermediate PNG and JPEG files after conversion.
# @RATIONALE Refactored v2: delegates to capture_dashboard_chunks for per-tab PNGs,
# then converts for LLM (JPEG) and archive (WebP). Temp intermediates are cleaned up.
# @REJECTED Returning bool rejected for v2 — callers need access to archived WebP paths
# for persistence and LLM pipeline.
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> tuple[list[str], list[dict]]:
"""Capture dashboard screenshots (multi-chunk), convert for LLM, archive to WebP.
Returns (jpeg_paths, archive_results) tuple.
jpeg_paths — list of JPEG paths ready for LLM analysis (caller must clean up).
archive_results — list of {original, webp_path} dicts from WebP archive.
"""
output_dir = os.path.dirname(output_path) or "."
os.makedirs(output_dir, exist_ok=True)
with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"):
logger.reason("Capturing dashboard screenshots", payload={"dashboard_id": dashboard_id})
# 1. Capture per-tab screenshots
chunks = await self.capture_dashboard_chunks(dashboard_id, output_dir)
png_paths = [c["path"] for c in chunks if c.get("path")]
if not png_paths:
logger.explore("No screenshots captured for dashboard", payload={"dashboard_id": dashboard_id}, error="All capture attempts returned empty")
return [], []
# 2. Convert PNGs to JPEGs for LLM
jpeg_paths = self._convert_screenshots_for_llm(png_paths, output_dir)
logger.reason(
"Converted PNGs to JPEGs for LLM analysis",
payload={"converted": len(jpeg_paths), "total": len(png_paths)},
)
# 3. Archive to WebP (deletes PNGs on success)
archive_results = self._archive_screenshots_as_webp(png_paths, output_dir)
archived_count = sum(1 for r in archive_results if r.get("webp_path"))
logger.reason(
"Archived screenshots to WebP",
payload={"archived": archived_count, "total": len(archive_results)},
)
# 4. Return JPEGs for LLM — caller cleans up after analysis
return jpeg_paths, archive_results
# #endregion ScreenshotService.capture_dashboard
# #endregion Plugin.Service.ScreenshotCaptureMixin
# #endregion Plugin.Service.Screenshot.CaptureModule

View File

@@ -0,0 +1,264 @@
# #region Plugin.Service.Screenshot.LoginModule [C:3] [TYPE Module] [SEMANTICS llm,screenshot,login,playwright]
# @defgroup LLMAnalysis Module group.
# @BRIEF Login and navigation locator helpers for ScreenshotService (mixin).
# @LAYER Plugin
from typing import Any
from urllib.parse import urlsplit
from ...core.logger import logger
from ._constants import HTTP_REQUEST_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS
# #region Plugin.Service.ScreenshotLoginMixin [C:3] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: login-form discovery, fallback form POST, resilient navigation.
class ScreenshotLoginMixin:
# #region ScreenshotService._find_first_visible_locator [C:3] [TYPE Function]
# @BRIEF Resolve the first visible locator from multiple Playwright locator strategies.
# @PRE candidates is a non-empty list of locator-like objects.
# @POST Returns a locator ready for interaction or None when nothing matches.
async def _find_first_visible_locator(self, candidates) -> Any:
for locator in candidates:
try:
match_count = await locator.count()
for index in range(match_count):
candidate = locator.nth(index)
if await candidate.is_visible():
return candidate
except Exception:
continue
return None
# #endregion ScreenshotService._find_first_visible_locator
# #region ScreenshotService._iter_login_roots [C:2] [TYPE Function]
# @BRIEF Enumerate page and child frames where login controls may be rendered.
# @PRE page is a Playwright page-like object.
# @POST Returns ordered roots starting with main page followed by frames.
def _iter_login_roots(self, page) -> list[Any]:
roots = [page]
page_frames = getattr(page, "frames", [])
try:
for frame in page_frames:
if frame not in roots:
roots.append(frame)
except Exception:
pass
return roots
# #endregion ScreenshotService._iter_login_roots
# #region ScreenshotService._extract_hidden_login_fields [C:2] [TYPE Function]
# @BRIEF Collect hidden form fields required for direct login POST fallback.
# @PRE Login page is loaded.
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
async def _extract_hidden_login_fields(self, page) -> dict[str, str]:
hidden_fields: dict[str, str] = {}
for root in self._iter_login_roots(page):
try:
locator = root.locator("input[type='hidden'][name]")
count = await locator.count()
for index in range(count):
candidate = locator.nth(index)
field_name = str(await candidate.get_attribute("name") or "").strip()
if not field_name or field_name in hidden_fields:
continue
hidden_fields[field_name] = str(await candidate.input_value()).strip()
except Exception:
continue
return hidden_fields
# #endregion ScreenshotService._extract_hidden_login_fields
# #region ScreenshotService._extract_csrf_token [C:2] [TYPE Function]
# @BRIEF Resolve CSRF token value from main page or embedded login frame.
# @PRE Login page is loaded.
# @POST Returns first non-empty csrf token or empty string.
async def _extract_csrf_token(self, page) -> str:
hidden_fields = await self._extract_hidden_login_fields(page)
return str(hidden_fields.get("csrf_token") or "").strip()
# #endregion ScreenshotService._extract_csrf_token
# #region ScreenshotService._response_looks_like_login_page [C:2] [TYPE Function]
# @BRIEF Detect when fallback login POST returned the login form again instead of an authenticated page.
# @PRE response_text is normalized HTML or text from login POST response.
# @POST Returns True when login-page markers dominate the response body.
def _response_looks_like_login_page(self, response_text: str) -> bool:
normalized = str(response_text or "").strip().lower()
if not normalized:
return False
markers = [
"enter your login and password below",
"username:",
"password:",
"sign in",
'name="csrf_token"',
]
return sum(marker in normalized for marker in markers) >= 3
# #endregion ScreenshotService._response_looks_like_login_page
# #region ScreenshotService._redirect_looks_authenticated [C:2] [TYPE Function]
# @BRIEF Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
# @PRE redirect_location may be empty or relative.
# @POST Returns True when redirect target does not point back to login flow.
def _redirect_looks_authenticated(self, redirect_location: str) -> bool:
normalized = str(redirect_location or "").strip().lower()
if not normalized:
return True
return "/login" not in normalized
# #endregion ScreenshotService._redirect_looks_authenticated
# #region ScreenshotService._submit_login_via_form_post [C:3] [TYPE Function]
# @BRIEF Fallback login path that submits credentials directly with csrf token.
# @PRE login_url is same-origin and csrf token can be read from DOM.
# @POST Browser context receives authenticated cookies when login succeeds.
async def _submit_login_via_form_post(self, page, login_url: str) -> bool:
hidden_fields = await self._extract_hidden_login_fields(page)
csrf_token = str(hidden_fields.get("csrf_token") or "").strip()
if not csrf_token:
logger.explore("Direct form login fallback skipped: csrf_token not found", error="Missing csrf_token in login form")
return False
try:
request_context = page.context.request
except Exception as context_error:
logger.explore("Direct form login fallback skipped: request context unavailable", payload={"error": str(context_error)}, error="Browser request context unavailable")
return False
parsed_url = urlsplit(login_url)
origin = f"{parsed_url.scheme}://{parsed_url.netloc}" if parsed_url.scheme and parsed_url.netloc else login_url
payload = dict(hidden_fields)
payload["username"] = self.env.username
payload["password"] = self.env.password
logger.reason(
"Attempting direct form login fallback via browser context request",
payload={"hidden_fields": sorted(hidden_fields.keys())},
)
response = await request_context.post(
login_url,
form=payload,
headers={
"Origin": origin,
"Referer": login_url,
},
timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS,
fail_on_status_code=False,
max_redirects=0,
)
response_url = str(getattr(response, "url", "") or "")
response_status = int(getattr(response, "status", 0) or 0)
response_headers = dict(getattr(response, "headers", {}) or {})
redirect_location = str(response_headers.get("location") or response_headers.get("Location") or "").strip()
redirect_statuses = {301, 302, 303, 307, 308}
if response_status in redirect_statuses:
redirect_authenticated = self._redirect_looks_authenticated(redirect_location)
logger.reason(
"Direct form login fallback redirect response",
payload={"status": response_status, "url": response_url, "location": redirect_location, "authenticated": redirect_authenticated},
)
return redirect_authenticated
response_text = await response.text()
text_snippet = " ".join(response_text.split())[:200]
looks_like_login_page = self._response_looks_like_login_page(response_text)
logger.reason(
"Direct form login fallback response",
payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet},
)
return not looks_like_login_page
# #endregion ScreenshotService._submit_login_via_form_post
# #region ScreenshotService._find_login_field_locator [C:3] [TYPE Function]
# @BRIEF Resolve login form input using semantic label text plus generic visible-input fallbacks.
# @PRE field_name is `username` or `password`.
# @POST Returns a locator for the corresponding input or None.
async def _find_login_field_locator(self, page, field_name: str) -> Any:
normalized = str(field_name or "").strip().lower()
for root in self._iter_login_roots(page):
if normalized == "username":
input_candidates = [
root.get_by_label("Username", exact=False),
root.get_by_label("Login", exact=False),
root.locator("label:text-matches('Username|Login', 'i')").locator("xpath=following::input[1]"),
root.locator("text=/Username|Login/i").locator("xpath=following::input[1]"),
root.locator("input[name='username']"),
root.locator("input#username"),
root.locator("input[placeholder*='Username']"),
root.locator("input[type='text']"),
root.locator("input:not([type='password'])"),
]
locator = await self._find_first_visible_locator(input_candidates)
if locator:
return locator
if normalized == "password":
input_candidates = [
root.get_by_label("Password", exact=False),
root.locator("label:text-matches('Password', 'i')").locator("xpath=following::input[1]"),
root.locator("text=/Password/i").locator("xpath=following::input[1]"),
root.locator("input[name='password']"),
root.locator("input#password"),
root.locator("input[placeholder*='Password']"),
root.locator("input[type='password']"),
]
locator = await self._find_first_visible_locator(input_candidates)
if locator:
return locator
return None
# #endregion ScreenshotService._find_login_field_locator
# #region ScreenshotService._find_submit_locator [C:3] [TYPE Function]
# @BRIEF Resolve login submit button from main page or embedded auth frame.
# @PRE page is ready for login interaction.
# @POST Returns visible submit locator or None.
async def _find_submit_locator(self, page) -> Any:
selectors = [
lambda root: root.get_by_role("button", name="Sign in", exact=False),
lambda root: root.get_by_role("button", name="Login", exact=False),
lambda root: root.locator("button[type='submit']"),
lambda root: root.locator("button#submit"),
lambda root: root.locator(".btn-primary"),
lambda root: root.locator("input[type='submit']"),
]
for root in self._iter_login_roots(page):
locator = await self._find_first_visible_locator([factory(root) for factory in selectors])
if locator:
return locator
return None
# #endregion ScreenshotService._find_submit_locator
# #region ScreenshotService._goto_resilient [C:3] [TYPE Function]
# @BRIEF Navigate without relying on networkidle for pages with long-polling or persistent requests.
# @PRE page is a valid Playwright page and url is non-empty.
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
async def _goto_resilient(
self,
page,
url: str,
primary_wait_until: str = "domcontentloaded",
fallback_wait_until: str = "load",
timeout: int = HTTP_REQUEST_TIMEOUT_MS,
):
try:
return await page.goto(url, wait_until=primary_wait_until, timeout=timeout)
except Exception as primary_error:
logger.explore(
"Primary navigation wait failed, falling back to fallback wait",
payload={"primary_wait_until": primary_wait_until, "url": url},
error=str(primary_error),
)
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
# #endregion ScreenshotService._goto_resilient
# #endregion Plugin.Service.ScreenshotLoginMixin
# #endregion Plugin.Service.Screenshot.LoginModule

View File

@@ -0,0 +1,102 @@
# #region Plugin.Service.Screenshot.MediaModule [C:2] [TYPE Module] [SEMANTICS llm,screenshot,jpeg,webp,archive]
# @defgroup LLMAnalysis Module group.
# @BRIEF Screenshot format conversion and cleanup helpers for ScreenshotService (mixin).
# @LAYER Plugin
import os
from PIL import Image
from ...core.logger import logger
# #region Plugin.Service.ScreenshotMediaMixin [C:2] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: PNG to JPEG conversion for LLM, WebP archiving, temp-file cleanup.
class ScreenshotMediaMixin:
# #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
# @BRIEF Convert PNG screenshots to JPEG for LLM transmission.
# @PRE png_paths is a list of existing PNG file paths.
# @POST Returns list of JPEG paths. JPEG files should be deleted after LLM call.
@staticmethod
def _convert_screenshots_for_llm(png_paths: list[str], output_dir: str) -> list[str]:
"""Convert PNG screenshots to JPEG for LLM transmission.
PNG → Pillow → JPEG quality=60, max 1024px width.
Returns list of JPEG paths.
"""
jpeg_paths: list[str] = []
for png_path in png_paths:
try:
img = Image.open(png_path)
if img.mode in ("RGBA", "P"):
img = img.convert("RGB")
max_width = 1024
if img.width > max_width:
scale = max_width / img.width
new_w = int(img.width * scale)
new_h = int(img.height * scale)
img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
base = os.path.splitext(os.path.basename(png_path))[0]
jpeg_path = os.path.join(output_dir, f"{base}_llm.jpg")
img.save(jpeg_path, format="JPEG", quality=60, optimize=True)
jpeg_paths.append(jpeg_path)
except Exception as e:
logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e))
return jpeg_paths
# #endregion ScreenshotService._convert_screenshots_for_llm
# #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
# @BRIEF Convert PNG screenshots to WebP for archive.
# @PRE png_paths is a list of existing PNG file paths.
# @POST Returns list of {original, webp_path} dicts. PNG deleted after successful WebP save.
@staticmethod
def _archive_screenshots_as_webp(png_paths: list[str], archive_dir: str) -> list[dict]:
"""Convert PNG screenshots to WebP for archive.
PNG → Pillow → WebP lossy quality=80.
Deletes PNG after WebP saved.
Returns list of {original, webp_path} dicts.
"""
results: list[dict] = []
for png_path in png_paths:
try:
img = Image.open(png_path)
if img.mode in ("RGBA", "P"):
img = img.convert("RGB")
base = os.path.splitext(os.path.basename(png_path))[0]
webp_path = os.path.join(archive_dir, f"{base}.webp")
img.save(webp_path, format="WEBP", quality=80, lossless=False)
# Delete PNG after successful WebP save
os.remove(png_path)
results.append({"original": png_path, "webp_path": webp_path})
except Exception as e:
logger.explore("Failed to archive screenshot to WebP, keeping PNG", payload={"png_path": png_path}, error=str(e))
results.append({"original": png_path, "webp_path": None})
return results
# #endregion ScreenshotService._archive_screenshots_as_webp
# #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
# @BRIEF Delete temporary files (PNG, JPEG intermediates).
@staticmethod
def _cleanup_temp_files(paths: list[str]) -> None:
"""Delete temporary files (PNG, JPEG intermediates)."""
for path in paths:
try:
if os.path.exists(path):
os.remove(path)
except Exception as e:
logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e))
# #endregion ScreenshotService._cleanup_temp_files
# #endregion Plugin.Service.ScreenshotMediaMixin
# #endregion Plugin.Service.Screenshot.MediaModule

View File

@@ -0,0 +1,187 @@
# #region Plugin.Service.Screenshot.SessionModule [C:4] [TYPE Module] [SEMANTICS llm,screenshot,session,login,navigation]
# @defgroup LLMAnalysis Module group.
# @BRIEF Browser session launch, UI login and dashboard navigation for ScreenshotService (mixin).
# @LAYER Plugin
from typing import Any
from ...core.logger import logger
from ._constants import HTTP_REQUEST_TIMEOUT_MS, PLAYWRIGHT_NAVIGATION_TIMEOUT_MS
# #region Plugin.Service.ScreenshotSessionMixin [C:4] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: headless Chromium launch with anti-automation flags, login flow, dashboard navigation.
class ScreenshotSessionMixin:
# #region ScreenshotService._launch_and_login [C:4] [TYPE Function]
# @BRIEF Launch browser, log in to Superset, navigate to dashboard URL.
# @PRE dashboard_id is valid, playwright instance is provided.
# @POST Returns (browser, context, page) tuple with active session.
# @SIDE_EFFECT Launches headless Chromium, performs UI login, navigates to dashboard.
# @RATIONALE Extracted from capture_dashboard to share login/navigation logic
# between capture_dashboard (backward compat) and capture_dashboard_chunks (multi-tab).
# @REJECTED Duplicating login logic rejected — any change to auth flow would require
# updating two code paths, leading to accidental drift.
async def _launch_and_login(
self,
playwright,
dashboard_id: str,
parsed_context: dict | None = None,
) -> tuple[Any, Any, Any]:
"""Launch browser, log in to Superset, and navigate to the dashboard.
Returns (browser, context, page).
"""
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
base_ui_url = self.env.url.rstrip("/")
if base_ui_url.endswith("/api/v1"):
base_ui_url = base_ui_url[: -len("/api/v1")]
browser = await playwright.chromium.launch(
headless=True,
args=[
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--no-sandbox",
],
)
context = await browser.new_context(
viewport={"width": 1280, "height": 720},
user_agent=user_agent,
extra_http_headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
},
)
page = await context.new_page()
await page.add_init_script("delete Object.getPrototypeOf(navigator).webdriver")
# 1. Navigate to login page and authenticate
login_url = f"{base_ui_url.rstrip('/')}/login/"
await self._goto_resilient(
page,
login_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
await page.wait_for_load_state("domcontentloaded")
try:
used_direct_form_login = False
username_locator = await self._find_login_field_locator(page, "username")
if not username_locator:
used_direct_form_login = await self._submit_login_via_form_post(page, login_url)
if not used_direct_form_login:
raise RuntimeError("Could not find username input field on login page")
if username_locator is not None:
await username_locator.fill(self.env.username)
password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None
if username_locator is not None and not password_locator:
raise RuntimeError("Could not find password input field on login page")
if password_locator is not None:
await password_locator.fill(self.env.password)
submit_locator = await self._find_submit_locator(page) if username_locator is not None else None
if username_locator is not None and not submit_locator:
raise RuntimeError("Could not find submit button on login page")
if submit_locator is not None:
await submit_locator.click()
if not used_direct_form_login:
try:
await page.wait_for_load_state("load", timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
except Exception:
pass
if not used_direct_form_login and "/login" in page.url:
error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error"
raise RuntimeError(f"Login failed: {error_msg}")
except Exception as e:
raise RuntimeError(f"Login failed: {e!s}")
# 2. Navigate to dashboard
dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true"
if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"):
dashboard_url = dashboard_url.replace("http://", "https://")
if parsed_context:
native_filters = parsed_context.get("native_filters")
active_tabs = parsed_context.get("activeTabs")
if native_filters:
dashboard_url += f"&native_filters={native_filters}"
if active_tabs:
dashboard_url += f"&activeTabs={active_tabs}"
await self._goto_resilient(
page,
dashboard_url,
primary_wait_until="domcontentloaded",
fallback_wait_until="load",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
if "/login" in page.url:
raise RuntimeError("Dashboard navigation redirected to login page after authentication")
# 3. Wait for dashboard content
try:
await page.wait_for_selector(
'.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]',
timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS,
)
try:
await page.wait_for_selector(
".loading, .ant-skeleton, .spinner",
state="hidden",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
except Exception:
pass
try:
await page.wait_for_selector(
".chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
except Exception:
pass
await page.wait_for_function(
"""() => {
const charts = document.querySelectorAll('.chart-container, .slice_container');
if (charts.length === 0) return true;
return Array.from(charts).every(chart => {
const hasCanvas = chart.querySelector('canvas') !== null;
const hasSvg = chart.querySelector('svg') !== null;
const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0;
return hasCanvas || hasSvg || hasContent;
});
}""",
timeout=HTTP_REQUEST_TIMEOUT_MS,
)
await page.evaluate("""async () => {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < document.body.scrollHeight; i += 500) {
window.scrollTo(0, i);
await delay(100);
}
window.scrollTo(0, 0);
await delay(500);
}""")
except Exception:
pass
await self._wait_for_charts_stabilized(page)
logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id})
return browser, context, page
# #endregion ScreenshotService._launch_and_login
# #endregion Plugin.Service.ScreenshotSessionMixin
# #endregion Plugin.Service.Screenshot.SessionModule

View File

@@ -0,0 +1,88 @@
# #region Plugin.Service.Screenshot.WaitModule [C:2] [TYPE Module] [SEMANTICS llm,screenshot,wait,polling]
# @defgroup LLMAnalysis Module group.
# @BRIEF Chart stabilization and debug-screenshot helpers for ScreenshotService (mixin).
# @LAYER Plugin
import asyncio
import os
from ...core.logger import logger
# #region Plugin.Service.ScreenshotWaitMixin [C:2] [TYPE Class]
# @ingroup LLMAnalysis
# @BRIEF Mixin: chart render polling, post-resize re-render wait, debug screenshots.
class ScreenshotWaitMixin:
# #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
# @BRIEF Wait until chart elements have non-zero dimensions, with polling.
# @PRE page is a valid Playwright page.
# @POST Waits for chart stabilization or raises on timeout (handled internally).
# @RATIONALE Polls for actual chart rendering dimensions rather than using a fixed delay — charts may load at different speeds depending on dashboard complexity and network conditions.
# @REJECTED Fixed sleep-based wait rejected — would either waste time (too long) or produce blank screenshots (too short); polling for actual canvas/svg dimensions is more reliable.
async def _wait_for_charts_stabilized(self, page, timeout_ms: int = 15000):
"""Wait until chart elements have non-zero dimensions, with polling."""
# Short initial delay for rendering pipeline to start
await asyncio.sleep(0.5)
try:
await page.wait_for_function(
"""() => {
const charts = document.querySelectorAll('.chart-container canvas, .slice_container svg, .grid-content canvas');
if (charts.length === 0) return true;
return Array.from(charts).some(c => {
if (c.tagName === 'CANVAS') return c.width > 10 && c.height > 10;
if (c.tagName === 'svg') {
const bbox = c.getBoundingClientRect();
return bbox.width > 10 && bbox.height > 10;
}
return false;
});
}""",
timeout=timeout_ms,
)
except Exception:
logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render")
# #endregion ScreenshotService._wait_for_charts_stabilized
# #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
# @BRIEF Wait for charts to re-render after viewport resize.
# @PRE page is a valid Playwright page; chart_count_before contains pre-resize element counts.
# @POST Waits for chart content to return or timeout.
# @RATIONALE After viewport resize, Superset triggers lazy chart re-rendering — this function polls for chart elements to reappear before taking the screenshot.
# @REJECTED Single fixed wait after resize rejected — some dashboards re-render instantly while others take seconds; fixed wait is brittle across dashboard types.
async def _wait_for_resize_rendered(self, page, chart_count_before: dict, timeout_ms: int = 10000):
"""Wait for charts to re-render after viewport resize, with polling."""
try:
await page.wait_for_function(
"""(preCounts) => {
const currentCharts = document.querySelectorAll('.chart-container, .slice_container').length;
const currentCanvases = document.querySelectorAll('canvas').length;
const currentSvgs = document.querySelectorAll('.chart-container svg, .slice_container svg').length;
// At least one chart element must be present
return currentCharts > 0 && (currentCanvases > 0 || currentSvgs > 0);
}""",
arg=chart_count_before,
timeout=timeout_ms,
)
except Exception:
logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize")
# #endregion ScreenshotService._wait_for_resize_rendered
# #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
# @BRIEF Save a debug screenshot to a temp directory for diagnostic purposes.
# @PRE debug_dir exists and is writable.
# @POST Returns the debug path or None on failure.
# @RATIONALE Uses tempfile.mkdtemp() to avoid accumulating .png files in production storage. Temp dir is cleaned up on success (rmtree) or preserved on failure for debugging.
# @REJECTED Old approach saved debug .png next to output path — accumulated _debug_failed_login.png, _preresize.png permanently in screenshots dir (F1). In-memory-only debug logging rejected — screenshot state is visual and cannot be captured in logs.
async def _save_debug_screenshot(self, page, debug_dir: str, suffix: str) -> str | None:
debug_path = os.path.join(debug_dir, suffix)
try:
await page.screenshot(path=debug_path)
return debug_path
except Exception:
return None
# #endregion ScreenshotService._save_debug_screenshot
# #endregion Plugin.Service.ScreenshotWaitMixin
# #endregion Plugin.Service.Screenshot.WaitModule

View File

@@ -1,4 +1,7 @@
#!/usr/bin/env python3
# #region Plugin.LlmAnalysis.AuthDiag [C:3] [TYPE Module] [SEMANTICS superset,auth,diagnostics]
# @defgroup LlmAnalysis Module group.
# @BRIEF Superset auth diagnostics — full-cycle verification of JWT/form/session auth paths.
"""
Superset Auth Diagnostics Script — Extended v3
@@ -31,6 +34,9 @@ except ImportError:
# ── helpers ──────────────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.PrintHelpers [C:1] [TYPE Block]
# @ingroup LlmAnalysis
# @BRIEF Console output primitives for the diagnostic report.
def banner(msg):
print(f"\n{'='*60}")
print(f" {msg}")
@@ -47,13 +53,21 @@ def warn(msg):
def section(msg):
print(f"\n --- {msg} ---")
# #endregion Plugin.LlmAnalysis.AuthDiag.PrintHelpers
# #region Plugin.LlmAnalysis.AuthDiag.HttpClient [C:2] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Build a shared httpx client with optional SSL verification bypass.
def http_client(verify: bool = True) -> httpx.Client:
import warnings
if not verify:
warnings.filterwarnings("ignore", message=".*verify=False.*", category=UserWarning)
return httpx.Client(verify=verify, timeout=30, follow_redirects=False)
# #endregion Plugin.LlmAnalysis.AuthDiag.HttpClient
# #region Plugin.LlmAnalysis.AuthDiag.ExtractSessionCookie [C:2] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Extract session cookie dict from a Set-Cookie header.
def extract_session_cookie(set_cookie_header: str) -> dict | None:
"""Извлекает session=... из Set-Cookie заголовка."""
if not set_cookie_header:
@@ -64,10 +78,14 @@ def extract_session_cookie(set_cookie_header: str) -> dict | None:
value = part.split("=", 1)[1]
return {"name": "session", "value": value}
return None
# #endregion Plugin.LlmAnalysis.AuthDiag.ExtractSessionCookie
# ── 1. Reachability ──────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckReachability [C:3] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Probe Superset reachability via /health/ with base-URL fallback.
def check_reachability(base_url: str, verify: bool) -> bool:
banner("1. Доступность Superset")
with http_client(verify) as c:
@@ -84,10 +102,14 @@ def check_reachability(base_url: str, verify: bool) -> bool:
except httpx.RequestError as e:
fail(f"Недоступен: {e}")
return False
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckReachability
# ── 2. JWT Login ─────────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtLogin [C:3] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Attempt JWT API login across db/ldap providers.
def check_jwt_login(base_url: str, username: str, password: str, verify: bool) -> str | None:
banner("2. JWT API Login")
with http_client(verify) as c:
@@ -107,10 +129,14 @@ def check_jwt_login(base_url: str, username: str, password: str, verify: bool) -
except Exception as e:
fail(f"provider='{provider}' → ошибка: {e}")
return None
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtLogin
# ── 3. JWT endpoints ─────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtEndpoints [C:2] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Verify JWT bearer access to core API endpoints.
def check_jwt_endpoints(base_url: str, token: str, verify: bool):
banner("3. JWT — доступ к API endpoints")
headers = {"Authorization": f"Bearer {token}"}
@@ -122,10 +148,14 @@ def check_jwt_endpoints(base_url: str, token: str, verify: bool):
else fail(f"GET {path} → HTTP {r.status_code}")
except Exception as e:
fail(f"GET {path} → ошибка: {e}")
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtEndpoints
# ── 3b. CSRF token + session cookie ─────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckCsrfSession [C:4] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Fetch CSRF token and probe anonymous/authenticated Set-Cookie session behavior.
def check_csrf_session(base_url: str, token: str, verify: bool) -> tuple[str | None, dict | None]:
"""
GET /api/v1/security/csrf_token/ с JWT.
@@ -173,10 +203,14 @@ def check_csrf_session(base_url: str, token: str, verify: bool) -> tuple[str | N
warn(f"GET / ошибка: {e}")
return csrf_token, csrf_session_cookie
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckCsrfSession
# ── 4. Guest Token ───────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckGuestToken [C:4] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Try guest-token issuance via three strategies: bare JWT, CSRF header, JWT+CSRF+session.
def check_guest_token(base_url: str, token: str,
csrf_token: str | None, csrf_session: dict | None,
verify: bool, dashboard_ids: list[str] | None) -> list[str] | None:
@@ -256,10 +290,14 @@ def check_guest_token(base_url: str, token: str,
warn(f"dashboard #{did}: пропускаем JWT+CSRF+sess — нет csrf_token или csrf_session")
return dashboard_ids
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckGuestToken
# ── 5. Form Login ────────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckFormLogin [C:3] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Attempt classic form login and capture the raw Set-Cookie header.
def check_form_login(base_url: str, username: str, password: str, verify: bool) -> str | None:
"""POST /login/, возвращает сырой Set-Cookie или None."""
banner("5. Form Login (POST /login/)")
@@ -301,10 +339,14 @@ def check_form_login(base_url: str, username: str, password: str, verify: bool)
except Exception as e:
fail(f"POST /login/ ошибка: {e}")
return None
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckFormLogin
# ── 6. Session Cookie → Dashboard ────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckSessionToDashboard [C:3] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Full-cycle check: session cookie against main page and dashboard load.
def check_session_to_dashboard(base_url: str, set_cookie_header: str | None,
verify: bool, dashboard_ids: list[str] | None):
banner("6. Session Cookie → Dashboard (ПОЛНЫЙ ЦИКЛ)")
@@ -358,10 +400,14 @@ def check_session_to_dashboard(base_url: str, set_cookie_header: str | None,
warn(f"dashboard #{did} → HTTP {r.status_code}")
except Exception as e:
fail(f"dashboard #{did} → ошибка: {e}")
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckSessionToDashboard
# ── 7. JWT → UI dashboard ────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtUi [C:2] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Check whether JWT bearer alone can load the dashboard UI.
def check_jwt_ui(base_url: str, token: str, verify: bool, dashboard_ids: list[str] | None):
banner("7. JWT Bearer → Dashboard UI (без session cookie)")
if not dashboard_ids:
@@ -381,10 +427,14 @@ def check_jwt_ui(base_url: str, token: str, verify: bool, dashboard_ids: list[st
warn(f"dashboard #{did} → HTTP {r.status_code}")
except Exception as e:
warn(f"dashboard #{did} → ошибка: {e}")
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtUi
# ── MAIN ──────────────────────────────────────────────────────
# #region Plugin.LlmAnalysis.AuthDiag.Main [C:4] [TYPE Function]
# @ingroup LlmAnalysis
# @BRIEF Orchestrate all auth-path checks and print the summary verdict.
def main():
parser = argparse.ArgumentParser(description="Superset Auth Diagnostics (Extended v3)")
parser.add_argument("--url", required=True, help="Superset base URL")
@@ -466,3 +516,5 @@ def main():
if __name__ == "__main__":
main()
# #endregion Plugin.LlmAnalysis.AuthDiag.Main
# #endregion Plugin.LlmAnalysis.AuthDiag

File diff suppressed because it is too large Load Diff

View File

@@ -21,7 +21,7 @@ from ..models.maintenance import MaintenanceSettings
from ..services.maintenance import end_all_maintenance, end_maintenance, start_maintenance
# #region Plugin.MaintenanceBanner.MaintenanceBannerPlugin [C:4] [TYPE Class]
# #region Plugin.MaintenanceBanner.MaintenanceBannerPlugin.Class [C:4] [TYPE Class]
# @defgroup Plugin Module group.
# @BRIEF Plugin for executing maintenance banner operations asynchronously via TaskManager.
# @RELATION CALLS -> [start_maintenance]
@@ -159,7 +159,7 @@ class MaintenanceBannerPlugin(PluginBase):
finally:
db.close()
# #endregion Plugin.MaintenanceBanner.Execute
# #endregion Plugin.MaintenanceBanner.MaintenanceBannerPlugin
# #endregion Plugin.MaintenanceBanner.MaintenanceBannerPlugin.Class
# #region Plugin.MaintenanceBanner.ReportProgress [C:2] [TYPE Function]

View File

@@ -23,7 +23,7 @@ from ...models.storage import FileCategory, StoredFile
from ...services.storage_service import StorageService
# #region Plugin.StoragePlugin [TYPE Class]
# #region Plugin.StoragePlugin.Class [TYPE Class]
# @defgroup Storage Module group.
# @BRIEF Implementation of the storage management plugin — thin REST adapter over StorageService.
class StoragePlugin(PluginBase):
@@ -274,5 +274,5 @@ class StoragePlugin(PluginBase):
return fp
# endregion Plugin.StoragePlugin.GetFilePath
# #endregion Plugin.StoragePlugin
# #endregion Plugin.StoragePlugin.Class
# #endregion Plugin.StoragePlugin

View File

@@ -34,7 +34,7 @@ from .dictionary import DictionaryManager
from .events import TranslationEventLog
# #region Plugin.BatchProc.BatchProcessingService [C:4] [TYPE Class]
# #region Plugin.BatchProc.BatchProcessingService.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Create batch records, classify rows, process LLM calls, persist results.
class BatchProcessingService:
@@ -411,5 +411,5 @@ class BatchProcessingService:
async def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
await insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
# #endregion Plugin.BatchProc.InsertBatchToTarget
# #endregion Plugin.BatchProc.BatchProcessingService
# #endregion Plugin.BatchProc.BatchProcessingService.Class
# #endregion Plugin.BatchProc.BatchProcessingService

View File

@@ -29,7 +29,7 @@ from ._token_budget import (
from ._utils import estimate_row_tokens
# #region Plugin.BatchSizer.AdaptiveBatchSizer [C:3] [TYPE Class]
# #region Plugin.BatchSizer.AdaptiveBatchSizer.Class [C:3] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Split source rows into auto-sized batches based on token budget estimates.
class AdaptiveBatchSizer:
@@ -226,5 +226,5 @@ class AdaptiveBatchSizer:
return batches
# #endregion Plugin.BatchSizer.AutoSizeBatches
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer.Class
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer

View File

@@ -38,7 +38,7 @@ from .prompt_builder import ContextAwarePromptBuilder
MAX_RETRIES_PER_BATCH = 3
# #region Plugin.LlmCall.LLMTranslationService [C:4] [TYPE Class]
# #region Plugin.LlmCall.LLMTranslationService.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Call LLM, handle retry/truncation, parse response, persist records.
class LLMTranslationService:
@@ -868,5 +868,5 @@ class LLMTranslationService:
def _parse_llm_response(*a, **kw):
return parse_llm_response(*a, **kw)
# #endregion Plugin.LlmCall.ParseLlmResponse
# #endregion Plugin.LlmCall.LLMTranslationService
# #endregion Plugin.LlmCall.LLMTranslationService.Class
# #endregion Plugin.LlmCall.LLMTranslationService

View File

@@ -23,7 +23,7 @@ from ...models.translate import TranslationJob, TranslationRecord, TranslationRu
from ._run_source import _extract_chart_data_rows, fetch_source_rows
# #region Plugin.RunService.RunExecutionService [C:4] [TYPE Class]
# #region Plugin.RunService.RunExecutionService.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Orchestrate full translation run: fetch data, process batches, finalize.
class RunExecutionService:
@@ -187,5 +187,5 @@ class RunExecutionService:
lang_stat.token_count = total_tokens // num_langs
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
self.db.flush()
# #endregion Plugin.RunService.RunExecutionService
# #endregion Plugin.RunService.RunExecutionService.Class
# #endregion Plugin.RunService.RunExecutionService

View File

@@ -14,7 +14,6 @@
# CJK ratio 1.5 — too optimistic for Qwen/DeepSeek tokenizers (actual ~1.0-1.2).
# Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length).
# #endregion Plugin.TokenBudget.EstimateTokenBudget
# #region Plugin.TokenBudget.DEFAULTCONTEXTWINDOW [TYPE Constant]
# @ingroup Translate
# @BRIEF Conservative default when provider has no context_window. Local LM Studio often

View File

@@ -41,7 +41,7 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
DEFAULT_RETENTION_DAYS = 90
# #region Plugin.Events.TranslationEventLog [C:5] [TYPE Class]
# #region Plugin.Events.TranslationEventLog.Class [C:5] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
# @PRE Database session is available.
@@ -275,5 +275,5 @@ class TranslationEventLog:
# endregion Plugin.Events.GetRunEventSummary
# #endregion Plugin.Events.TranslationEventLog
# #endregion Plugin.Events.TranslationEventLog.Class
# #endregion Plugin.Events.TranslationEventLog

View File

@@ -47,7 +47,7 @@ __all__ = [
]
# #endregion Plugin.Executor.TranslationExecutor
# #region Plugin.Executor.TranslationExecutor [C:4] [TYPE Class] [SEMANTICS translate,executor,orchestrator]
# #region Plugin.Executor.TranslationExecutor.Class [C:4] [TYPE Class] [SEMANTICS translate,executor,orchestrator]
# @ingroup TranslationExecutor
# @BRIEF Thin orchestrator for translation execution. Delegates heavy logic to sub-services for INV_7 compliance.
# @PRE DB session and ConfigManager provided.
@@ -570,7 +570,6 @@ class TranslationExecutor:
# @RATIONALE The previous elif s == 0 unconditionally marked any zero-success run as FAILED, even when all work was legitimately skipped (new_key_only scheduled runs with 8 skipped rows). This exactly matched the prod symptom "status: FAILED, successful:0, failed:0, skipped:8".
# Separately, f>0 with s>0 (e.g. LLM down but some cache hits) was silently COMPLETED — that hides LLM unavailability from operators.
# @REJECTED Treating sk>0 + s==0 as failure — it would turn healthy "nothing to do" scheduled runs into red alerts and pollute history.
# @TEST_INVARIANT Pure-skip scheduled translation must end COMPLETED (see Test.Executor.FinalizeRun).
# @RELATION BINDS_TO -> [Test.Executor.FinalizeRun]
@staticmethod
def _finalize_run(run: TranslationRun, s: int, f: int, sk: int) -> TranslationRun:
@@ -778,4 +777,4 @@ class TranslationExecutor:
from ._llm_call import LLMTranslationService
return LLMTranslationService._parse_llm_response(response_text, expected_count, target_languages, finish_reason)
# #endregion TranslationExecutor._parse_llm_response
# #endregion Plugin.Executor.TranslationExecutor
# #endregion Plugin.Executor.TranslationExecutor.Class

View File

@@ -29,7 +29,7 @@ def _optional_int(value: Any) -> int | None:
# #endregion Plugin.Metrics.optional_int
# #region Plugin.Metrics.TranslationMetrics [C:3] [TYPE Class]
# #region Plugin.Metrics.TranslationMetrics.Class [C:3] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Aggregate translation metrics from live events and MetricSnapshot.
class TranslationMetrics:
@@ -37,7 +37,7 @@ class TranslationMetrics:
def __init__(self, db: Session):
self.db = db
#region Plugin.Metrics.GetJobMetrics [C:3] [TYPE Function]
# #region Plugin.Metrics.GetJobMetrics [C:3] [TYPE Function]
# @BRIEF Get aggregated metrics for a specific job.
# @PRE job_id exists.
# @POST Returns dict with metrics from events + latest snapshot.
@@ -205,9 +205,9 @@ class TranslationMetrics:
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
"per_language_metrics": per_language,
}
# endregion Plugin.Metrics.GetJobMetrics
# #endregion Plugin.Metrics.GetJobMetrics
# region Plugin.Metrics.GetAllMetrics [C:2] [TYPE Function]
# #region Plugin.Metrics.GetAllMetrics [C:2] [TYPE Function]
# @BRIEF Get aggregated metrics for all jobs by iterating get_job_metrics.
# @POST Returns list of per-job metrics.
def get_all_metrics(self) -> list[dict[str, Any]]:
@@ -218,8 +218,8 @@ class TranslationMetrics:
.all()
)
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
# endregion Plugin.Metrics.GetAllMetrics
# #endregion Plugin.Metrics.GetAllMetrics
# #endregion Plugin.Metrics.TranslationMetrics
# #endregion Plugin.Metrics.TranslationMetrics.Class
# #endregion Plugin.Metrics.TranslationMetrics

View File

@@ -37,7 +37,7 @@ from .orchestrator_runner import TranslationStageRunner
# #endregion Plugin.Orchestrator.TranslationOrchestrator
# #region Plugin.Orchestrator.TranslationOrchestrator [C:5] [TYPE Class]
# #region Plugin.Orchestrator.TranslationOrchestrator.Class [C:5] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
# @PRE DB session and config manager are available.
@@ -266,4 +266,4 @@ class TranslationOrchestrator:
return self._aggregator.get_run_history(job_id, page, page_size)
# endregion Plugin.Orchestrator.GetRunHistory
# #endregion Plugin.Orchestrator.TranslationOrchestrator
# #endregion Plugin.Orchestrator.TranslationOrchestrator.Class

View File

@@ -30,7 +30,7 @@ from .orchestrator_cancel import cancel_run as _cancel_run, retry_insert as _ret
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager
# #region Plugin.OrchestratorRetry.TranslationRunRetryManager [C:4] [TYPE Class] [SEMANTICS translate,retry,manager]
# #region Plugin.OrchestratorRetry.TranslationRunRetryManager.Class [C:4] [TYPE Class] [SEMANTICS translate,retry,manager]
# @ingroup Translate
# @BRIEF Manages retry of failed batches and delegates cancel/insert.
# @PRE db, config, event_log provided.
@@ -189,4 +189,4 @@ class TranslationRunRetryManager:
return _cancel_run(self.db, self.event_log, self.current_user, run_id)
# #endregion TranslationRunRetryManager.cancel_run
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager.Class

View File

@@ -81,11 +81,6 @@ INSERT_RESULT_SKIPPED = "skipped"
# PostgreSQL conflict_target, leading to "ON CONFLICT DO UPDATE cannot affect row a second time"
# errors at the DB level. The safe approach is to fail fast and require target_language_column
# to be included in target_key_cols for MERGE multi-language configurations.
# @TEST_INVARIANT dedup_merge_key_matches_conflict_target -> VERIFIED_BY: [
# test_merge_dedup_preserves_language_rows,
# test_merge_dedup_collapses_single_lang,
# test_merge_precondition_fails_without_lang_in_key
# ]
def dedup_rows_for_merge(
rows: list[dict[str, object]],
key_cols: list[str] | None,

View File

@@ -43,7 +43,7 @@ from .preview_response_parser import (
from .preview_review import PreviewSessionManager
# #region Plugin.Preview.TranslationPreview [C:4] [TYPE Class]
# #region Plugin.Preview.TranslationPreview.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
# @PRE Database session and config manager are available.
@@ -292,7 +292,7 @@ class TranslationPreview:
def get_preview_session(self, job_id: str) -> dict[str, Any]:
return self._session_mgr.get_preview_session(job_id)
# #endregion Plugin.Preview.TranslationPreview
# #endregion Plugin.Preview.TranslationPreview.Class
# Re-export for backward compatibility
# #endregion Plugin.Preview.TranslationPreview

View File

@@ -15,7 +15,7 @@
import json
from typing import Any
# #region Plugin.PromptBuilder.ContextAwarePromptBuilder [C:2] [TYPE Class]
# #region Plugin.PromptBuilder.ContextAwarePromptBuilder.Class [C:2] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority.
@@ -149,7 +149,7 @@ class ContextAwarePromptBuilder:
ContextAwarePromptBuilder.render_entry(entry, priority, row_context)
for entry, priority in results
]
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder.Class
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder

View File

@@ -76,7 +76,7 @@ def _ensure_aware(dt: datetime | None) -> datetime | None:
# #endregion Plugin.Scheduler.EnsureAware
# #region Plugin.Scheduler.TranslationScheduler [C:4] [TYPE Class]
# #region Plugin.Scheduler.TranslationScheduler.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
class TranslationScheduler:
@@ -289,7 +289,7 @@ class TranslationScheduler:
# endregion Plugin.Scheduler.GetNextExecutions
# #endregion Plugin.Scheduler.TranslationScheduler
# #endregion Plugin.Scheduler.TranslationScheduler.Class
# #region Plugin.Scheduler.ExecuteScheduledTranslation [C:4] [TYPE Function]

View File

@@ -26,7 +26,7 @@ from .service_datasource import fetch_datasource_metadata
from .service_utils import _extract_dialect
# #region Plugin.Service.TranslateJobService [TYPE Class]
# #region Plugin.Service.TranslateJobService.Class [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Service for translation job CRUD with validation and Superset integration.
class TranslateJobService:
@@ -313,7 +313,7 @@ class TranslateJobService:
})
return result
# endregion Plugin.Service.FetchAvailableDatasources
# #endregion Plugin.Service.TranslateJobService
# #endregion Plugin.Service.TranslateJobService.Class
# Re-exports for backward compatibility

View File

@@ -249,7 +249,7 @@ def generate_upsert_sql(
# #endregion Plugin.SqlGenerator.GenerateUpsertSql
# #region Plugin.SqlGenerator.SQLGenerator [C:3] [TYPE Class]
# #region Plugin.SqlGenerator.SQLGenerator.Class [C:3] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
# @PRE Job has target_schema, target_table, key columns configured.
@@ -427,5 +427,5 @@ class SQLGenerator:
# endregion SQLGenerator.generate_batch
# #endregion Plugin.SqlGenerator.SQLGenerator
# #endregion Plugin.SqlGenerator.SQLGenerator.Class
# #endregion Plugin.SqlGenerator.SQLGenerator

View File

@@ -21,7 +21,7 @@ from ...core.superset_client import SupersetClient
from ...core.utils.client_registry import get_superset_client
# #region Plugin.SupersetExecutor.SupersetSqlLabExecutor [C:4] [TYPE Class]
# #region Plugin.SupersetExecutor.SupersetSqlLabExecutor.Class [C:4] [TYPE Class]
# @defgroup Translate Module group.
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
# @PRE Valid environment ID and ConfigManager.
@@ -436,5 +436,5 @@ class SupersetSqlLabExecutor:
# endregion Plugin.SupersetExecutor.GetQueryResults
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor.Class
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas [C:5] [TYPE Module] [SEMANTICS baseline,dashboard-testing,dto,schema]
# #region DashboardTesting.Schemas [C:5] [TYPE Module] [SEMANTICS baseline,dashboard-testing,dto,schema]
# @defgroup DashboardTesting Pydantic DTOs for Superset-native dashboard query inspection, filter normalization, execution, comparison, and baseline lifecycle.
# @LAYER DTO
# @RELATION DEPENDS_ON -> [SupersetBaselineEngine.DataModel]
@@ -146,4 +146,4 @@ from .verification import (
VerificationRunRequest as VerificationRunRequest,
)
#endregion DashboardTesting.Schemas
# #endregion DashboardTesting.Schemas

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Candidates [C:3] [TYPE Module] [SEMANTICS baseline,candidates,dto]
# #region DashboardTesting.Schemas.Candidates [C:3] [TYPE Module] [SEMANTICS baseline,candidates,dto]
# @defgroup DashboardTesting.Candidates Candidate request, response, and approval gate schemas.
# @LAYER DTO
@@ -269,5 +269,5 @@ class ApprovalConsumeResponse(BaseModel):
release_commit_hash: str = Field(description="40-char git commit hash")
# #endregion DashboardTesting.Schemas.ApprovalConsumeResponse
#endregion DashboardTesting.Schemas.Candidates
# #endregion DashboardTesting.Schemas.Candidates

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Capture [C:2] [TYPE Module] [SEMANTICS baseline,capture,dto]
# #region DashboardTesting.Schemas.Capture [C:2] [TYPE Module] [SEMANTICS baseline,capture,dto]
# @defgroup DashboardTesting.Capture Server-side capture request schema for metric candidates.
# @LAYER DTO
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Candidates]
@@ -69,4 +69,4 @@ class CaptureCandidateResponse(BaseModel):
source_response_hash: str = Field(description="SHA-256 of the raw Superset response bytes")
# #endregion DashboardTesting.Schemas.CaptureCandidateResponse
#endregion DashboardTesting.Schemas.Capture
# #endregion DashboardTesting.Schemas.Capture

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Catalog [C:3] [TYPE Module] [SEMANTICS baseline,catalog,dto]
# #region DashboardTesting.Schemas.Catalog [C:3] [TYPE Module] [SEMANTICS baseline,catalog,dto]
# @defgroup DashboardTesting.Catalog Baseline entry, immutability block, and catalog container schemas.
# @LAYER DTO
@@ -175,4 +175,4 @@ class BaselineCatalog(BaseModel):
warnings: list[Warning] = Field(default_factory=list)
# #endregion DashboardTesting.Schemas.BaselineCatalog
#endregion DashboardTesting.Schemas.Catalog
# #endregion DashboardTesting.Schemas.Catalog

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Common [C:1] [TYPE Module] [SEMANTICS baseline,common,dto]
# #region DashboardTesting.Schemas.Common [C:1] [TYPE Module] [SEMANTICS baseline,common,dto]
# @defgroup DashboardTesting.Common Shared DTOs — Warning, Provenance.
# @LAYER DTO
@@ -38,4 +38,4 @@ class ApprovalInfo(BaseModel):
at: datetime = Field(description="Timestamp of approval")
# #endregion DashboardTesting.Schemas.ApprovalInfo
#endregion DashboardTesting.Schemas.Common
# #endregion DashboardTesting.Schemas.Common

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Enums [C:1] [TYPE Module] [SEMANTICS baseline,enum]
# #region DashboardTesting.Schemas.Enums [C:1] [TYPE Module] [SEMANTICS baseline,enum]
# @defgroup DashboardTesting.Enums Core enums for dashboard testing schemas.
# @LAYER DTO
# @INVARIANT All enums are StrEnum for JSON serialization compatibility.
@@ -112,4 +112,4 @@ class DiffKind(StrEnum):
TIME_GRAIN_CHANGED = "time_grain_changed"
# #endregion DashboardTesting.Schemas.DiffKind
#endregion DashboardTesting.Schemas.Enums
# #endregion DashboardTesting.Schemas.Enums

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Execution [C:1] [TYPE Module] [SEMANTICS baseline,execution,dto]
# #region DashboardTesting.Schemas.Execution [C:1] [TYPE Module] [SEMANTICS baseline,execution,dto]
# @defgroup DashboardTesting.Execution Query execution request schema.
# @LAYER DTO
@@ -26,4 +26,4 @@ class ExecuteQueryRequest(BaseModel):
max_rows: int = Field(default=10000, le=10000, description="Bounded result limit")
# #endregion DashboardTesting.Schemas.ExecuteQueryRequest
#endregion DashboardTesting.Schemas.Execution
# #endregion DashboardTesting.Schemas.Execution

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Filters [C:2] [TYPE Module] [SEMANTICS baseline,filter,dto]
# #region DashboardTesting.Schemas.Filters [C:2] [TYPE Module] [SEMANTICS baseline,filter,dto]
# @defgroup DashboardTesting.Filters Filter normalization schemas — FilterValue, NormalizedFilter, NormalizedFilterContext.
# @LAYER DTO
@@ -56,4 +56,4 @@ class NormalizeFiltersRequest(BaseModel):
query_model_fingerprint: str | None = None
# #endregion DashboardTesting.Schemas.NormalizeFiltersRequest
#endregion DashboardTesting.Schemas.Filters
# #endregion DashboardTesting.Schemas.Filters

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Inheritance [C:3] [TYPE Module] [SEMANTICS baseline,inheritance,dto,plan]
# #region DashboardTesting.Schemas.Inheritance [C:3] [TYPE Module] [SEMANTICS baseline,inheritance,dto,plan]
# @defgroup DashboardTesting.Inheritance Request/response DTOs for baseline inheritance planning and execution.
# @LAYER DTO
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Catalog]
@@ -99,4 +99,4 @@ class InheritancePlan(BaseModel):
new_entries: list[dict[str, Any]] = Field(default_factory=list, description="New entries needing fresh capture")
# #endregion DashboardTesting.Schemas.InheritancePlan
#endregion DashboardTesting.Schemas.Inheritance
# #endregion DashboardTesting.Schemas.Inheritance

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.QueryModel [C:3] [TYPE Module] [SEMANTICS baseline,query-model,dto]
# #region DashboardTesting.Schemas.QueryModel [C:3] [TYPE Module] [SEMANTICS baseline,query-model,dto]
# @defgroup DashboardTesting.QueryModel Structured chart/dashboard/filter metadata extracted from Superset.
# @LAYER DTO
@@ -134,4 +134,4 @@ class DashboardQueryModel(BaseModel):
query_model_fingerprint: str = Field(description="Deterministic hash of the query model structure")
# #endregion DashboardTesting.Schemas.DashboardQueryModel
#endregion DashboardTesting.Schemas.QueryModel
# #endregion DashboardTesting.Schemas.QueryModel

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.Results [C:2] [TYPE Module] [SEMANTICS baseline,result,comparison,dto]
# #region DashboardTesting.Schemas.Results [C:2] [TYPE Module] [SEMANTICS baseline,result,comparison,dto]
# @defgroup DashboardTesting.Results Normalized values, comparison policies, and comparison results.
# @LAYER DTO
@@ -100,7 +100,7 @@ class ComparisonResult(BaseModel):
evidence_refs: list[str] = Field(default_factory=list, description="036 evidence artifact references")
# #endregion DashboardTesting.Schemas.ComparisonResult
#endregion DashboardTesting.Schemas.Results
# #endregion DashboardTesting.Schemas.Results
# Resolve forward reference for ComparisonPolicy.per_column
ComparisonPolicy.model_rebuild()

View File

@@ -1,4 +1,4 @@
#region ScenarioRegistry.Schemas [C:2] [TYPE Module] [SEMANTICS scenario,registry,dto,response,schema]
# #region ScenarioRegistry.Schemas [C:2] [TYPE Module] [SEMANTICS scenario,registry,dto,response,schema]
# @defgroup ScenarioRegistry Pydantic response DTOs for the Scenario Registry (042).
# @LAYER DTO
# @RELATION DEPENDS_ON -> [ScenarioRegistry.DataModel]
@@ -218,4 +218,4 @@ class ScenarioEditorProposalSaveRequest(BaseModel):
agent_action_id: str | None = None
# #endregion ScenarioEditor.Schemas.ProposalSaveRequest
#endregion ScenarioRegistry.Schemas
# #endregion ScenarioRegistry.Schemas

View File

@@ -1,4 +1,4 @@
#region DashboardTesting.Schemas.StructureDiff [C:2] [TYPE Module] [SEMANTICS baseline,structure-diff,dto]
# #region DashboardTesting.Schemas.StructureDiff [C:2] [TYPE Module] [SEMANTICS baseline,structure-diff,dto]
# @defgroup DashboardTesting.StructureDiff Dashboard structure diff schemas — request + response.
# @LAYER DTO
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Enums]
@@ -53,7 +53,7 @@ class StructureChange(BaseModel):
# #endregion DashboardTesting.Schemas.StructureChange
# #region DashboardTesting.Schemas.StructureDiff [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,release]
# #region DashboardTesting.Schemas.StructureDiff.Class [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,release]
class StructureDiff(BaseModel):
model_config = ConfigDict(extra="forbid")
@@ -64,7 +64,7 @@ class StructureDiff(BaseModel):
changes: list[StructureChange] = Field(default_factory=list)
summary: dict[str, int] = Field(default_factory=dict, description="Counts by severity (critical, warning, info, pass)")
blocked: bool = Field(default=False, description="True if critical changes block progression")
# #endregion DashboardTesting.Schemas.StructureDiff
# #endregion DashboardTesting.Schemas.StructureDiff.Class
# #region DashboardTesting.Schemas.SnapshotCaptureResponse [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,capture,response,provenance]
@@ -87,4 +87,4 @@ class SnapshotCaptureResponse(BaseModel):
warnings: int = Field(default=0, description="Number of warnings from inspection")
# #endregion DashboardTesting.Schemas.SnapshotCaptureResponse
#endregion DashboardTesting.Schemas.StructureDiff
# #endregion DashboardTesting.Schemas.StructureDiff

Some files were not shown because too many files have changed in this diff Show More