chore: accumulate uncommitted workspace changes
This commit is contained in:
@@ -31,7 +31,6 @@ indexing:
|
||||
- '*.html'
|
||||
- 'coverage_html_frontend/'
|
||||
- 'coverage/'
|
||||
- agent/
|
||||
- '*,cover'
|
||||
- docker/
|
||||
source_dirs:
|
||||
@@ -42,9 +41,16 @@ indexing:
|
||||
- backend/tests
|
||||
- frontend/src
|
||||
- frontend/tests
|
||||
- agent/src
|
||||
- agent/tests
|
||||
- shared/src
|
||||
doc_dirs:
|
||||
- docs
|
||||
- specs
|
||||
- .agents
|
||||
- .agents/agents
|
||||
- .agents/skills
|
||||
- .agents/commands
|
||||
- .opencode
|
||||
- .specify
|
||||
- .opencode/agents
|
||||
@@ -246,7 +252,7 @@ tags:
|
||||
multiline: false
|
||||
is_reference: true
|
||||
description: 'Graph dependency edge. Links contracts. Recommended on any function/module with external dependencies.'
|
||||
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES, USES, CONTAINS, BELONGS_TO, ASSOCIATED_WITH]
|
||||
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES]
|
||||
PRE:
|
||||
type: string
|
||||
multiline: true
|
||||
@@ -258,12 +264,12 @@ tags:
|
||||
RATIONALE:
|
||||
type: string
|
||||
multiline: true
|
||||
description: 'Architectural decision rationale. WHY this implementation was chosen. Decision Memory — prevents regression loops.'
|
||||
description: 'Architectural decision rationale. WHY this implementation was chosen. Optional. Omit rather than invent. Synthetic text is a defect (INV_9).'
|
||||
decision_memory: true
|
||||
REJECTED:
|
||||
type: string
|
||||
multiline: true
|
||||
description: 'Rejected alternative and disqualification reason. WHAT was tried and WHY it failed. Decision Memory — active guardrail against re-implementation.'
|
||||
description: 'Rejected alternative and disqualification reason. Optional. Omit rather than invent a generic alternative. Synthetic text is a defect (INV_9).'
|
||||
decision_memory: true
|
||||
DATA_CONTRACT:
|
||||
type: string
|
||||
@@ -438,13 +444,13 @@ belief_runtime:
|
||||
# #endregion AxiomConfig.BeliefRuntime
|
||||
|
||||
# #region AxiomConfig.ComplexityRules [C:2] [TYPE Block] [SEMANTICS config,complexity,rules]
|
||||
# @BRIEF Per-tier tag requirements from GRACE-Poly SSOT. All tags allowed everywhere,
|
||||
# but C4+ require formal contract annotations.
|
||||
# @BRIEF Advisory typical tags from GRACE-Poly SSOT. Empty required lists:
|
||||
# missing tags are not violations (INV_9). Do not fill tags to silence audits.
|
||||
complexity_rules:
|
||||
"4":
|
||||
required: [PRE, POST, SIDE_EFFECT, DATA_CONTRACT]
|
||||
required: []
|
||||
"5":
|
||||
required: [PRE, POST, SIDE_EFFECT, DATA_CONTRACT, INVARIANT]
|
||||
required: []
|
||||
|
||||
# #endregion AxiomConfig.ComplexityRules
|
||||
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -103,6 +103,7 @@ e2e_*.png
|
||||
|
||||
#generated doxygen
|
||||
docs/api/html
|
||||
docs/api/nav/
|
||||
docs/api/build/
|
||||
superset-tools.bundle
|
||||
|
||||
|
||||
35
AGENTS.md
35
AGENTS.md
@@ -163,20 +163,37 @@ CLI-бинарём `doc-gen` из Rust-проекта `axiom-mcp` (соседн
|
||||
# 1. Собрать бинарник (один раз)
|
||||
cargo build --release --bin doc-gen --manifest-path ../axiom-mcp/Cargo.toml
|
||||
|
||||
# 2. Переиндексировать workspace и собрать Doxygen HTML
|
||||
# 2. Фрактальный граф: модули → функции (карты + Doxygen HTML)
|
||||
make docs-nav
|
||||
# эквивалент:
|
||||
../axiom-mcp/target/release/doc-gen \
|
||||
--workspace-root /home/busya/dev/ss-tools \
|
||||
--html /tmp/ss-tools-doxygen
|
||||
|
||||
# 3. Только навигационный граф (без doxygen)
|
||||
../axiom-mcp/target/release/doc-gen \
|
||||
--workspace-root /home/busya/dev/ss-tools \
|
||||
--nav /tmp/ss-tools-nav
|
||||
--nav docs/api/nav \
|
||||
--html docs/api/html
|
||||
```
|
||||
|
||||
Как ходить по графу:
|
||||
|
||||
1. **Модули** — `docs/api/nav/root.map` (или HTML mainpage). Не читать функции с корня.
|
||||
2. **Функции** — `docs/api/nav/<Module>.map` секция `@FUNCTIONS`, либо Doxygen group → `\ingroup` страница функции.
|
||||
|
||||
Примечания:
|
||||
|
||||
- `--workspace-root` передавать **абсолютным** путём (относительный путь ломает
|
||||
проверку `safe_join` при записи в DuckDB).
|
||||
- Результат doxygen: `.dox`-страницы, `Doxyfile` и каталог `html/` внутри `--html`-директории.
|
||||
- Остальные опции: `doc-gen --help` (формат, per-file вывод, `--filter`, `--group-cap`, `--body-lines`).
|
||||
- `make docs-doxygen` — отдельный XML/HTML extract из исходных комментариев в `docs/api/build`.
|
||||
- Навигационный граф агента — `doc-gen --nav/--html`, не плоский список `axiom_*.html`.
|
||||
- Остальные опции: `doc-gen --help` (`--filter`, `--group-cap`, `--body-lines`).
|
||||
|
||||
## Agent prompts and skills
|
||||
|
||||
`.agents/skills/` is the canonical source for semantic skills. The Kilo runtime
|
||||
loads the generated copy from `.kilo/skills/`; after changing a skill, run:
|
||||
|
||||
```bash
|
||||
./scripts/sync-skills.sh
|
||||
```
|
||||
|
||||
Do not edit `.kilo/skills/` directly. Agent prompts follow the same source/runtime
|
||||
split: canonical prompts are in `.agents/agents/` and the Kilo-loaded copies are
|
||||
in `.kilo/agents/`.
|
||||
|
||||
18
Makefile
18
Makefile
@@ -20,7 +20,7 @@ TIMEOUT_SLOW := 600
|
||||
.PHONY: help test test-unit test-frontend test-related test-integration test-e2e test-all
|
||||
.PHONY: coverage coverage-backend coverage-frontend
|
||||
.PHONY: lint lint-backend lint-frontend
|
||||
.PHONY: docs-doxygen docs-doxygen-check
|
||||
.PHONY: docs-doxygen docs-doxygen-check docs-nav
|
||||
|
||||
# ── Help ───────────────────────────────────────────────────
|
||||
help: ## Show this help message
|
||||
@@ -88,7 +88,21 @@ lint-frontend: ## Frontend eslint
|
||||
@cd $(FRONTEND) && npx eslint .
|
||||
|
||||
# ── Documentation ───────────────────────────────────────────
|
||||
docs-doxygen: ## Generate Doxygen XML and HTML documentation
|
||||
DOC_GEN ?= $(ROOT)/../axiom-mcp/target/release/doc-gen
|
||||
|
||||
docs-nav: ## Fractal module→function navigation graph (doc-gen --nav + --html)
|
||||
@if [ ! -x "$(DOC_GEN)" ]; then \
|
||||
echo " ▶ Building doc-gen..."; \
|
||||
cargo build --release --bin doc-gen --manifest-path "$(ROOT)/../axiom-mcp/Cargo.toml"; \
|
||||
fi
|
||||
@echo " ▶ Navigation graph (modules → functions)..."
|
||||
@LD_LIBRARY_PATH="$(ROOT)/../axiom-mcp/target/release/deps:$$LD_LIBRARY_PATH" \
|
||||
"$(DOC_GEN)" --workspace-root "$(ROOT)" --nav "$(ROOT)/docs/api/nav" --html "$(ROOT)/docs/api/html"
|
||||
@test -f "$(ROOT)/docs/api/nav/root.map"
|
||||
@echo " ✅ Nav maps: docs/api/nav/root.map"
|
||||
@echo " ✅ HTML: docs/api/html/index.html"
|
||||
|
||||
docs-doxygen: ## Generate Doxygen XML and HTML documentation from source comments
|
||||
@if ! command -v doxygen >/dev/null 2>&1; then \
|
||||
echo " ✗ doxygen not found."; \
|
||||
echo " Debian/Ubuntu: sudo apt-get install -y doxygen"; \
|
||||
|
||||
@@ -155,9 +155,6 @@ def _guard_tool_permission(tool_name: str) -> None:
|
||||
# @BRIEF Fixed-delay retry wrapper for read-only tool HTTP calls on transient errors.
|
||||
# @PRE Tool is read-only (risk_level="safe"). Error is 5xx or httpx.ConnectError.
|
||||
# @POST Retries once with fixed 1s asyncio.sleep. On success: returns response. On exhaust: raises with retry_exhausted=True.
|
||||
# @TEST_EDGE: first_attempt_502 -> Auto-retries once, succeeds.
|
||||
# @TEST_EDGE: both_attempts_502 -> Raises error with retry_exhausted=True.
|
||||
# @TEST_EDGE: write_tool_502 -> No retry, raises immediately.
|
||||
# @RATIONALE 1 retry with 1s delay catches ~80% of transient failures. Read-only only.
|
||||
# @REJECTED Exponential backoff — single retry cannot be exponential.
|
||||
# @REJECTED Retry all tools — write operations not idempotent.
|
||||
@@ -237,7 +234,6 @@ def _summarise_response(text: str, limit: int = 4000) -> str:
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Configurable timeout wrapper (default 30s). Write tools: retryable=false.
|
||||
# @POST Returns tool result within timeout. On TimeoutError: yields tool_timeout SSE.
|
||||
# @TEST_EDGE: complete_under_timeout→normal, read_exceed→retryable timeout, write_exceed→retryable=false
|
||||
|
||||
async def _execute_with_timeout(tool_name: str, tool_fn, is_write: bool = False, timeout_s: int = 30):
|
||||
"""Execute tool with configurable timeout. Write tools: no retry on timeout.
|
||||
@@ -365,7 +361,6 @@ class SearchDashboardsInput(BaseModel):
|
||||
# #region AgentChat.Tools.SearchDashboards [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,search,dashboard]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Search and list dashboards by name, with optional environment filter.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns formatted dashboard list string.
|
||||
# @POST Response surfaces effective profile filter metadata so hidden dashboards are never reported as absent.
|
||||
# @RATIONALE Agent search must see the full environment catalog: without page_context=other the backend
|
||||
@@ -460,7 +455,6 @@ class HealthSummaryInput(BaseModel):
|
||||
# #region AgentChat.Tools.HealthSummary [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,health,summary]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Get system health summary — dashboard validation status, recent failures.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns health summary text (trimmed to 2000 chars).
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/health/summary.
|
||||
@tool(args_schema=HealthSummaryInput)
|
||||
@@ -485,7 +479,6 @@ async def get_health_summary(env_id: str | None = None) -> str:
|
||||
# #region AgentChat.Tools.ListEnvironments [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,environment,list]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF List configured deployment environments.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns JSON string of environments (sensitive fields redacted).
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/settings/environments.
|
||||
@tool
|
||||
@@ -503,7 +496,6 @@ async def list_environments() -> str:
|
||||
# #region AgentChat.Tools.TaskStatus [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,task,status]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Check the status of a background task by its task_id.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns task status text from FastAPI.
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/tasks/{task_id}.
|
||||
@tool
|
||||
@@ -535,7 +527,6 @@ async def show_capabilities() -> str:
|
||||
# #region AgentChat.Tools.ListLlmProviders [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,llm,providers]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF List configured LLM providers.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns provider list text.
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/llm/providers.
|
||||
@tool
|
||||
@@ -553,7 +544,6 @@ async def list_llm_providers() -> str:
|
||||
# #region AgentChat.Tools.LlmStatus [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,llm,status]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Check whether the LLM runtime is configured and usable.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns LLM status text.
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/llm/status.
|
||||
@tool
|
||||
@@ -579,7 +569,6 @@ class RunBackupInput(BaseModel):
|
||||
# #region AgentChat.Tools.RunBackup [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,backup,operation]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Run a Superset backup for an environment, optionally scoped to dashboard IDs.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns task creation result (201).
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/tasks (superset-backup plugin).
|
||||
@tool(args_schema=RunBackupInput)
|
||||
@@ -617,7 +606,6 @@ class ExecuteMigrationInput(BaseModel):
|
||||
# #region AgentChat.Tools.ExecuteMigration [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,migration,operation]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Execute dashboard migration between two environments.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns migration result text.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/migration/execute.
|
||||
@tool(args_schema=ExecuteMigrationInput)
|
||||
@@ -667,7 +655,6 @@ class CreateBranchInput(GitDashboardInput):
|
||||
# #region AgentChat.Tools.CreateBranch [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,branch]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Create a branch in a dashboard Git repository.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns branch creation result.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/branches.
|
||||
@tool(args_schema=CreateBranchInput)
|
||||
@@ -704,7 +691,6 @@ class CommitChangesInput(GitDashboardInput):
|
||||
# #region AgentChat.Tools.CommitChanges [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,commit]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Stage and commit changes in a dashboard Git repository.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns commit result.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/commit.
|
||||
@tool(args_schema=CommitChangesInput)
|
||||
@@ -740,7 +726,6 @@ class DeployDashboardInput(GitDashboardInput):
|
||||
# #region AgentChat.Tools.DeployDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,git,deploy]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Deploy a dashboard from Git to a target environment.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns deployment result.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/git/repositories/{ref}/deploy.
|
||||
@tool(args_schema=DeployDashboardInput)
|
||||
@@ -892,7 +877,6 @@ async def run_llm_validation(
|
||||
# #region AgentChat.Tools.ListMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,list]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF List active and completed maintenance events.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns maintenance events text.
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/maintenance/events.
|
||||
@tool
|
||||
@@ -920,7 +904,6 @@ class StartMaintenanceInput(BaseModel):
|
||||
# #region AgentChat.Tools.StartMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,start]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Start a maintenance event and apply banners to affected dashboards.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns maintenance start result (200/202).
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/maintenance/start.
|
||||
@tool(args_schema=StartMaintenanceInput)
|
||||
@@ -962,7 +945,6 @@ class EndMaintenanceInput(BaseModel):
|
||||
# #region AgentChat.Tools.EndMaintenance [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,maintenance,end]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF End one maintenance event, or end all active events when end_all is true.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns maintenance end result (202).
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/maintenance/{id}/end or /api/maintenance/end-all.
|
||||
@tool(args_schema=EndMaintenanceInput)
|
||||
@@ -1139,7 +1121,6 @@ class AuditPermissionsInput(BaseModel):
|
||||
# #region AgentChat.Tools.SupersetAudit [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,audit,permissions]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Audit access rights: user x dashboard x dataset x RLS permission matrix.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns audit report with per-user dashboard/dataset access and RLS regions.
|
||||
# @SIDE_EFFECT HTTP GET to FastAPI /api/agent/superset/audit/permissions.
|
||||
@tool(args_schema=AuditPermissionsInput)
|
||||
@@ -1176,7 +1157,6 @@ class CreateSupersetDashboardInput(BaseModel):
|
||||
# #region AgentChat.Tools.SupersetCreateDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dashboard,create]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Create a new dashboard in Superset.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns created dashboard dict.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/dashboards.
|
||||
@tool(args_schema=CreateSupersetDashboardInput)
|
||||
@@ -1215,7 +1195,6 @@ class CopySupersetDashboardInput(BaseModel):
|
||||
# #region AgentChat.Tools.SupersetCopyDashboard [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dashboard,copy]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Deep-copy a Superset dashboard including all charts.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns copy result dict.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/dashboards/{id}/copy.
|
||||
@tool(args_schema=CopySupersetDashboardInput)
|
||||
@@ -1250,7 +1229,6 @@ class CreateSupersetDatasetInput(BaseModel):
|
||||
# #region AgentChat.Tools.SupersetCreateDataset [C:3] [TYPE Function] [SEMANTICS agent-chat,tools,superset,dataset,create]
|
||||
# @ingroup AgentChat
|
||||
# @BRIEF Create a new dataset in Superset.
|
||||
# @PRE User authenticated via dual-identity JWT.
|
||||
# @POST Returns created dataset dict.
|
||||
# @SIDE_EFFECT HTTP POST to FastAPI /api/agent/superset/datasets.
|
||||
@tool(args_schema=CreateSupersetDatasetInput)
|
||||
|
||||
@@ -43,9 +43,6 @@ router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
# @SIDE_EFFECT DB read for user verification; writes security event LOGIN.
|
||||
# @SIDE_EFFECT Molecular CoT: REASON on entry, REFLECT on success, EXPLORE on failure.
|
||||
# @RELATION CALLS -> [Services.Auth.Service]
|
||||
# @TEST_EDGE: invalid_credentials -> 401
|
||||
# @TEST_EDGE: locked_account -> 423
|
||||
# @TEST_EDGE: missing_fields -> 422
|
||||
|
||||
@router.post("/login", response_model=Token)
|
||||
async def login_for_access_token(
|
||||
@@ -186,8 +183,6 @@ async def get_session_policy(
|
||||
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.GetCurrentUser]
|
||||
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.TouchSessionActivity]
|
||||
# @RELATION DEPENDS_ON -> [Services.Auth.Service]
|
||||
# @TEST_EDGE: idle_expired -> 401 SESSION_IDLE_EXPIRED
|
||||
# @TEST_EDGE: renewed -> 200 with new access_token and same sid
|
||||
|
||||
@router.post("/session/activity", response_model=SessionActivityResponse)
|
||||
async def post_session_activity(
|
||||
@@ -256,7 +251,6 @@ async def post_session_activity(
|
||||
# @RELATION DEPENDS_ON -> [Dependencies.AppDependencies.GetCurrentUser]
|
||||
# @RELATION CALLS -> [Auth.Jwt.BlacklistToken]
|
||||
# @RELATION CALLS -> [Services.Auth.Service]
|
||||
# @TEST_EDGE: already_expired_token -> 200 (idempotent)
|
||||
|
||||
@router.post("/logout")
|
||||
async def logout(
|
||||
@@ -321,8 +315,6 @@ async def login_adfs(request: starlette.requests.Request):
|
||||
# @SIDE_EFFECT DB write for user provisioning; writes security event LOGIN_ADFS.
|
||||
# @SIDE_EFFECT Molecular CoT: REASON/REFLECT/EXPLORE markers.
|
||||
# @RELATION CALLS -> [Services.Auth.Service]
|
||||
# @TEST_EDGE: adfs_timeout -> 504
|
||||
# @TEST_EDGE: invalid_state -> 401
|
||||
|
||||
@router.get("/callback/adfs", name="auth_callback_adfs")
|
||||
async def auth_callback_adfs(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting.Package [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing,package]
|
||||
# #region Api.DashboardTesting.Package [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing,package]
|
||||
# @defgroup Api Dashboard testing API route package — core + structure-diff + structure-snapshot + verification-runs.
|
||||
# @LAYER API
|
||||
# @BRIEF Combined router from submodules: core (existing), structure (feature-037),
|
||||
@@ -50,4 +50,4 @@ router.include_router(scenario_automation_router)
|
||||
|
||||
# Re-export for backward-compatible imports
|
||||
__all__ = ["router"]
|
||||
#endregion Api.DashboardTesting.Package
|
||||
# #endregion Api.DashboardTesting.Package
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing]
|
||||
# #region Api.DashboardTesting [C:4] [TYPE Module] [SEMANTICS baseline,api,routes,dashboard-testing]
|
||||
# @defgroup Api Dashboard testing API routes — inspection, execution, comparison, baselines, candidates.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.QueryModel.Inspect]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting.Inheritance [C:4] [TYPE Module] [SEMANTICS baseline,api,inheritance,plan,execute]
|
||||
# #region Api.DashboardTesting.Inheritance [C:4] [TYPE Module] [SEMANTICS baseline,api,inheritance,plan,execute]
|
||||
# @defgroup Api Baseline inheritance API endpoints — plan and execute.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.Inheritance.PlanInheritance]
|
||||
@@ -174,4 +174,4 @@ async def inheritance_execute(
|
||||
) from err
|
||||
# #endregion Api.DashboardTesting.Inheritance.ExecuteEndpoint
|
||||
|
||||
#endregion Api.DashboardTesting.Inheritance
|
||||
# #endregion Api.DashboardTesting.Inheritance
|
||||
|
||||
@@ -107,7 +107,6 @@ class ScenarioDispositionRequest(BaseModel):
|
||||
# @BRIEF Deterministically compile a dashboard goal into a ScenarioGraph.
|
||||
# @PRE Caller has dashboard-testing WRITE permission.
|
||||
# @POST Returns ScenarioResponse (graph + validation) or 422 for invalid canonical inputs.
|
||||
# @TEST_EDGE invalid_inputs -> 422 VALIDATION_ERROR.
|
||||
@router.post("/scenarios/compile", status_code=status.HTTP_200_OK)
|
||||
def api_compile_scenario(
|
||||
req: ScenarioCompileRequest,
|
||||
@@ -250,7 +249,6 @@ def api_draft_pack(
|
||||
# @ingroup Api
|
||||
# @BRIEF Execute a screenshot capture step via the 036 Evidence bridge.
|
||||
# @POST Returns artifact refs (original + masked when mask_selectors present).
|
||||
# @TEST_EDGE invalid_profile -> 422.
|
||||
@router.post("/scenarios/{scenario_id}/capture", status_code=status.HTTP_200_OK)
|
||||
def api_capture_screenshot(
|
||||
scenario_id: str,
|
||||
|
||||
@@ -313,8 +313,6 @@ def api_decide_human(run_id: str, body: HumanDecisionRequest, db=_DB, current_us
|
||||
# @RELATION DEPENDS_ON -> [ScenarioExecution.Runner.QueuedDispatch]
|
||||
# @INVARIANT Approval CAS can expose an approved run as queued, but only the separate server
|
||||
# dispatcher may claim and execute its initial frontier.
|
||||
# @TEST_EDGE approve -> queued; deny -> blocked; already_decided -> 409; missing_gate -> 404;
|
||||
# missing_run_prod_scope -> 403; invalid_decision -> 422.
|
||||
@runs_router.post("/{run_id}/approval/decision")
|
||||
def api_decide_approval_gate(run_id: str, body: ApprovalDecisionRequest, db=_DB, current_user=_USER, _=_RUN_PROD_PERMISSION):
|
||||
gate = (
|
||||
|
||||
@@ -111,7 +111,6 @@ def api_list_scenarios(
|
||||
# @BRIEF Atomically register a validated draft pack as a candidate revision.
|
||||
# @PRE Caller has dashboard:testing WRITE; handles identify server-owned artifacts.
|
||||
# @POST Returns scenario/revision ids with materialization_status=materialized; failures roll back both rows.
|
||||
# @TEST_EDGE digest_mismatch -> 409; missing_runner_plan -> 409; access_denied -> 409.
|
||||
@router.post("", response_model=ScenarioCreateResponse, status_code=status.HTTP_201_CREATED)
|
||||
def api_create_scenario(
|
||||
body: ScenarioCreateRequest,
|
||||
@@ -505,7 +504,6 @@ def api_revalidate_scenario(
|
||||
# @BRIEF Load a scenario detail by id (operationId scenarioRegistry.detail; fixes getScenarioDraft 404).
|
||||
# @PRE Caller has dashboard:testing READ.
|
||||
# @POST Returns ScenarioDetailResponse; 404 NOT_FOUND when the scenario is not in the registry.
|
||||
# @TEST_EDGE not_found -> 404 with {"code": "NOT_FOUND"} envelope.
|
||||
@router.get("/{scenario_id}", response_model=ScenarioDetailResponse)
|
||||
def api_get_scenario(
|
||||
scenario_id: str,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting.StructureDiff [C:3] [TYPE Module] [SEMANTICS baseline,api,structure-diff,feature-037]
|
||||
# #region Api.DashboardTesting.StructureDiff [C:3] [TYPE Module] [SEMANTICS baseline,api,structure-diff,feature-037]
|
||||
# @defgroup Api Structure diff API route — POST /dashboard-testing/structure-diff.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.StructureDiff.Service]
|
||||
@@ -51,4 +51,4 @@ async def compute_structure_diff_endpoint(
|
||||
# #endregion Api.DashboardTesting.ComputeStructureDiff
|
||||
|
||||
|
||||
#endregion Api.DashboardTesting.StructureDiff
|
||||
# #endregion Api.DashboardTesting.StructureDiff
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting.StructureSnapshot [C:4] [TYPE Module] [SEMANTICS baseline,api,structure-snapshot,release-bound,capture,diff]
|
||||
# #region Api.DashboardTesting.StructureSnapshot [C:4] [TYPE Module] [SEMANTICS baseline,api,structure-snapshot,release-bound,capture,diff]
|
||||
# @defgroup Api Release-bound structure snapshot API routes — capture + diff with identity validation.
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.StructureSnapshot.Service]
|
||||
@@ -45,6 +45,9 @@ _EXECUTE_PERMISSION = Depends(has_permission("dashboard:testing", "EXECUTE"))
|
||||
_DB_SESSION = Depends(get_db)
|
||||
|
||||
|
||||
# #region Api.DashboardTesting.StructureSnapshot.ResolveEnvById [C:2] [TYPE Function]
|
||||
# @ingroup Api
|
||||
# @BRIEF Resolve a configured Environment by id — raises 404 if unknown.
|
||||
async def _resolve_env_by_id(environment_id: str) -> Environment:
|
||||
"""Resolve a configured Environment by id — raises 404 if unknown."""
|
||||
env = get_config_manager().get_environment(environment_id)
|
||||
@@ -54,6 +57,7 @@ async def _resolve_env_by_id(environment_id: str) -> Environment:
|
||||
detail=f"Environment '{environment_id}' not found.",
|
||||
)
|
||||
return env
|
||||
# #endregion Api.DashboardTesting.StructureSnapshot.ResolveEnvById
|
||||
|
||||
|
||||
# #region Api.DashboardTesting.CaptureReleaseSnapshot [C:5] [TYPE Function] [SEMANTICS baseline,api,capture,release-bound,provenance]
|
||||
@@ -254,4 +258,4 @@ async def diff_release_snapshots_endpoint(
|
||||
) from err
|
||||
# #endregion Api.DashboardTesting.DiffReleaseSnapshots
|
||||
|
||||
#endregion Api.DashboardTesting.StructureSnapshot
|
||||
# #endregion Api.DashboardTesting.StructureSnapshot
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region Api.DashboardTesting.VerificationRuns [C:3] [TYPE Module] [SEMANTICS baseline,api,verification-runs,feature-037]
|
||||
# #region Api.DashboardTesting.VerificationRuns [C:3] [TYPE Module] [SEMANTICS baseline,api,verification-runs,feature-037]
|
||||
# @defgroup Api Verification runs API route — POST create + GET history/detail (037 T081).
|
||||
# @LAYER API
|
||||
# @RELATION DEPENDS_ON -> [BaselineEngine.Verification.Service]
|
||||
@@ -71,8 +71,6 @@ async def create_verification_run_endpoint(
|
||||
# @BRIEF List verification runs chronologically, optionally filtered by dashboard_id/environment_id (037 T081).
|
||||
# @POST Returns VerificationRun[] ordered by created_at desc; matches frontend getVerificationHistory().
|
||||
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
|
||||
# @TEST_EDGE dashboard_filter -> only runs for that dashboard returned.
|
||||
# @TEST_EDGE env_filter -> only runs for that environment returned.
|
||||
@router.get("/verification/history", response_model=list[VerificationRun])
|
||||
def list_verification_history(
|
||||
dashboard_id: int | None = None,
|
||||
@@ -96,7 +94,6 @@ def list_verification_history(
|
||||
# @BRIEF Return a single verification run by id (037 T081).
|
||||
# @POST Returns VerificationRun; 404 when the run does not exist.
|
||||
# @RELATION CALLS -> [BaselineEngine.Verification.RecordToResponse]
|
||||
# @TEST_EDGE missing_run -> 404.
|
||||
@router.get("/verification/{run_id}", response_model=VerificationRun)
|
||||
def get_verification_run(
|
||||
run_id: str,
|
||||
@@ -109,4 +106,4 @@ def get_verification_run(
|
||||
return _record_to_response(record)
|
||||
# #endregion Api.DashboardTesting.VerificationDetail
|
||||
|
||||
#endregion Api.DashboardTesting.VerificationRuns
|
||||
# #endregion Api.DashboardTesting.VerificationRuns
|
||||
|
||||
@@ -14,25 +14,6 @@
|
||||
# @SIDE_EFFECT Performs external calls to Superset API and potentially Git providers.
|
||||
# @DATA_CONTRACT Input(env_id, filters) -> Output(DashboardsResponse)
|
||||
#
|
||||
# @TEST_CONTRACT DashboardsAPI -> {
|
||||
# required_fields: {env_id: string, page: integer, page_size: integer},
|
||||
# optional_fields: {search: string},
|
||||
# invariants: ["Pagination must be valid", "Environment must exist"]
|
||||
# }
|
||||
#
|
||||
# @TEST_FIXTURE dashboard_list_happy -> {
|
||||
# "env_id": "prod",
|
||||
# "expected_count": 1,
|
||||
# "dashboards": [{"id": 1, "title": "Main Revenue"}]
|
||||
# }
|
||||
#
|
||||
# @TEST_EDGE pagination_zero_page -> {"env_id": "prod", "page": 0, "status": 400}
|
||||
# @TEST_EDGE pagination_oversize -> {"env_id": "prod", "page_size": 101, "status": 400}
|
||||
# @TEST_EDGE missing_env -> {"env_id": "ghost", "status": 404}
|
||||
# @TEST_EDGE empty_dashboards -> {"env_id": "empty_env", "expected_total": 0}
|
||||
# @TEST_EDGE external_superset_failure -> {"env_id": "bad_conn", "status": 503}
|
||||
#
|
||||
# @TEST_INVARIANT metadata_consistency -> verifies: [dashboard_list_happy, empty_dashboards]
|
||||
|
||||
from ._action_routes import *
|
||||
from ._detail_routes import *
|
||||
|
||||
@@ -11,10 +11,6 @@
|
||||
# @RATIONALE Centralizes secret inventory and recovery after ENCRYPTION_KEY change.
|
||||
# Without this, operators must manually trace decrypt failures across
|
||||
# scattered API responses.
|
||||
# @TEST_EDGE: empty_payload — POST /recover with empty items list → returns failed/failed
|
||||
# @TEST_EDGE: provider_not_found — recovery for non-existent provider ID → skipped
|
||||
# @TEST_EDGE: connection_not_found — recovery for non-existent connection ID → skipped
|
||||
# @TEST_EDGE: invalid_type — recovery item with unknown type → skipped
|
||||
|
||||
import hashlib
|
||||
|
||||
|
||||
@@ -127,7 +127,7 @@ def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig:
|
||||
# #endregion Api.Helpers.GetGitConfigOr404
|
||||
|
||||
|
||||
# #region Api.Helpers.FindDashboardIdBySlug [C:2] [TYPE Function]
|
||||
# #region Api.Helpers.GitHelpers.FindDashboardIdBySlug [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve dashboard numeric ID by slug in a specific environment.
|
||||
async def _find_dashboard_id_by_slug(
|
||||
client: SupersetClient,
|
||||
@@ -158,10 +158,10 @@ async def _find_dashboard_id_by_slug(
|
||||
return None
|
||||
|
||||
|
||||
# #endregion Api.Helpers.FindDashboardIdBySlug
|
||||
# #endregion Api.Helpers.GitHelpers.FindDashboardIdBySlug
|
||||
|
||||
|
||||
# #region Api.Helpers.ResolveDashboardIdFromRef [C:2] [TYPE Function]
|
||||
# #region Api.Helpers.GitHelpers.ResolveDashboardIdFromRef [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve dashboard ID from slug-or-id reference for Git routes.
|
||||
async def _resolve_dashboard_id_from_ref(
|
||||
dashboard_ref: str,
|
||||
@@ -192,10 +192,10 @@ async def _resolve_dashboard_id_from_ref(
|
||||
return dashboard_id
|
||||
|
||||
|
||||
# #endregion Api.Helpers.ResolveDashboardIdFromRef
|
||||
# #endregion Api.Helpers.GitHelpers.ResolveDashboardIdFromRef
|
||||
|
||||
|
||||
# #region Api.Helpers.FindDashboardIdBySlugAsync [C:2] [TYPE Function]
|
||||
# #region Api.Helpers.GitHelpers.FindDashboardIdBySlugAsync [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve dashboard numeric ID by slug asynchronously for hot-path Git routes.
|
||||
async def _find_dashboard_id_by_slug_async(
|
||||
client: "AsyncSupersetClient",
|
||||
@@ -226,10 +226,10 @@ async def _find_dashboard_id_by_slug_async(
|
||||
return None
|
||||
|
||||
|
||||
# #endregion Api.Helpers.FindDashboardIdBySlugAsync
|
||||
# #endregion Api.Helpers.GitHelpers.FindDashboardIdBySlugAsync
|
||||
|
||||
|
||||
# #region Api.Helpers.ResolveDashboardIdFromRefAsync [C:2] [TYPE Function]
|
||||
# #region Api.Helpers.GitHelpers.ResolveDashboardIdFromRefAsync [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve dashboard ID asynchronously from slug-or-id reference for hot Git routes.
|
||||
async def _resolve_dashboard_id_from_ref_async(
|
||||
dashboard_ref: str,
|
||||
@@ -266,7 +266,7 @@ async def _resolve_dashboard_id_from_ref_async(
|
||||
await client.aclose()
|
||||
|
||||
|
||||
# #endregion Api.Helpers.ResolveDashboardIdFromRefAsync
|
||||
# #endregion Api.Helpers.GitHelpers.ResolveDashboardIdFromRefAsync
|
||||
|
||||
|
||||
# #region Api.Helpers.ResolveRepoKeyFromRef [C:2] [TYPE Function]
|
||||
|
||||
@@ -14,13 +14,6 @@
|
||||
# @POST Migration tasks are enqueued or dry-run results are computed and returned.
|
||||
# @SIDE_EFFECT Enqueues long-running tasks, potentially mutates ResourceMapping table, and performs remote Superset API calls.
|
||||
# @DATA_CONTRACT [DashboardSelection | QueryParams] -> [TaskResponse | DryRunResult | MappingSummary]
|
||||
# @TEST_CONTRACT [DashboardSelection + configured envs] -> [task_id | dry-run result | sync summary]
|
||||
# @TEST_SCENARIO [invalid_environment] -> [HTTP_400_or_404]
|
||||
# @TEST_SCENARIO [valid_execution] -> [success_payload_with_required_fields]
|
||||
# @TEST_EDGE [missing_field] ->[HTTP_400]
|
||||
# @TEST_EDGE [invalid_type] ->[validation_error]
|
||||
# @TEST_EDGE [external_fail] ->[HTTP_500]
|
||||
# @TEST_INVARIANT [EnvironmentValidationBeforeAction] -> VERIFIED_BY: [invalid_environment, valid_execution]
|
||||
# @RATIONALE Separates API concerns (routing, permission checks, request/response serialization) from core migration business logic, enabling independent evolution of HTTP contract and domain logic.
|
||||
# @REJECTED Embedding API logic directly in the core migration layer was rejected — it would couple HTTP concerns with business logic, making both layers harder to test, version, and maintain independently.
|
||||
|
||||
|
||||
@@ -84,19 +84,6 @@ def _parse_csv_enum_list(raw: str | None, enum_cls, field_name: str) -> list:
|
||||
# @RELATION DEPENDS_ON -> [Models.Report.ReportQuery]
|
||||
# @RELATION DEPENDS_ON -> [Services.ReportService.ReportsService]
|
||||
#
|
||||
# @TEST_CONTRACT ListReportsApi ->
|
||||
# {
|
||||
# required_fields: {page: int, page_size: int, sort_by: str, sort_order: str},
|
||||
# optional_fields: {task_types: str, statuses: str, search: str},
|
||||
# invariants: [
|
||||
# "Returns ReportCollection on success",
|
||||
# "Raises HTTPException 400 for invalid query parameters"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_list_request -> {"page": 1, "page_size": 20}
|
||||
# @TEST_EDGE invalid_task_type_filter -> raises HTTPException(400)
|
||||
# @TEST_EDGE malformed_query -> raises HTTPException(400)
|
||||
# @TEST_INVARIANT consistent_list_payload -> verifies: [valid_list_request]
|
||||
@router.get("", response_model=ReportCollection)
|
||||
async def list_reports(
|
||||
page: int = Query(1, ge=1),
|
||||
|
||||
@@ -231,13 +231,6 @@ async def get_task(
|
||||
# @POST Returns a list of log entries or raises 404.
|
||||
# @RELATION CALLS -> [Core.Manager.TaskManager]
|
||||
# @RELATION DEPENDS_ON -> [Core.Models.LogFilter]
|
||||
# @TEST_CONTRACT TaskLogQueryInput -> List[LogEntry]
|
||||
# @TEST_SCENARIO existing_task_logs_filtered -> Returns filtered logs by level/source/search with pagination.
|
||||
# @TEST_FIXTURE valid_task_with_mixed_logs -> backend/tests/fixtures/task_logs/valid_task_with_mixed_logs.json
|
||||
# @TEST_EDGE missing_task -> Unknown task_id returns 404 Task not found.
|
||||
# @TEST_EDGE invalid_level_type -> Non-string/invalid level query rejected by validation or yields empty result.
|
||||
# @TEST_EDGE pagination_bounds -> offset=0 and limit=1000 remain within API bounds and do not overflow.
|
||||
# @TEST_INVARIANT logs_only_for_existing_task -> VERIFIED_BY: [existing_task_logs_filtered, missing_task]
|
||||
@router.get("/{task_id}/logs")
|
||||
async def get_task_logs(
|
||||
task_id: str,
|
||||
|
||||
@@ -490,14 +490,25 @@ _POLLING_EXACT_PATHS = frozenset({
|
||||
"/api/agent/llm-config",
|
||||
"/api/auth/session/activity",
|
||||
"/api/settings/consolidated",
|
||||
"/api/environments",
|
||||
"/api/auth/me",
|
||||
"/api/auth/session",
|
||||
"/api/settings/features",
|
||||
"/api/settings/allowed-languages",
|
||||
"/api/profile/preferences",
|
||||
"/api/dashboards",
|
||||
"/api/maintenance/dashboard-banners",
|
||||
"/api/maintenance/events",
|
||||
"/api/validation-tasks/status/batch",
|
||||
"/api/git/repositories/status/batch",
|
||||
})
|
||||
|
||||
|
||||
def _is_suppressed_request(request: Request) -> bool:
|
||||
"""Return True for high-frequency polling requests (framing suppressed)."""
|
||||
path = request.url.path
|
||||
return path in _POLLING_EXACT_PATHS or (
|
||||
request.method == "GET" and path.startswith("/api/tasks")
|
||||
return request.method == "GET" and (
|
||||
path in _POLLING_EXACT_PATHS or path.startswith("/api/tasks")
|
||||
)
|
||||
|
||||
|
||||
@@ -884,22 +895,6 @@ def _set_websocket_trace_id(websocket: WebSocket) -> str:
|
||||
# — sends unnecessary data over the network when the server can discard early. Polling
|
||||
# /api/tasks for status changes was rejected — introduces latency and load.
|
||||
#
|
||||
# @TEST_CONTRACT WebSocketLogStreamApi ->
|
||||
# {
|
||||
# required_fields: {websocket: WebSocket, task_id: str},
|
||||
# optional_fields: {source: str, level: str},
|
||||
# invariants: [
|
||||
# "Accepts the WebSocket connection",
|
||||
# "Applies source and level filters correctly to streamed logs",
|
||||
# "Cleans up subscriptions on disconnect"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_ws_connection -> {"task_id": "test_1", "source": "plugin"}
|
||||
# @TEST_EDGE task_not_found_ws -> closes connection or sends error
|
||||
# @TEST_EDGE empty_task_logs -> waits for new logs
|
||||
# @TEST_INVARIANT consistent_streaming -> verifies: [valid_ws_connection]
|
||||
# @TEST_EDGE ws_auth_missing_token -> connection rejected with 4001
|
||||
# @TEST_EDGE ws_auth_invalid_token -> connection rejected with 4001
|
||||
@app.websocket("/ws/logs/{task_id}")
|
||||
async def websocket_endpoint(websocket: WebSocket, task_id: str, source: str = None, level: str = None):
|
||||
"""
|
||||
|
||||
@@ -17,7 +17,7 @@ import asyncio
|
||||
from typing import Any
|
||||
|
||||
|
||||
# #region Core.AsyncJobRunner [C:4] [TYPE Class] [SEMANTICS async,runner,scheduler]
|
||||
# #region Core.AsyncJobRunner.Class [C:4] [TYPE Class] [SEMANTICS async,runner,scheduler]
|
||||
# @ingroup Core
|
||||
# @BRIEF Centralized async/sync bridge for dispatching coroutines from sync threads.
|
||||
# @PRE Event loop is captured at construction time.
|
||||
@@ -126,5 +126,5 @@ class AsyncJobRunner:
|
||||
error=traceback.format_exc())
|
||||
asyncio.run_coroutine_threadsafe(_guarded(), self.loop)
|
||||
# #endregion Core.AsyncJobRunner.Dispatch
|
||||
# #endregion Core.AsyncJobRunner
|
||||
# #endregion Core.AsyncJobRunner.Class
|
||||
# #endregion Core.AsyncJobRunner
|
||||
|
||||
@@ -129,7 +129,6 @@ def _prune_blacklist(db: "Session") -> None:
|
||||
# @SIDE_EFFECT Writes to token_blacklist table; prunes expired entries.
|
||||
# @RELATION DEPENDS_ON -> [Models.Auth.TokenBlacklist]
|
||||
# @RELATION CALLS -> [Auth.Jwt._HashToken]
|
||||
# @TEST_EDGE: already_expired -> skips blacklisting
|
||||
def blacklist_token(token: str, db: "Session") -> None:
|
||||
from sqlalchemy.orm import Session # noqa: F811
|
||||
from ...models.auth import TokenBlacklist
|
||||
|
||||
@@ -40,7 +40,7 @@ from .rate_limiter import invalidate_rate_limiter_config
|
||||
_AUTH_POLICY_FIELDS = ("auth_max_attempts", "auth_attempt_window", "auth_ban_duration")
|
||||
|
||||
|
||||
# #region Core.ConfigManager [C:5] [TYPE Class]
|
||||
# #region Core.ConfigManager.Class [C:5] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Handles application configuration load, validation, mutation, and persistence lifecycle.
|
||||
# @PRE Database is accessible and AppConfigRecord schema is loaded.
|
||||
@@ -638,5 +638,5 @@ class ConfigManager:
|
||||
self.save()
|
||||
return True
|
||||
# #endregion Core.ConfigManager.DeleteEnvironment
|
||||
# #endregion Core.ConfigManager
|
||||
# #endregion Core.ConfigManager.Class
|
||||
# #endregion Core.ConfigManager
|
||||
|
||||
@@ -75,7 +75,7 @@ def _validate_name_uniqueness(
|
||||
# #endregion Core.ConnectionService.ValidateNameUniqueness
|
||||
|
||||
|
||||
# #region Core.ConnectionService [C:3] [TYPE Class] [SEMANTICS settings,connections,service]
|
||||
# #region Core.ConnectionService.Class [C:3] [TYPE Class] [SEMANTICS settings,connections,service]
|
||||
# @ingroup Core
|
||||
# @BRIEF Service class wrapping all DatabaseConnection CRUD + test operations.
|
||||
class ConnectionService:
|
||||
@@ -522,5 +522,5 @@ class ConnectionService:
|
||||
# #endregion Core.ConnectionService.ValidateConnectionRefs
|
||||
|
||||
|
||||
# #endregion Core.ConnectionService
|
||||
# #endregion Core.ConnectionService.Class
|
||||
# #endregion Core.ConnectionService
|
||||
|
||||
@@ -76,17 +76,6 @@ def _require_fernet_key() -> bytes:
|
||||
# run — cached at module level via get_encryption_manager() singleton to eliminate
|
||||
# redundant env-var reads and key validation.
|
||||
#
|
||||
# @TEST_CONTRACT EncryptionManagerModel ->
|
||||
# {
|
||||
# required_fields: {},
|
||||
# invariants: [
|
||||
# "encrypted data can be decrypted back to the original string"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE basic_encryption_cycle -> {"data": "my_secret_key"}
|
||||
# @TEST_EDGE decrypt_invalid_data -> raises Exception
|
||||
# @TEST_EDGE empty_string_encryption -> {"data": ""}
|
||||
# @TEST_INVARIANT symmetric_encryption -> verifies: [basic_encryption_cycle, empty_string_encryption]
|
||||
class EncryptionManager:
|
||||
# region Core.Encryption.EncryptionManagerInit [TYPE Function]
|
||||
# @BRIEF: Initialize the encryption manager with a Fernet key.
|
||||
|
||||
@@ -10,8 +10,6 @@
|
||||
# @POST Mapping synchronization and lookup APIs are available for environment-scoped UUID-to-integer resolution.
|
||||
# @SIDE_EFFECT Reads/writes ResourceMapping rows, emits logs.
|
||||
# @DATA_CONTRACT Input[environment_id, resource_type, uuid] -> Output[remote_integer_id|None]
|
||||
# @TEST_DATA: mock_superset_resources -> {'chart': [{'id': 42, 'uuid': '1234', 'slice_name': 'test'}], 'dataset': [{'id': 99, 'uuid': '5678', 'table_name': 'data'}]}
|
||||
#
|
||||
# @INVARIANT sync_environment must handle remote API failures gracefully.
|
||||
# @RATIONALE Centralizes UUID-to-integer ID resolution for Superset resources because the Superset API uses different ID schemes across endpoints (UUIDs for import/export, integer IDs for CRUD operations), enabling cross-environment migration with consistent resource references.
|
||||
# @REJECTED BackgroundScheduler — was never started; replaced by AsyncJobRunner for async/sync bridge.
|
||||
@@ -36,20 +34,6 @@ from src.models.mapping import Environment, ResourceMapping, ResourceType
|
||||
# @SIDE_EFFECT Performs database writes during sync cycles.
|
||||
# @DATA_CONTRACT Input[db_session] -> Output[IdMappingService]
|
||||
#
|
||||
# @TEST_CONTRACT IdMappingServiceModel ->
|
||||
# {
|
||||
# required_fields: {db_session: Session},
|
||||
# invariants: [
|
||||
# "sync_environment correctly creates or updates ResourceMapping records",
|
||||
# "get_remote_id returns an integer or None",
|
||||
# "get_remote_ids_batch returns a dictionary of valid UUIDs to integers"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_mapping_service -> {"db_session": "MockSession()"}
|
||||
# @TEST_EDGE sync_api_failure -> handles exception gracefully
|
||||
# @TEST_EDGE get_remote_id_not_found -> returns None
|
||||
# @TEST_EDGE get_batch_empty_list -> returns empty dict
|
||||
# @TEST_INVARIANT resilient_fetching -> verifies: [sync_api_failure]
|
||||
class IdMappingService:
|
||||
# #region Core.MappingService.Init [C:2] [TYPE Function]
|
||||
# @BRIEF: Initializes the mapping service.
|
||||
|
||||
@@ -14,14 +14,6 @@
|
||||
# @POST Risk scoring output preserves item list and provides bounded score with derived level.
|
||||
# @SIDE_EFFECT Emits diagnostic logs and performs read-only metadata requests via Superset client.
|
||||
# @DATA_CONTRACT Module[build_risks, score_risks]
|
||||
# @TEST_CONTRACT [source_objects,target_objects,diff,target_client] -> [List[RiskItem]]
|
||||
# @TEST_SCENARIO [overwrite_update_objects] -> [confirmation overwrite_existing item is emitted for each update diff item without increasing risk score]
|
||||
# @TEST_SCENARIO [missing_datasource_dataset] -> [high missing_datasource risk is emitted]
|
||||
# @TEST_SCENARIO [owner_mismatch_dashboard] -> [low owner_mismatch risk is emitted]
|
||||
# @TEST_EDGE [missing_field] -> [object without uuid is ignored by indexer]
|
||||
# @TEST_EDGE [invalid_type] -> [non-list owners input normalizes to empty identifiers]
|
||||
# @TEST_EDGE [external_fail] -> [target_client get_databases exception propagates to caller]
|
||||
# @TEST_INVARIANT [score_upper_bound_100] -> VERIFIED_BY: [severity_weight_aggregation]
|
||||
# @UX_STATE [Idle] -> [N/A backend domain module]
|
||||
# @UX_FEEDBACK [N/A] -> [No direct UI side effects in this module]
|
||||
# @UX_RECOVERY [N/A] -> [Caller-level retry/recovery]
|
||||
|
||||
@@ -16,7 +16,7 @@ from ..logger import belief_scope, logger as app_logger
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# #region Core.Audit.SupersetAuditMixin [C:4] [TYPE Class]
|
||||
# #region Core.Audit.SupersetAuditMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing comprehensive permissions audit across users, dashboards, datasets, and RLS.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -354,5 +354,5 @@ class SupersetAuditMixin:
|
||||
return found
|
||||
# #endregion SupersetAudit.FindDatasourcePermissions
|
||||
|
||||
# #endregion Core.Audit.SupersetAuditMixin
|
||||
# #endregion Core.Audit.SupersetAuditMixin.Class
|
||||
# #endregion Core.Audit.SupersetAuditMixin
|
||||
|
||||
@@ -23,7 +23,7 @@ from ..utils.async_network import AsyncAPIClient
|
||||
from ..utils.network import SupersetAPIError
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.Base.SupersetClientBase [C:4] [TYPE Class]
|
||||
# #region Core.Base.SupersetClientBase.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Base class providing Superset client initialization, auth, pagination, and import/export plumbing.
|
||||
# @RELATION DEPENDS_ON -> [Core.ConfigModels]
|
||||
@@ -300,5 +300,5 @@ class SupersetClientBase:
|
||||
async def aclose(self) -> None:
|
||||
await self.client.aclose()
|
||||
# #endregion Core.Base.Aclose
|
||||
# #endregion Core.Base.SupersetClientBase
|
||||
# #endregion Core.Base.SupersetClientBase.Class
|
||||
# #endregion Core.Base.SupersetClientBase
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region SupersetClient.ChartData.Execute [C:4] [TYPE Module] [SEMANTICS baseline,superset,chart-data,async]
|
||||
# #region SupersetClient.ChartData.Execute [C:4] [TYPE Module] [SEMANTICS baseline,superset,chart-data,async]
|
||||
# @defgroup Core Superset chart-data POST /api/v1/chart/data mixin.
|
||||
# @LAYER Infrastructure
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -30,13 +30,13 @@ class ChartDataResponse:
|
||||
source_response_hash: str = field(repr=False)
|
||||
# #endregion SupersetClient.ChartData.ResponseDTO
|
||||
|
||||
# @region SupersetClient.ChartData.SupersetChartDataMixin [C:4] [TYPE Class]
|
||||
# #region SupersetClient.ChartData.SupersetChartDataMixin [C:4] [TYPE Class]
|
||||
# @defgroup Core Chart data execution mixin for SupersetClient.
|
||||
# @SIDE_EFFECT Async POST to Superset /api/v1/chart/data.
|
||||
class SupersetChartDataMixin:
|
||||
"""Mixin for executing Superset chart-data queries."""
|
||||
|
||||
# @region SupersetClient.ChartData.ExecuteQueryRaw [C:4] [TYPE Function] [SEMANTICS chart-data,raw,response,hash]
|
||||
# #region SupersetClient.ChartData.ExecuteQueryRaw [C:4] [TYPE Function] [SEMANTICS chart-data,raw,response,hash]
|
||||
# @ingroup Core
|
||||
# @BRIEF Execute chart-data query with raw_response=True; return ChartDataResponse with raw bytes + hash.
|
||||
# @PRE Saved chart exists and is accessible.
|
||||
@@ -114,9 +114,9 @@ class SupersetChartDataMixin:
|
||||
except Exception as e:
|
||||
app_logger.explore("Chart data raw execution failed", extra={"chart_id": chart_id}, error=str(e))
|
||||
raise SupersetAPIError(f"Chart data raw execution failed: {e}") from e
|
||||
# @endregion SupersetClient.ChartData.ExecuteQueryRaw
|
||||
# #endregion SupersetClient.ChartData.ExecuteQueryRaw
|
||||
|
||||
# @region SupersetClient.ChartData.ExecuteQuery [C:4] [TYPE Function]
|
||||
# #region SupersetClient.ChartData.ExecuteQuery [C:4] [TYPE Function]
|
||||
# @ingroup Core
|
||||
# @BRIEF Backward-compatible wrapper — execute chart-data query, return only the parsed dict.
|
||||
# @PRE Saved chart exists and is accessible.
|
||||
@@ -150,9 +150,9 @@ class SupersetChartDataMixin:
|
||||
row_limit=row_limit,
|
||||
)
|
||||
return response.parsed
|
||||
# @endregion SupersetClient.ChartData.ExecuteQuery
|
||||
# #endregion SupersetClient.ChartData.ExecuteQuery
|
||||
|
||||
# @region SupersetClient.ChartData.BuildAdhocFilters [C:2] [TYPE Function] [SEMANTICS superse,chart-data,filters]
|
||||
# #region SupersetClient.ChartData.BuildAdhocFilters [C:2] [TYPE Function] [SEMANTICS superse,chart-data,filters]
|
||||
# @ingroup Core
|
||||
# @BRIEF Normalize filter dicts into Superset adhoc filter format, preserving TEMPORAL_RANGE.
|
||||
# @PRE Input filters use subject/comparator/expressionType keys.
|
||||
@@ -191,8 +191,8 @@ class SupersetChartDataMixin:
|
||||
entry["to"] = to_val
|
||||
adhoc_filters.append(entry)
|
||||
return adhoc_filters
|
||||
# @endregion SupersetClient.ChartData.BuildAdhocFilters
|
||||
# #endregion SupersetClient.ChartData.BuildAdhocFilters
|
||||
|
||||
# @endregion SupersetClient.ChartData.SupersetChartDataMixin
|
||||
# #endregion SupersetClient.ChartData.SupersetChartDataMixin
|
||||
|
||||
#endregion SupersetClient.ChartData.Execute
|
||||
# #endregion SupersetClient.ChartData.Execute
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Any, cast
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.Charts.SupersetChartsMixin [C:3] [TYPE Class]
|
||||
# #region Core.Charts.SupersetChartsMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing all chart-related Superset API operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -74,5 +74,5 @@ class SupersetChartsMixin:
|
||||
walk(payload)
|
||||
return found
|
||||
# #endregion Core.Charts.SupersetClientExtractChartIdsFromLayout
|
||||
# #endregion Core.Charts.SupersetChartsMixin
|
||||
# #endregion Core.Charts.SupersetChartsMixin.Class
|
||||
# #endregion Core.Charts.SupersetChartsMixin
|
||||
|
||||
@@ -16,7 +16,7 @@ import httpx
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.DashboardsCrud.SupersetDashboardsCrudMixin [C:3] [TYPE Class]
|
||||
# #region Core.DashboardsCrud.SupersetDashboardsCrudMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing dashboard detail resolution, export, import, and delete operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -430,5 +430,5 @@ class SupersetDashboardsCrudMixin:
|
||||
raise
|
||||
# #endregion Core.DashboardsCrud.SupersetClientGetDashboard
|
||||
|
||||
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin
|
||||
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin.Class
|
||||
# #endregion Core.DashboardsCrud.SupersetDashboardsCrudMixin
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Any, cast
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.DashboardsFilters.SupersetDashboardsFiltersMixin [C:3] [TYPE Class]
|
||||
# #region Core.DashboardsFilters.SupersetDashboardsFiltersMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing dashboard native filter extraction from permalink and URL state.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -236,5 +236,5 @@ class SupersetDashboardsFiltersMixin:
|
||||
)
|
||||
return result
|
||||
# #endregion Core.DashboardsFilters.SupersetClientParseDashboardUrlForFilters
|
||||
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin
|
||||
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin.Class
|
||||
# #endregion Core.DashboardsFilters.SupersetDashboardsFiltersMixin
|
||||
|
||||
@@ -11,7 +11,7 @@ from typing import Any, cast
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.DashboardsList.SupersetDashboardsListMixin [C:3] [TYPE Class]
|
||||
# #region Core.DashboardsList.SupersetDashboardsListMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing dashboard listing and summary projection operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -205,5 +205,5 @@ class SupersetDashboardsListMixin:
|
||||
})
|
||||
return total_count, result
|
||||
# #endregion Core.DashboardsList.SupersetClientGetDashboardsSummaryPage
|
||||
# #endregion Core.DashboardsList.SupersetDashboardsListMixin
|
||||
# #endregion Core.DashboardsList.SupersetDashboardsListMixin.Class
|
||||
# #endregion Core.DashboardsList.SupersetDashboardsListMixin
|
||||
|
||||
@@ -26,7 +26,7 @@ from ._layout_utils import (
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
|
||||
# #region Core.DashboardsWrite.SupersetDashboardsWriteMixin [C:4] [TYPE Class]
|
||||
# #region Core.DashboardsWrite.SupersetDashboardsWriteMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing markdown chart CRUD and dashboard layout manipulation for maintenance banners.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -532,5 +532,5 @@ class SupersetDashboardsWriteMixin:
|
||||
raise
|
||||
# #endregion Core.DashboardsWrite.UnpublishDashboard
|
||||
|
||||
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin
|
||||
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin.Class
|
||||
# #endregion Core.DashboardsWrite.SupersetDashboardsWriteMixin
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Any, cast
|
||||
from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #region Core.Databases.SupersetDatabasesMixin [C:3] [TYPE Class]
|
||||
# #region Core.Databases.SupersetDatabasesMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing all database-related Superset API operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -462,5 +462,5 @@ class SupersetDatabasesMixin:
|
||||
raise
|
||||
# #endregion Core.Databases.SupersetClientGetAvailableEngines
|
||||
|
||||
# #endregion Core.Databases.SupersetDatabasesMixin
|
||||
# #endregion Core.Databases.SupersetDatabasesMixin.Class
|
||||
# #endregion Core.Databases.SupersetDatabasesMixin
|
||||
|
||||
@@ -13,7 +13,7 @@ from ..logger import belief_scope, logger as app_logger
|
||||
from ..utils.network import SupersetAPIError
|
||||
|
||||
|
||||
# #region Core.DatasetsPreview.SupersetDatasetsPreviewMixin [C:4] [TYPE Class]
|
||||
# #region Core.DatasetsPreview.SupersetDatasetsPreviewMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing dataset preview compilation and query context building.
|
||||
# @SIDE_EFFECT Асинхронные HTTP-вызовы к Superset API.
|
||||
@@ -217,5 +217,5 @@ class SupersetDatasetsPreviewMixin:
|
||||
)
|
||||
return payload
|
||||
# #endregion Core.DatasetsPreview.SupersetClientBuildDatasetPreviewQueryContext
|
||||
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin
|
||||
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin.Class
|
||||
# #endregion Core.DatasetsPreview.SupersetDatasetsPreviewMixin
|
||||
|
||||
@@ -12,7 +12,7 @@ from ..logger import belief_scope, logger as app_logger
|
||||
from ..utils.network import SupersetAPIError
|
||||
|
||||
|
||||
# #region Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin [C:3] [TYPE Class]
|
||||
# #region Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing filter normalization and compiled-SQL extraction for dataset preview operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -181,5 +181,5 @@ class SupersetDatasetsPreviewFiltersMixin:
|
||||
f"(diagnostics={response_diagnostics!r})"
|
||||
)
|
||||
# #endregion Core.DatasetsPreviewFilters.SupersetClientExtractCompiledSqlFromPreviewResponse
|
||||
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin
|
||||
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin.Class
|
||||
# #endregion Core.DatasetsPreviewFilters.SupersetDatasetsPreviewFiltersMixin
|
||||
|
||||
@@ -11,7 +11,7 @@ from ..logger import belief_scope, logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
# #region Core.SavedQueries.SupersetSavedQueriesMixin [C:3] [TYPE Class]
|
||||
# #region Core.SavedQueries.SupersetSavedQueriesMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing saved query CRUD operations.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -156,5 +156,5 @@ class SupersetSavedQueriesMixin:
|
||||
raise
|
||||
# #endregion SupersetSavedQueries.Delete
|
||||
|
||||
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin
|
||||
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin.Class
|
||||
# #endregion Core.SavedQueries.SupersetSavedQueriesMixin
|
||||
|
||||
@@ -14,7 +14,7 @@ from .safety import is_dangerous_sql
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
|
||||
# #region Core.SqlLab.SupersetSqlLabMixin [C:4] [TYPE Class]
|
||||
# #region Core.SqlLab.SupersetSqlLabMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing SQL Lab execution, formatting, results retrieval, cost estimation, CSV export, and query history.
|
||||
# @RELATION DEPENDS_ON -> [Core.Base.SupersetClientBase]
|
||||
@@ -261,5 +261,5 @@ class SupersetSqlLabMixin:
|
||||
raise
|
||||
# #endregion SupersetSqlLab.StopQuery
|
||||
|
||||
# #endregion Core.SqlLab.SupersetSqlLabMixin
|
||||
# #endregion Core.SqlLab.SupersetSqlLabMixin.Class
|
||||
# #endregion Core.SqlLab.SupersetSqlLabMixin
|
||||
|
||||
@@ -38,19 +38,6 @@ from .task_logger import TaskLogger
|
||||
# plugin signatures. A global singleton logger was rejected — it cannot scope log
|
||||
# entries to individual task executions when multiple tasks run concurrently.
|
||||
#
|
||||
# @TEST_CONTRACT TaskContextContract ->
|
||||
# {
|
||||
# required_fields: {task_id: str, add_log_fn: Callable, params: dict},
|
||||
# optional_fields: {default_source: str},
|
||||
# invariants: [
|
||||
# "task_id matches initialized logger's task_id",
|
||||
# "logger is a valid TaskLogger instance"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_context -> {"task_id": "123", "add_log_fn": lambda *args: None, "params": {"k": "v"}, "default_source": "plugin"}
|
||||
# @TEST_EDGE missing_task_id -> raises TypeError
|
||||
# @TEST_EDGE missing_add_log_fn -> raises TypeError
|
||||
# @TEST_INVARIANT logger_initialized -> verifies: [valid_context]
|
||||
class TaskContext:
|
||||
"""
|
||||
Execution context provided to plugins during task execution.
|
||||
|
||||
@@ -15,17 +15,6 @@
|
||||
# @RELATION DEPENDS_ON -> [Core.Lifecycle.JobLifecycle]
|
||||
# @RELATION DEPENDS_ON -> [Core.EventBus]
|
||||
# @INVARIANT Task IDs are unique.
|
||||
# @TEST_CONTRACT TaskManagerRuntime -> {
|
||||
# required_fields: {plugin_loader: PluginLoader},
|
||||
# optional_fields: {},
|
||||
# invariants: ["Must use belief_scope for logging"]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_module -> {"manager_initialized": true}
|
||||
# @TEST_EDGE missing_required_field -> {"plugin_loader": null}
|
||||
# @TEST_EDGE empty_response -> {"tasks": []}
|
||||
# @TEST_EDGE invalid_type -> {"plugin_loader": "string_instead_of_object"}
|
||||
# @TEST_EDGE external_failure -> {"db_unavailable": true}
|
||||
# @TEST_INVARIANT logger_compliance -> verifies: [valid_module]
|
||||
# @RATIONALE Decomposed from 708-line monolithic module into four focused modules (TaskGraph,
|
||||
# EventBus, JobLifecycle, and this facade) to satisfy INV_7. TaskManager now delegates
|
||||
# to sub-services while preserving the public API contract.
|
||||
@@ -73,7 +62,7 @@ class TaskManager:
|
||||
JobLifecycle (state machine) into a single TaskManager interface.
|
||||
"""
|
||||
|
||||
# #region Core.Manager.Init [C:4] [TYPE Function] [C:5]
|
||||
# #region Core.Manager.Init [C:4] [TYPE Function]
|
||||
# @BRIEF Initialize sub-services, create add_log callback, start background flusher.
|
||||
# @PRE plugin_loader is initialized.
|
||||
# @POST TaskManager is ready to accept tasks.
|
||||
@@ -168,7 +157,7 @@ class TaskManager:
|
||||
|
||||
# ── Task CRUD delegates to TaskGraph ──
|
||||
|
||||
# #region Core.Manager.GetTask [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.GetTask [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retrieves a task by its ID.
|
||||
def get_task(self, task_id: str) -> Task | None:
|
||||
@@ -181,7 +170,7 @@ class TaskManager:
|
||||
return self.graph.get_all_tasks()
|
||||
# #endregion Core.Manager.GetAllTasks
|
||||
|
||||
# #region Core.Manager.GetTasks [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.GetTasks [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retrieves tasks with pagination and optional status/plugin/search filters.
|
||||
def get_tasks(
|
||||
@@ -196,14 +185,14 @@ class TaskManager:
|
||||
return self.graph.get_tasks(limit, offset, status, plugin_ids, completed_only, search)
|
||||
# #endregion Core.Manager.GetTasks
|
||||
|
||||
# #region Core.Manager.LoadPersistedTasks [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.LoadPersistedTasks [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Load persisted tasks using persistence service.
|
||||
def load_persisted_tasks(self) -> None:
|
||||
self.graph.load_persisted_tasks(limit=100)
|
||||
# #endregion Core.Manager.LoadPersistedTasks
|
||||
|
||||
# #region Core.Manager.ClearTasks [C:4] [TYPE Function] [C:4]
|
||||
# #region Core.Manager.ClearTasks [C:4] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Clears tasks based on status filter (also deletes associated logs).
|
||||
# @SIDE_EFFECT Removes tasks from registry and persistence; cancels futures.
|
||||
@@ -235,7 +224,7 @@ class TaskManager:
|
||||
|
||||
# ── Log delegates to EventBus ──
|
||||
|
||||
# #region Core.Manager.GetTaskLogs [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.GetTaskLogs [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retrieves logs for a specific task (from memory or persistence).
|
||||
def get_task_logs(
|
||||
@@ -249,14 +238,14 @@ class TaskManager:
|
||||
)
|
||||
# #endregion Core.Manager.GetTaskLogs
|
||||
|
||||
# #region Core.Manager.GetTaskLogStats [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.GetTaskLogStats [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Get statistics about logs for a task.
|
||||
def get_task_log_stats(self, task_id: str) -> LogStats:
|
||||
return self.event_bus.get_task_log_stats(task_id)
|
||||
# #endregion Core.Manager.GetTaskLogStats
|
||||
|
||||
# #region Core.Manager.GetTaskLogSources [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.GetTaskLogSources [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Get unique sources for a task's logs.
|
||||
def get_task_log_sources(self, task_id: str) -> list[str]:
|
||||
@@ -265,14 +254,14 @@ class TaskManager:
|
||||
|
||||
# ── Subscription delegates to EventBus ──
|
||||
|
||||
# #region Core.Manager.SubscribeLogs [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.SubscribeLogs [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Subscribes to real-time logs for a task.
|
||||
async def subscribe_logs(self, task_id: str) -> asyncio.Queue:
|
||||
return await self.event_bus.subscribe_logs(task_id)
|
||||
# #endregion Core.Manager.SubscribeLogs
|
||||
|
||||
# #region Core.Manager.UnsubscribeLogs [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.UnsubscribeLogs [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Unsubscribes from real-time logs for a task.
|
||||
def unsubscribe_logs(self, task_id: str, queue: asyncio.Queue):
|
||||
@@ -281,28 +270,28 @@ class TaskManager:
|
||||
|
||||
# ── Status subscribers ──
|
||||
|
||||
# #region Core.Manager.SubscribeStatus [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.SubscribeStatus [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Subscribes to real-time status updates for a task.
|
||||
async def subscribe_status(self, task_id: str) -> asyncio.Queue:
|
||||
return await self.event_bus.subscribe_status(task_id)
|
||||
# #endregion Core.Manager.SubscribeStatus
|
||||
|
||||
# #region Core.Manager.UnsubscribeStatus [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.UnsubscribeStatus [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Unsubscribes from status updates for a task.
|
||||
def unsubscribe_status(self, task_id: str, queue: asyncio.Queue):
|
||||
self.event_bus.unsubscribe_status(task_id, queue)
|
||||
# #endregion Core.Manager.UnsubscribeStatus
|
||||
|
||||
# #region Core.Manager.SubscribeTaskEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.SubscribeTaskEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Subscribes to global task events (all task status changes).
|
||||
async def subscribe_task_events(self) -> asyncio.Queue:
|
||||
return await self.event_bus.subscribe_task_events()
|
||||
# #endregion Core.Manager.SubscribeTaskEvents
|
||||
|
||||
# #region Core.Manager.UnsubscribeTaskEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.UnsubscribeTaskEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Unsubscribes from global task events.
|
||||
def unsubscribe_task_events(self, queue: asyncio.Queue):
|
||||
@@ -311,7 +300,7 @@ class TaskManager:
|
||||
|
||||
# ── Lifecycle delegates to JobLifecycle ──
|
||||
|
||||
# #region Core.Manager.CreateTask [C:4] [TYPE Function] [C:4]
|
||||
# #region Core.Manager.CreateTask [C:4] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Creates and queues a new task for execution.
|
||||
async def create_task(
|
||||
@@ -330,7 +319,7 @@ class TaskManager:
|
||||
return task
|
||||
# #endregion Core.Manager.CreateTask
|
||||
|
||||
# #region Core.Manager.RunTask [C:4] [TYPE Function] [C:4]
|
||||
# #region Core.Manager.RunTask [C:4] [TYPE Function]
|
||||
# @BRIEF Internal method to execute a task with TaskContext support (delegates to lifecycle).
|
||||
# Tracks the asyncio.Task for management/cancellation.
|
||||
async def _run_task(self, task_id: str):
|
||||
@@ -344,7 +333,7 @@ class TaskManager:
|
||||
self._async_tasks.pop(task_id, None)
|
||||
# #endregion Core.Manager.RunTask
|
||||
|
||||
# #region Core.Manager.CancelTask [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.CancelTask [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Cancel a running task by ID.
|
||||
# @PRE Task must be currently tracked as running.
|
||||
@@ -358,28 +347,28 @@ class TaskManager:
|
||||
return True
|
||||
# #endregion Core.Manager.CancelTask
|
||||
|
||||
# #region Core.Manager.ResolveTask [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.ResolveTask [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Resumes a task that is awaiting mapping.
|
||||
async def resolve_task(self, task_id: str, resolution_params: dict[str, Any]):
|
||||
await self.lifecycle.resolve_task(task_id, resolution_params)
|
||||
# #endregion Core.Manager.ResolveTask
|
||||
|
||||
# #region Core.Manager.WaitForResolution [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.WaitForResolution [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Pauses execution and waits for a resolution signal.
|
||||
async def wait_for_resolution(self, task_id: str):
|
||||
await self.lifecycle.wait_for_resolution(task_id)
|
||||
# #endregion Core.Manager.WaitForResolution
|
||||
|
||||
# #region Core.Manager.WaitForInput [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.WaitForInput [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Pauses execution and waits for user input.
|
||||
async def wait_for_input(self, task_id: str):
|
||||
await self.lifecycle.wait_for_input(task_id)
|
||||
# #endregion Core.Manager.WaitForInput
|
||||
|
||||
# #region Core.Manager.AwaitInput [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.AwaitInput [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Transition a task to AWAITING_INPUT state with input request.
|
||||
async def await_input(self, task_id: str, input_request: dict[str, Any]) -> None:
|
||||
@@ -389,7 +378,7 @@ class TaskManager:
|
||||
)
|
||||
# #endregion Core.Manager.AwaitInput
|
||||
|
||||
# #region Core.Manager.ResumeTaskWithPassword [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.ResumeTaskWithPassword [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Resume a task that is awaiting input with provided passwords.
|
||||
async def resume_task_with_password(
|
||||
@@ -407,7 +396,7 @@ class TaskManager:
|
||||
)
|
||||
# #endregion Core.Manager.ResumeTaskWithPassword
|
||||
|
||||
# #region Core.Manager.RetryTask [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Manager.RetryTask [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Retry a failed task by resetting state and re-queuing execution.
|
||||
async def retry_task(self, task_id: str) -> Task:
|
||||
@@ -426,21 +415,21 @@ class TaskManager:
|
||||
|
||||
# ── Maintenance event delegates to EventBus ──
|
||||
|
||||
# #region Core.Manager.SubscribeMaintenanceEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.SubscribeMaintenanceEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Subscribes to global maintenance events.
|
||||
async def subscribe_maintenance_events(self) -> asyncio.Queue:
|
||||
return await self.event_bus.subscribe_maintenance_events()
|
||||
# #endregion Core.Manager.SubscribeMaintenanceEvents
|
||||
|
||||
# #region Core.Manager.UnsubscribeMaintenanceEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.UnsubscribeMaintenanceEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Unsubscribes from global maintenance events.
|
||||
def unsubscribe_maintenance_events(self, queue: asyncio.Queue):
|
||||
self.event_bus.unsubscribe_maintenance_events(queue)
|
||||
# #endregion Core.Manager.UnsubscribeMaintenanceEvents
|
||||
|
||||
# #region Core.Manager.BroadcastMaintenanceEvent [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.BroadcastMaintenanceEvent [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Broadcast a maintenance event to all subscribers.
|
||||
async def broadcast_maintenance_event(self, event: dict):
|
||||
@@ -449,14 +438,14 @@ class TaskManager:
|
||||
|
||||
# ── Dataset event delegates to JobLifecycle ──
|
||||
|
||||
# #region Core.Manager.SubscribeDatasetEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.SubscribeDatasetEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Subscribe to dataset.updated events for an environment.
|
||||
async def subscribe_dataset_events(self, env_id: str) -> asyncio.Queue:
|
||||
return await self.lifecycle.subscribe_dataset_events(env_id)
|
||||
# #endregion Core.Manager.SubscribeDatasetEvents
|
||||
|
||||
# #region Core.Manager.UnsubscribeDatasetEvents [C:2] [TYPE Function] [C:2]
|
||||
# #region Core.Manager.UnsubscribeDatasetEvents [C:2] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF Unsubscribe from dataset.updated events.
|
||||
def unsubscribe_dataset_events(self, env_id: str, queue: asyncio.Queue):
|
||||
|
||||
@@ -45,19 +45,6 @@ from .models import LogEntry, LogFilter, LogStats, Task, TaskLog, TaskStatus
|
||||
# @RELATION DEPENDS_ON -> [Core.Graph.TaskGraph]
|
||||
# @INVARIANT Persistence must handle potentially missing task fields natively.
|
||||
#
|
||||
# @TEST_CONTRACT TaskPersistenceContract ->
|
||||
# {
|
||||
# required_fields: {},
|
||||
# invariants: [
|
||||
# "persist_task creates or updates a record",
|
||||
# "load_tasks retrieves valid Task instances",
|
||||
# "delete_tasks correctly removes records from the database"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_task_persistence -> {"task_id": "123", "status": "PENDING"}
|
||||
# @TEST_EDGE persist_invalid_task_type -> raises Exception
|
||||
# @TEST_EDGE load_corrupt_json_params -> handled gracefully
|
||||
# @TEST_INVARIANT accurate_round_trip -> verifies: [valid_task_persistence, load_corrupt_json_params]
|
||||
class TaskPersistenceService:
|
||||
# #region Core.Persistence.JsonLoadIfNeeded [TYPE Function] [C:1]
|
||||
# @BRIEF: Safely load JSON strings from DB if necessary
|
||||
@@ -137,7 +124,7 @@ class TaskPersistenceService:
|
||||
return str(env.id)
|
||||
return None
|
||||
# #endregion Core.Persistence.ResolveEnvironmentId
|
||||
# #region Core.Persistence.Init [C:2] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.TaskPersistenceService.__init__ [C:2] [TYPE Function]
|
||||
# @BRIEF: Initializes the persistence service.
|
||||
# @PRE None.
|
||||
# @POST Service is ready.
|
||||
@@ -145,8 +132,8 @@ class TaskPersistenceService:
|
||||
with belief_scope("TaskPersistenceService.__init__"):
|
||||
# Use the unified SessionLocal from database.py.
|
||||
pass
|
||||
# #endregion Core.Persistence.Init
|
||||
# #region Core.Persistence.PersistTask [C:3] [TYPE Function] [C:3]
|
||||
# #endregion Core.Persistence.TaskPersistenceService.__init__
|
||||
# #region Core.Persistence.PersistTask [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Persists or updates a single task in the database.
|
||||
# @PRE isinstance(task, Task)
|
||||
@@ -222,7 +209,7 @@ class TaskPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.PersistTask
|
||||
# #region Core.Persistence.PersistTasks [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.PersistTasks [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Persists multiple tasks.
|
||||
# @PRE isinstance(tasks, list)
|
||||
@@ -234,7 +221,7 @@ class TaskPersistenceService:
|
||||
for task in tasks:
|
||||
self.persist_task(task)
|
||||
# #endregion Core.Persistence.PersistTasks
|
||||
# #region Core.Persistence.LoadTasks [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.LoadTasks [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Loads tasks from the database.
|
||||
# @PRE limit is an integer.
|
||||
@@ -300,7 +287,7 @@ class TaskPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.LoadTasks
|
||||
# #region Core.Persistence.DeleteTasks [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.DeleteTasks [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Deletes specific tasks from the database.
|
||||
# @PRE task_ids is a list of strings.
|
||||
@@ -345,31 +332,19 @@ class TaskPersistenceService:
|
||||
# efficient SQL filtering/aggregation. File-per-task logging was rejected — loses
|
||||
# cross-task query capability and complicates lifecycle cleanup.
|
||||
#
|
||||
# @TEST_CONTRACT TaskLogPersistenceContract ->
|
||||
# {
|
||||
# required_fields: {},
|
||||
# invariants: [
|
||||
# "add_logs efficiently saves logs to the database",
|
||||
# "get_logs retrieves properly filtered LogEntry objects"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_log_batch -> {"task_id": "123", "logs": [{"level": "INFO", "message": "msg"}]}
|
||||
# @TEST_EDGE empty_log_list -> no-op behavior
|
||||
# @TEST_EDGE add_logs_db_error -> rollback and log error
|
||||
# @TEST_INVARIANT accurate_log_aggregation -> verifies: [valid_log_batch]
|
||||
class TaskLogPersistenceService:
|
||||
"""
|
||||
Service for persisting and querying task logs.
|
||||
Supports batch inserts, filtering, and statistics.
|
||||
"""
|
||||
# #region Core.Persistence.Init [C:2] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.TaskLogPersistenceService.__init__ [C:2] [TYPE Function]
|
||||
# @BRIEF: Initializes the TaskLogPersistenceService
|
||||
# @PRE config is provided or defaults are used
|
||||
# @POST Service is ready for log persistence
|
||||
def __init__(self, config=None):
|
||||
pass
|
||||
# #endregion Core.Persistence.Init
|
||||
# #region Core.Persistence.AddLogs [C:3] [TYPE Function] [C:3]
|
||||
# #endregion Core.Persistence.TaskLogPersistenceService.__init__
|
||||
# #region Core.Persistence.AddLogs [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Batch insert log entries for a task.
|
||||
# @PRE logs is a list of LogEntry objects.
|
||||
@@ -425,7 +400,7 @@ class TaskLogPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.AddLogs
|
||||
# #region Core.Persistence.GetLogs [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.GetLogs [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Query logs for a task with filtering and pagination.
|
||||
# @PRE task_id is a valid task ID.
|
||||
@@ -480,7 +455,7 @@ class TaskLogPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.GetLogs
|
||||
# #region Core.Persistence.GetLogStats [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.GetLogStats [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Get statistics about logs for a task.
|
||||
# @PRE task_id is a valid task ID.
|
||||
@@ -525,7 +500,7 @@ class TaskLogPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.GetLogStats
|
||||
# #region Core.Persistence.GetSources [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.GetSources [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Get unique sources for a task's logs.
|
||||
# @PRE task_id is a valid task ID.
|
||||
@@ -604,7 +579,7 @@ class TaskLogPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.IterLogsForExport
|
||||
# #region Core.Persistence.DeleteLogsForTask [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.DeleteLogsForTask [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Delete all logs for a specific task.
|
||||
# @PRE task_id is a valid task ID.
|
||||
@@ -628,7 +603,7 @@ class TaskLogPersistenceService:
|
||||
finally:
|
||||
session.close()
|
||||
# #endregion Core.Persistence.DeleteLogsForTask
|
||||
# #region Core.Persistence.DeleteLogsForTasks [C:3] [TYPE Function] [C:3]
|
||||
# #region Core.Persistence.DeleteLogsForTasks [C:3] [TYPE Function]
|
||||
# @ingroup TaskManager
|
||||
# @BRIEF: Delete all logs for multiple tasks.
|
||||
# @PRE task_ids is a list of task IDs.
|
||||
|
||||
@@ -23,19 +23,6 @@ from ..logger import logger as main_cot_logger # noqa: TID252
|
||||
# @INVARIANT All log calls include the task_id and source.
|
||||
# @UX_STATE Idle -> Logging -> (system records log)
|
||||
#
|
||||
# @TEST_CONTRACT TaskLoggerContract ->
|
||||
# {
|
||||
# required_fields: {task_id: str, add_log_fn: Callable},
|
||||
# optional_fields: {source: str},
|
||||
# invariants: [
|
||||
# "All specific log methods (info, error) delegate to _log",
|
||||
# "with_source creates a new logger with the same task_id"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_task_logger -> {"task_id": "test_123", "add_log_fn": lambda *args: None, "source": "test_plugin"}
|
||||
# @TEST_EDGE missing_task_id -> raises TypeError
|
||||
# @TEST_EDGE invalid_add_log_fn -> raises TypeError
|
||||
# @TEST_INVARIANT consistent_delegation -> verifies: [valid_task_logger]
|
||||
class TaskLogger:
|
||||
"""
|
||||
A dedicated logger for tasks that automatically tags logs with source attribution.
|
||||
|
||||
@@ -23,7 +23,7 @@ from ..logger import belief_scope, logger as app_logger
|
||||
# #region Core.Network.SupersetAPIError [C:1] [TYPE Class]
|
||||
# @BRIEF Base exception for all Superset API related errors.
|
||||
class SupersetAPIError(Exception):
|
||||
# #region Core.Network.Init [TYPE Function] [C:1]
|
||||
# #region Core.Network.SupersetAPIError.__init__ [TYPE Function] [C:1]
|
||||
# @BRIEF: Initializes the exception with a message and context.
|
||||
# @PRE message is a string, context is a dict.
|
||||
# @POST Exception is initialized with context.
|
||||
@@ -31,48 +31,48 @@ class SupersetAPIError(Exception):
|
||||
with belief_scope("SupersetAPIError.__init__"):
|
||||
self.context = context
|
||||
super().__init__(f"[API_FAILURE] {message} | Context: {self.context}")
|
||||
# #endregion Core.Network.Init
|
||||
# #endregion Core.Network.SupersetAPIError.__init__
|
||||
# #endregion Core.Network.SupersetAPIError
|
||||
# #region Core.Network.AuthenticationError [C:1] [TYPE Class]
|
||||
# @BRIEF Exception raised when authentication fails.
|
||||
class AuthenticationError(SupersetAPIError):
|
||||
# #region Core.Network.Init [TYPE Function] [C:1]
|
||||
# #region Core.Network.AuthenticationError.__init__ [TYPE Function] [C:1]
|
||||
# @BRIEF: Initializes the authentication error.
|
||||
# @PRE message is a string, context is a dict.
|
||||
# @POST AuthenticationError is initialized.
|
||||
def __init__(self, message: str = "Authentication failed", **context: Any):
|
||||
with belief_scope("AuthenticationError.__init__"):
|
||||
super().__init__(message, type="authentication", **context)
|
||||
# #endregion Core.Network.Init
|
||||
# #endregion Core.Network.AuthenticationError.__init__
|
||||
# #endregion Core.Network.AuthenticationError
|
||||
# #region Core.Network.PermissionDeniedError [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @ingroup Core
|
||||
# @BRIEF Exception raised when access is denied.
|
||||
class PermissionDeniedError(AuthenticationError):
|
||||
# #region Core.Network.Init [TYPE Function]
|
||||
# #region Core.Network.PermissionDeniedError.__init__ [TYPE Function]
|
||||
# @BRIEF: Initializes the permission denied error.
|
||||
# @PRE message is a string, context is a dict.
|
||||
# @POST PermissionDeniedError is initialized.
|
||||
def __init__(self, message: str = "Permission denied", **context: Any):
|
||||
with belief_scope("PermissionDeniedError.__init__"):
|
||||
super().__init__(message, **context)
|
||||
# #endregion Core.Network.Init
|
||||
# #endregion Core.Network.PermissionDeniedError.__init__
|
||||
# #endregion Core.Network.PermissionDeniedError
|
||||
# #region Core.Network.DashboardNotFoundError [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @ingroup Core
|
||||
# @BRIEF Exception raised when a dashboard cannot be found.
|
||||
class DashboardNotFoundError(SupersetAPIError):
|
||||
# #region Core.Network.Init [TYPE Function]
|
||||
# #region Core.Network.DashboardNotFoundError.__init__ [TYPE Function]
|
||||
# @BRIEF: Initializes the not found error with resource ID.
|
||||
# @PRE resource_id is provided.
|
||||
# @POST DashboardNotFoundError is initialized.
|
||||
def __init__(self, resource_id: int | str, message: str = "Dashboard not found", **context: Any):
|
||||
with belief_scope("DashboardNotFoundError.__init__"):
|
||||
super().__init__(f"Dashboard '{resource_id}' {message}", subtype="not_found", resource_id=resource_id, **context)
|
||||
# #endregion Core.Network.Init
|
||||
# #endregion Core.Network.DashboardNotFoundError.__init__
|
||||
# #endregion Core.Network.DashboardNotFoundError
|
||||
# #region Core.Network.NetworkError [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @ingroup Core
|
||||
# @BRIEF Exception raised when a network level error occurs.
|
||||
class NetworkError(Exception):
|
||||
# #region NetworkError.__init__ [TYPE Function]
|
||||
@@ -87,7 +87,7 @@ class NetworkError(Exception):
|
||||
# #endregion NetworkError.__init__
|
||||
# #endregion Core.Network.NetworkError
|
||||
# #region Core.Network.SupersetAuthCache [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @ingroup Core
|
||||
# @BRIEF Process-local cache for Superset access/csrf tokens keyed by environment credentials.
|
||||
# @PRE base_url and username are stable strings.
|
||||
# @POST Cached entries expire automatically by TTL and can be reused across requests.
|
||||
|
||||
@@ -39,7 +39,7 @@ class SupersetParsedContext:
|
||||
partial_recovery: bool = False
|
||||
dataset_payload: dict[str, Any] | None = None
|
||||
# #endregion Core.Base.SupersetParsedContext
|
||||
# #region Core.Base.SupersetContextExtractorBase [C:4] [TYPE Class]
|
||||
# #region Core.Base.SupersetContextExtractorBase.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Bind extractor to one Superset environment and client instance; provide shared URL-parsing helpers.
|
||||
# @RELATION DEPENDS_ON -> [Core.ConfigModels.Environment]
|
||||
@@ -191,4 +191,4 @@ class SupersetContextExtractorBase:
|
||||
query_state[key] = decoded_value
|
||||
return query_state
|
||||
# #endregion SupersetContextExtractorBase._decode_query_state
|
||||
# #endregion Core.Base.SupersetContextExtractorBase
|
||||
# #endregion Core.Base.SupersetContextExtractorBase.Class
|
||||
|
||||
@@ -14,7 +14,7 @@ from ...logger import logger as app_logger
|
||||
|
||||
app_logger = cast(Any, app_logger)
|
||||
# #endregion Core.Filters.FiltersImports
|
||||
# #region Core.Filters.SupersetContextFiltersExtractMixin [C:3] [TYPE Class]
|
||||
# #region Core.Filters.SupersetContextFiltersExtractMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing query-state filter extraction for the composed SupersetContextExtractor.
|
||||
class SupersetContextFiltersExtractMixin:
|
||||
@@ -236,4 +236,4 @@ class SupersetContextFiltersExtractMixin:
|
||||
)
|
||||
return imported_filters
|
||||
# #endregion SupersetContextFiltersExtractMixin._extract_imported_filters
|
||||
# #endregion Core.Filters.SupersetContextFiltersExtractMixin
|
||||
# #endregion Core.Filters.SupersetContextFiltersExtractMixin.Class
|
||||
|
||||
@@ -16,7 +16,7 @@ from ._base import SupersetParsedContext
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion Core.Parsing.ParsingImports
|
||||
# #region Core.Parsing.SupersetContextParsingMixin [C:4] [TYPE Class]
|
||||
# #region Core.Parsing.SupersetContextParsingMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing Superset URL parsing logic for the composed SupersetContextExtractor.
|
||||
class SupersetContextParsingMixin:
|
||||
@@ -356,4 +356,4 @@ class SupersetContextParsingMixin:
|
||||
unresolved_references.append("dashboard_dataset_binding_missing")
|
||||
return None, unresolved_references
|
||||
# #endregion SupersetContextParsingMixin._recover_dataset_binding_from_dashboard
|
||||
# #endregion Core.Parsing.SupersetContextParsingMixin
|
||||
# #endregion Core.Parsing.SupersetContextParsingMixin.Class
|
||||
|
||||
@@ -17,7 +17,7 @@ from ._base import SupersetParsedContext
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion Core.Recovery.RecoveryImports
|
||||
# #region Core.Recovery.SupersetContextRecoveryMixin [C:4] [TYPE Class]
|
||||
# #region Core.Recovery.SupersetContextRecoveryMixin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing filter recovery for the composed SupersetContextExtractor.
|
||||
class SupersetContextRecoveryMixin:
|
||||
@@ -286,4 +286,4 @@ class SupersetContextRecoveryMixin:
|
||||
"notes": str(payload.get("notes") or default_note),
|
||||
}
|
||||
# #endregion SupersetContextRecoveryMixin._normalize_imported_filter_payload
|
||||
# #endregion Core.Recovery.SupersetContextRecoveryMixin
|
||||
# #endregion Core.Recovery.SupersetContextRecoveryMixin.Class
|
||||
|
||||
@@ -14,7 +14,7 @@ from ...logger import belief_scope, logger
|
||||
|
||||
logger = cast(Any, logger)
|
||||
# #endregion Core.Templates.TemplatesImports
|
||||
# #region Core.Templates.SupersetContextTemplatesMixin [C:3] [TYPE Class]
|
||||
# #region Core.Templates.SupersetContextTemplatesMixin.Class [C:3] [TYPE Class]
|
||||
# @defgroup Core Module group.
|
||||
# @BRIEF Mixin providing template variable discovery for the composed SupersetContextExtractor.
|
||||
class SupersetContextTemplatesMixin:
|
||||
@@ -227,4 +227,4 @@ class SupersetContextTemplatesMixin:
|
||||
except ValueError:
|
||||
return normalized_literal
|
||||
# #endregion SupersetContextTemplatesMixin._normalize_default_literal
|
||||
# #endregion Core.Templates.SupersetContextTemplatesMixin
|
||||
# #endregion Core.Templates.SupersetContextTemplatesMixin.Class
|
||||
|
||||
@@ -101,7 +101,6 @@ class MigrationJob(Base):
|
||||
# #region Models.Mapping.ResourceMapping [C:3] [TYPE Class]
|
||||
# @defgroup Models Module group.
|
||||
# @BRIEF Maps a universal UUID for a resource to its actual ID on a specific environment.
|
||||
# @TEST_DATA: resource_mapping_record -> {'environment_id': 'prod-env-1', 'resource_type': 'chart', 'uuid': '123e4567-e89b-12d3-a456-426614174000', 'remote_integer_id': '42'}
|
||||
# @RELATION DEPENDS_ON -> Models.Mapping.MappingModels
|
||||
class ResourceMapping(Base):
|
||||
__tablename__ = "resource_mappings"
|
||||
|
||||
@@ -53,18 +53,6 @@ class ReportStatus(str, Enum):
|
||||
# @INVARIANT The properties accurately describe error state.
|
||||
# @BRIEF Error and recovery context for failed/partial reports.
|
||||
#
|
||||
# @TEST_CONTRACT ErrorContextModel ->
|
||||
# {
|
||||
# required_fields: {
|
||||
# message: str
|
||||
# },
|
||||
# optional_fields: {
|
||||
# code: str,
|
||||
# next_actions: list[str]
|
||||
# }
|
||||
# }
|
||||
# @TEST_FIXTURE basic_error -> {"message": "Connection timeout", "code": "ERR_504", "next_actions": ["retry"]}
|
||||
# @TEST_EDGE missing_message -> {"code": "ERR_504"}
|
||||
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
|
||||
class ErrorContext(BaseModel):
|
||||
code: str | None = None
|
||||
@@ -80,35 +68,6 @@ class ErrorContext(BaseModel):
|
||||
# @INVARIANT Must represent canonical task record attributes.
|
||||
# @BRIEF Canonical normalized report envelope for one task execution.
|
||||
#
|
||||
# @TEST_CONTRACT TaskReportModel ->
|
||||
# {
|
||||
# required_fields: {
|
||||
# report_id: str,
|
||||
# task_id: str,
|
||||
# task_type: TaskType,
|
||||
# status: ReportStatus,
|
||||
# updated_at: datetime,
|
||||
# summary: str
|
||||
# },
|
||||
# invariants: [
|
||||
# "report_id is a non-empty string",
|
||||
# "task_id is a non-empty string",
|
||||
# "summary is a non-empty string"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_task_report ->
|
||||
# {
|
||||
# report_id: "rep-123",
|
||||
# task_id: "task-456",
|
||||
# task_type: "migration",
|
||||
# status: "success",
|
||||
# updated_at: "2026-02-26T12:00:00Z",
|
||||
# summary: "Migration completed successfully"
|
||||
# }
|
||||
# @TEST_EDGE empty_report_id -> {"report_id": " ", "task_id": "task-456", "task_type": "migration", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}
|
||||
# @TEST_EDGE empty_summary -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "migration", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": ""}
|
||||
# @TEST_EDGE invalid_task_type -> {"report_id": "rep-123", "task_id": "task-456", "task_type": "invalid_type", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}
|
||||
# @TEST_INVARIANT non_empty_validators -> verifies: [empty_report_id, empty_summary]
|
||||
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
|
||||
class TaskReport(BaseModel):
|
||||
report_id: str
|
||||
@@ -139,24 +98,6 @@ class TaskReport(BaseModel):
|
||||
# @INVARIANT Time and pagination queries are mutually consistent.
|
||||
# @BRIEF Query object for server-side report filtering, sorting, and pagination.
|
||||
#
|
||||
# @TEST_CONTRACT ReportQueryModel ->
|
||||
# {
|
||||
# optional_fields: {
|
||||
# page: int, page_size: int, task_types: list[TaskType], statuses: list[ReportStatus],
|
||||
# time_from: datetime, time_to: datetime, search: str, sort_by: str, sort_order: str
|
||||
# },
|
||||
# invariants: [
|
||||
# "page >= 1", "1 <= page_size <= 100",
|
||||
# "sort_by in {'updated_at', 'status', 'task_type'}",
|
||||
# "sort_order in {'asc', 'desc'}",
|
||||
# "time_from <= time_to if both exist"
|
||||
# ]
|
||||
# }
|
||||
# @TEST_FIXTURE valid_query -> {"page": 1, "page_size":20, "sort_by": "updated_at", "sort_order": "desc"}
|
||||
# @TEST_EDGE invalid_page_size_large -> {"page_size": 150}
|
||||
# @TEST_EDGE invalid_sort_by -> {"sort_by": "unknown_field"}
|
||||
# @TEST_EDGE invalid_time_range -> {"time_from": "2026-02-26T12:00:00Z", "time_to": "2026-02-25T12:00:00Z"}
|
||||
# @TEST_INVARIANT attribute_constraints_enforced -> verifies: [invalid_page_size_large, invalid_sort_by, invalid_time_range]
|
||||
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
|
||||
class ReportQuery(BaseModel):
|
||||
page: int = Field(default=1, ge=1)
|
||||
@@ -199,15 +140,6 @@ class ReportQuery(BaseModel):
|
||||
# @INVARIANT Represents paginated data correctly.
|
||||
# @BRIEF Paginated collection of normalized task reports.
|
||||
#
|
||||
# @TEST_CONTRACT ReportCollectionModel ->
|
||||
# {
|
||||
# required_fields: {
|
||||
# items: list[TaskReport], total: int, page: int, page_size: int, has_next: bool, applied_filters: ReportQuery
|
||||
# },
|
||||
# invariants: ["total >= 0", "page >= 1", "page_size >= 1"]
|
||||
# }
|
||||
# @TEST_FIXTURE empty_collection -> {"items": [], "total": 0, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}}
|
||||
# @TEST_EDGE negative_total -> {"items": [], "total": -5, "page": 1, "page_size": 20, "has_next": False, "applied_filters": {}}
|
||||
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
|
||||
class ReportCollection(BaseModel):
|
||||
items: list[TaskReport]
|
||||
@@ -226,13 +158,6 @@ class ReportCollection(BaseModel):
|
||||
# @INVARIANT Incorporates a report and logs correctly.
|
||||
# @BRIEF Detailed report representation including diagnostics and recovery actions.
|
||||
#
|
||||
# @TEST_CONTRACT ReportDetailViewModel ->
|
||||
# {
|
||||
# required_fields: {report: TaskReport},
|
||||
# optional_fields: {timeline: list[dict], diagnostics: dict, next_actions: list[str]}
|
||||
# }
|
||||
# @TEST_FIXTURE valid_detail -> {"report": {"report_id": "rep-1", "task_id": "task-1", "task_type": "backup", "status": "success", "updated_at": "2026-02-26T12:00:00Z", "summary": "Done"}}
|
||||
# @TEST_EDGE missing_report -> {}
|
||||
# @RELATION DEPENDS_ON -> Models.Report.ReportModels
|
||||
class ReportDetailView(BaseModel):
|
||||
report: TaskReport
|
||||
@@ -249,7 +174,6 @@ class ReportDetailView(BaseModel):
|
||||
# @BRIEF Aggregated task counts by type and status for the summary dashboard.
|
||||
# @DATA_CONTRACT Output: TaskSummary → frontend interface TaskSummary
|
||||
# @RELATION DEPENDS_ON -> [Models.Report.TaskReport]
|
||||
# @TEST_CONTRACT TaskSummaryModel -> { invariants: ["all counts >= 0", "total == sum of all status counts across all types"] }
|
||||
class StatusCounts(BaseModel):
|
||||
"""Counts per status for a single task type."""
|
||||
pending: int = Field(default=0, ge=0)
|
||||
|
||||
@@ -39,60 +39,6 @@ class TaskRecord(Base):
|
||||
# @RELATION DEPENDS_ON -> Models.Task.TaskRecord
|
||||
# @INVARIANT Each log entry belongs to exactly one task.
|
||||
#
|
||||
# @TEST_CONTRACT TaskLogCreate ->
|
||||
# {
|
||||
# required_fields: {
|
||||
# task_id: str,
|
||||
# timestamp: datetime,
|
||||
# level: str,
|
||||
# source: str,
|
||||
# message: str
|
||||
# },
|
||||
# optional_fields: {
|
||||
# metadata_json: str,
|
||||
# id: int
|
||||
# },
|
||||
# invariants: [
|
||||
# "task_id matches an existing TaskRecord.id"
|
||||
# ]
|
||||
# }
|
||||
#
|
||||
# @TEST_FIXTURE basic_info_log ->
|
||||
# {
|
||||
# task_id: "00000000-0000-0000-0000-000000000000",
|
||||
# timestamp: "2026-02-26T12:00:00Z",
|
||||
# level: "INFO",
|
||||
# source: "system",
|
||||
# message: "Task initialization complete"
|
||||
# }
|
||||
#
|
||||
# @TEST_EDGE missing_required_field ->
|
||||
# {
|
||||
# timestamp: "2026-02-26T12:00:00Z",
|
||||
# level: "ERROR",
|
||||
# source: "system",
|
||||
# message: "Missing task_id"
|
||||
# }
|
||||
#
|
||||
# @TEST_EDGE invalid_type ->
|
||||
# {
|
||||
# task_id: "00000000-0000-0000-0000-000000000000",
|
||||
# timestamp: "2026-02-26T12:00:00Z",
|
||||
# level: 500,
|
||||
# source: "system",
|
||||
# message: "Integer level"
|
||||
# }
|
||||
#
|
||||
# @TEST_EDGE empty_message ->
|
||||
# {
|
||||
# task_id: "00000000-0000-0000-0000-000000000000",
|
||||
# timestamp: "2026-02-26T12:00:00Z",
|
||||
# level: "DEBUG",
|
||||
# source: "system",
|
||||
# message: ""
|
||||
# }
|
||||
#
|
||||
# @TEST_INVARIANT exact_one_task_association -> verifies: [basic_info_log, missing_required_field]
|
||||
class TaskLogRecord(Base):
|
||||
__tablename__ = "task_logs"
|
||||
|
||||
|
||||
16
backend/src/plugins/llm_analysis/_constants.py
Normal file
16
backend/src/plugins/llm_analysis/_constants.py
Normal file
@@ -0,0 +1,16 @@
|
||||
# #region Plugin.Service.Constants [C:1] [TYPE Module] [SEMANTICS llm,screenshot,timeouts]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Named timeout constants for LLM analysis and screenshot services.
|
||||
# @LAYER Plugin
|
||||
# @RATIONALE Extracted all hardcoded timeouts into named module-level constants. Zero remaining numeric timeout literals.
|
||||
# @REJECTED Keeping inline numeric timeout literals was rejected — they create a maintenance hazard where any timeout adjustment requires grep-and-fix across 1700+ lines; named constants centralize tuning and make timeout configuration auditable.
|
||||
|
||||
# Timeout constants (milliseconds unless noted)
|
||||
PLAYWRIGHT_NAVIGATION_TIMEOUT_MS = 30000
|
||||
PLAYWRIGHT_WAIT_TIMEOUT_MS = 10000
|
||||
PLAYWRIGHT_SHORT_TIMEOUT_MS = 5000
|
||||
HTTP_REQUEST_TIMEOUT_MS = 60000
|
||||
SCREENSHOT_SERVICE_TIMEOUT_MS = 120000
|
||||
LLM_HTTP_TIMEOUT_S = 120 # seconds (httpx client timeout)
|
||||
DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
|
||||
# #endregion Plugin.Service.Constants
|
||||
215
backend/src/plugins/llm_analysis/_dataset_health.py
Normal file
215
backend/src/plugins/llm_analysis/_dataset_health.py
Normal file
@@ -0,0 +1,215 @@
|
||||
# #region Plugin.Service.DatasetHealthModule [C:3] [TYPE Module] [SEMANTICS llm,dataset,health,superset]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Dataset accessibility and chart-data health checks via Superset API.
|
||||
# @LAYER Plugin
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
# #region Plugin.Service.DatasetHealthChecker [C:3] [TYPE Class]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Checks dataset accessibility and KXD connectivity via Superset API.
|
||||
# @LAYER Service
|
||||
# @RELATION CALLS -> [Core.Init.SupersetClient]
|
||||
# @INVARIANT Every unique dataset referenced by dashboard charts is checked.
|
||||
# @RATIONALE Without dataset health checking, silent KXD errors (connection refused, timeout)
|
||||
# are invisible to the LLM validation. Screenshot captures visual state but doesn't
|
||||
# verify that data actually arrived (vs. cache).
|
||||
class DatasetHealthChecker:
|
||||
# #region DatasetHealthChecker.__init__ [C:2] [TYPE Function]
|
||||
# @BRIEF Initialize with a SupersetClient-compatible instance.
|
||||
# @PRE client is a SupersetClient (sync, wrapped via asyncio.to_thread) or AsyncSupersetClient.
|
||||
# @POST self.client is ready for health checks.
|
||||
def __init__(self, client: Any):
|
||||
self.client = client
|
||||
|
||||
# #endregion DatasetHealthChecker.__init__
|
||||
|
||||
# #region DatasetHealthChecker._call_sync [C:2] [TYPE Function]
|
||||
# @BRIEF Wrap a sync client method call in asyncio.to_thread for async compat.
|
||||
# @PRE method is a callable on self.client.
|
||||
# @POST Returns the result of method(*args, **kwargs) executed in a thread.
|
||||
@staticmethod
|
||||
async def _call_sync(method, *args: Any, **kwargs: Any) -> Any:
|
||||
"""Call a potentially sync method in a thread, or await if already async."""
|
||||
if asyncio.iscoroutinefunction(method):
|
||||
return await method(*args, **kwargs)
|
||||
return await asyncio.to_thread(method, *args, **kwargs)
|
||||
|
||||
# #endregion DatasetHealthChecker._call_sync
|
||||
|
||||
# #region DatasetHealthChecker.check_dataset_health [C:3] [TYPE Function]
|
||||
# @BRIEF Fetch dataset metadata and verify level 1-2 accessibility.
|
||||
# @PRE dataset_id is a valid Superset dataset ID.
|
||||
# @POST Returns dict with level 1-2 health fields.
|
||||
# @SIDE_EFFECT Calls GET /api/v1/dataset/{id} via client.get_dataset
|
||||
async def check_dataset_health(self, dataset_id: int) -> dict:
|
||||
"""
|
||||
Check a single dataset's accessibility (levels 1-2 per FR-044).
|
||||
|
||||
Level 1: metadata_accessible — HTTP 200 from GET /api/v1/dataset/{id}
|
||||
Level 2: datasource_resolvable — database info available
|
||||
|
||||
Returns dict with:
|
||||
dataset_id, dataset_name, database_name, backend, kind,
|
||||
metadata_accessible (bool), error (str|None)
|
||||
"""
|
||||
try:
|
||||
dataset = await self._call_sync(self.client.get_dataset, dataset_id)
|
||||
# The response from Superset may have a 'result' wrapper or be flat
|
||||
result_data = dataset.get("result", dataset) if isinstance(dataset, dict) else {}
|
||||
# Extract database info
|
||||
database = result_data.get("database", {}) or {}
|
||||
result = {
|
||||
"dataset_id": dataset_id,
|
||||
"dataset_name": result_data.get("table_name", f"dataset_{dataset_id}"),
|
||||
"database_name": database.get("database_name", "unknown"),
|
||||
"backend": database.get("backend", "unknown"),
|
||||
"kind": result_data.get("kind", "physical"),
|
||||
"metadata_accessible": True,
|
||||
"error": None,
|
||||
}
|
||||
return result
|
||||
except Exception as e:
|
||||
return {
|
||||
"dataset_id": dataset_id,
|
||||
"dataset_name": f"dataset_{dataset_id}",
|
||||
"database_name": "unknown",
|
||||
"backend": "unknown",
|
||||
"kind": "unknown",
|
||||
"metadata_accessible": False,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
# #endregion DatasetHealthChecker.check_dataset_health
|
||||
|
||||
# #region DatasetHealthChecker.check_chart_data [C:3] [TYPE Function]
|
||||
# @BRIEF Execute chart data query (level 3-4 per FR-044).
|
||||
# @PRE chart_id is valid, form_data is constructed from chart params.
|
||||
# @POST Returns dict with execution result.
|
||||
# @SIDE_EFFECT Calls POST /api/v1/chart/data via client.network.request
|
||||
async def check_chart_data(self, chart_id: int, form_data: dict) -> dict:
|
||||
"""
|
||||
Execute a chart query to verify data returns.
|
||||
|
||||
Level 3: query_executable — POST /api/v1/chart/data succeeds
|
||||
Level 4: data_returned — row_count > 0 or no error
|
||||
|
||||
Returns dict with:
|
||||
chart_id, executed (bool), duration_ms (int|None),
|
||||
row_count (int|None), error (str|None)
|
||||
"""
|
||||
import time
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
# Use the client's network layer for the chart data POST.
|
||||
# For sync SupersetClient: network.request(...) is synchronous.
|
||||
# We wrap it via asyncio.to_thread if it's a sync method.
|
||||
payload = json.dumps(form_data)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
network_request = self.client.network.request
|
||||
result = await self._call_sync(
|
||||
network_request,
|
||||
"POST",
|
||||
"/chart/data",
|
||||
data=payload,
|
||||
headers=headers,
|
||||
)
|
||||
duration_ms = int((time.time() - start) * 1000)
|
||||
|
||||
# Normalize response — may have 'result' wrapper
|
||||
rows = []
|
||||
if isinstance(result, dict):
|
||||
rows = result.get("result", []) or []
|
||||
elif isinstance(result, list):
|
||||
rows = result
|
||||
|
||||
return {
|
||||
"chart_id": chart_id,
|
||||
"executed": True,
|
||||
"duration_ms": duration_ms,
|
||||
"row_count": len(rows),
|
||||
"error": None,
|
||||
}
|
||||
except Exception as e:
|
||||
duration_ms = int((time.time() - start) * 1000)
|
||||
return {
|
||||
"chart_id": chart_id,
|
||||
"executed": False,
|
||||
"duration_ms": duration_ms,
|
||||
"row_count": None,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
# #endregion DatasetHealthChecker.check_chart_data
|
||||
|
||||
# #region DatasetHealthChecker.check_dashboard_datasets [C:3] [TYPE Function]
|
||||
# @BRIEF For every unique dataset in dashboard charts, check health.
|
||||
# @PRE chart_list has chart dicts with slice_id and datasource_id.
|
||||
# @POST Returns dict with datasets and optional chart_data lists.
|
||||
async def check_dashboard_datasets(
|
||||
self,
|
||||
chart_list: list[dict],
|
||||
execute_chart_data: bool = False,
|
||||
) -> dict:
|
||||
"""
|
||||
Check all unique datasets referenced by dashboard charts.
|
||||
|
||||
Args:
|
||||
chart_list: list of chart dicts with at least
|
||||
{'slice_id', 'datasource_id', 'viz_type', 'params'}
|
||||
execute_chart_data: if True, also execute chart queries (level 3-4)
|
||||
|
||||
Returns:
|
||||
{datasets: [...], chart_data: [...]}
|
||||
"""
|
||||
# Collect unique datasource_ids
|
||||
unique_ds_ids: set[int] = set()
|
||||
for chart in chart_list:
|
||||
ds_id = chart.get("datasource_id")
|
||||
if ds_id is not None:
|
||||
unique_ds_ids.add(int(ds_id))
|
||||
|
||||
# Check each dataset
|
||||
dataset_results: list[dict] = []
|
||||
for ds_id in sorted(unique_ds_ids):
|
||||
result = await self.check_dataset_health(ds_id)
|
||||
# Map affected charts
|
||||
affected_charts = [{"chart_id": c.get("slice_id"), "chart_name": c.get("slice_name", f"chart_{c.get('slice_id')}")} for c in chart_list if c.get("datasource_id") == ds_id]
|
||||
result["affected_charts"] = affected_charts
|
||||
dataset_results.append(result)
|
||||
|
||||
# Optionally execute chart data
|
||||
chart_data_results: list[dict] = []
|
||||
if execute_chart_data:
|
||||
for chart in chart_list:
|
||||
chart_id = chart.get("slice_id")
|
||||
params = chart.get("params", {})
|
||||
if isinstance(params, str):
|
||||
params = json.loads(params)
|
||||
form_data = {
|
||||
"slice_id": chart_id,
|
||||
"viz_type": chart.get("viz_type", "table"),
|
||||
"datasource_id": chart.get("datasource_id"),
|
||||
"datasource_type": chart.get("datasource_type", "table"),
|
||||
"granularity_sqla": params.get("granularity_sqla"),
|
||||
"time_range": params.get("time_range", "Last 30 days"),
|
||||
"metrics": params.get("metrics", []),
|
||||
"groupby": params.get("groupby", []),
|
||||
"adhoc_filters": params.get("adhoc_filters", []),
|
||||
}
|
||||
result = await self.check_chart_data(chart_id, form_data)
|
||||
chart_data_results.append(result)
|
||||
|
||||
return {
|
||||
"datasets": dataset_results,
|
||||
"chart_data": chart_data_results,
|
||||
}
|
||||
|
||||
# #endregion DatasetHealthChecker.check_dashboard_datasets
|
||||
|
||||
|
||||
# #endregion Plugin.Service.DatasetHealthChecker
|
||||
# #endregion Plugin.Service.DatasetHealthModule
|
||||
351
backend/src/plugins/llm_analysis/_llm_client_analysis.py
Normal file
351
backend/src/plugins/llm_analysis/_llm_client_analysis.py
Normal file
@@ -0,0 +1,351 @@
|
||||
# #region Plugin.Service.LLMClient.AnalysisModule [C:4] [TYPE Module] [SEMANTICS llm,analysis,multimodal,batch]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Dashboard analysis paths over the LLM transport: multimodal images and text batch (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt
|
||||
from .exceptions import ProviderAuthenticationFailure, ProviderConfigurationFailure
|
||||
from ._llm_client_core import _should_retry
|
||||
|
||||
|
||||
# #region Plugin.Service.LLMClientAnalysisMixin [C:4] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: image optimization, chunk merging, multimodal and text-batch dashboard analysis.
|
||||
class LLMClientAnalysisMixin:
|
||||
# #region LLMClient.analyze_dashboard [C:4] [TYPE Function]
|
||||
# @BRIEF Sends dashboard data (screenshot + logs) to LLM for health analysis.
|
||||
# @PRE screenshot_path exists, logs is a list of strings.
|
||||
# @POST Returns a structured analysis dictionary (status, summary, issues).
|
||||
# @SIDE_EFFECT Reads screenshot file and calls external LLM API.
|
||||
# @RATIONALE Delegates to analyze_dashboard_multimodal for single-screenshot
|
||||
# backward compatibility. Keeps the same contract for v1 consumers.
|
||||
async def analyze_dashboard(
|
||||
self,
|
||||
screenshot_path: str,
|
||||
logs: list[str],
|
||||
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
|
||||
) -> dict[str, Any]:
|
||||
# Delegate to multimodal variant for backward compat with v1 consumers.
|
||||
return await self.analyze_dashboard_multimodal(
|
||||
screenshot_paths=[screenshot_path],
|
||||
logs=logs,
|
||||
prompt_template=prompt_template,
|
||||
)
|
||||
|
||||
# #endregion LLMClient.analyze_dashboard
|
||||
|
||||
# #region LLMClient._reduce_image_quality [TYPE Function] [C:2]
|
||||
# @BRIEF Open, resize, and compress a screenshot image for LLM consumption.
|
||||
# @PRE path points to an existing image file.
|
||||
# @POST Returns (base64_str, byte_size) tuple.
|
||||
@staticmethod
|
||||
def _reduce_image_quality(
|
||||
path: str,
|
||||
max_width: int = 1024,
|
||||
image_quality: int = 60,
|
||||
) -> tuple[str, int]:
|
||||
"""
|
||||
Open, resize, compress, and base64-encode an image.
|
||||
|
||||
Returns (base64_str, byte_size).
|
||||
"""
|
||||
with Image.open(path) as img:
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
if img.width > max_width or img.height > 2048:
|
||||
scale = min(max_width / img.width, 2048 / img.height)
|
||||
if scale < 1.0:
|
||||
new_width = int(img.width * scale)
|
||||
new_height = int(img.height * scale)
|
||||
img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
|
||||
buffer = io.BytesIO()
|
||||
img.save(buffer, format="JPEG", quality=image_quality, optimize=True)
|
||||
raw = buffer.getvalue()
|
||||
return base64.b64encode(raw).decode("utf-8"), len(raw)
|
||||
|
||||
# #endregion LLMClient._reduce_image_quality
|
||||
|
||||
# #region LLMClient._estimate_payload_size [TYPE Function] [C:2]
|
||||
# @BRIEF Estimate LLM payload size in tokens before sending.
|
||||
# @POST Returns {estimated_tokens, exceeds_limit, pct_of_limit} dict.
|
||||
# @RATIONALE FR-056: if >80% of model context window, trigger quality reduction.
|
||||
@staticmethod
|
||||
def _estimate_payload_size(
|
||||
image_paths: list[str],
|
||||
text_length: int,
|
||||
model_context: int = 128000,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Estimate token usage for multimodal payload.
|
||||
|
||||
Rough heuristic: 1 image token ~ 258 tokens (GPT-4o), text ~4 chars/token.
|
||||
Returns {estimated_tokens, exceeds_limit, pct_of_limit}
|
||||
"""
|
||||
image_tokens = len(image_paths) * 258 * 5 # rough upper bound for compressed images
|
||||
text_tokens = text_length // 4
|
||||
total_tokens = image_tokens + text_tokens
|
||||
exceeds_limit = total_tokens > (model_context * 0.8)
|
||||
return {
|
||||
"estimated_tokens": total_tokens,
|
||||
"exceeds_limit": exceeds_limit,
|
||||
"pct_of_limit": round(total_tokens / model_context * 100, 1),
|
||||
}
|
||||
|
||||
# #endregion LLMClient._estimate_payload_size
|
||||
|
||||
# #region LLMClient._deduplicate_issues [TYPE Function] [C:2]
|
||||
# @BRIEF Deduplicate issues by (severity, message, location) while preserving order.
|
||||
def _deduplicate_issues(self, issues: list[dict]) -> list[dict]:
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
result: list[dict] = []
|
||||
for issue in issues:
|
||||
key = (issue.get("severity", ""), issue.get("message", ""), issue.get("location", "") or "")
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
result.append(issue)
|
||||
return result
|
||||
|
||||
# #endregion LLMClient._deduplicate_issues
|
||||
|
||||
# #region LLMClient._optimize_images [TYPE Function] [C:2]
|
||||
# @BRIEF Convert screenshot paths to base64 at given quality, with fallback to raw read.
|
||||
def _optimize_images(self, paths: list[str], max_width: int, quality: int) -> list[str]:
|
||||
encoded: list[str] = []
|
||||
for path in paths:
|
||||
try:
|
||||
b64, _ = self._reduce_image_quality(path, max_width, quality)
|
||||
encoded.append(b64)
|
||||
except Exception as e:
|
||||
logger.explore("Image optimization failed, falling back to raw read", payload={"path": path}, error=str(e))
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
b64 = base64.b64encode(raw).decode("utf-8")
|
||||
encoded.append(b64)
|
||||
return encoded
|
||||
|
||||
# #endregion LLMClient._optimize_images
|
||||
|
||||
# #region LLMClient._merge_chunk_results [TYPE Function] [C:2]
|
||||
# @BRIEF Merge multiple chunk analyses into one. Takes the worst status,
|
||||
# concatenates summaries, and deduplicates issues.
|
||||
# @PRE chunks is a non-empty list of {status, summary, issues} dicts.
|
||||
# @POST Returns a single merged dict with chunk_count.
|
||||
def _merge_chunk_results(self, chunks: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
STATUS_ORDER = {"FAIL": 0, "WARN": 1, "PASS": 2, "UNKNOWN": 3}
|
||||
worst_status = "UNKNOWN"
|
||||
all_summaries: list[str] = []
|
||||
all_issues: list[dict] = []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
s = chunk.get("status", "UNKNOWN")
|
||||
if STATUS_ORDER.get(s, 3) < STATUS_ORDER.get(worst_status, 3):
|
||||
worst_status = s
|
||||
all_summaries.append(f"[Chunk {i + 1}/{len(chunks)}] {chunk.get('summary', 'No summary')}")
|
||||
all_issues.extend(chunk.get("issues", []))
|
||||
|
||||
merged: dict[str, Any] = {
|
||||
"status": worst_status,
|
||||
"summary": " | ".join(all_summaries),
|
||||
"issues": self._deduplicate_issues(all_issues),
|
||||
"chunk_count": len(chunks),
|
||||
}
|
||||
return merged
|
||||
|
||||
# #endregion LLMClient._merge_chunk_results
|
||||
|
||||
# #region LLMClient._call_llm_for_images [TYPE Function] [C:2]
|
||||
# @BRIEF Send a single chunk of images to the LLM and return parsed result.
|
||||
async def _call_llm_for_images(self, encoded_images: list[str], prompt: str) -> dict[str, Any]:
|
||||
content: list[dict] = [{"type": "text", "text": prompt}]
|
||||
for b64_img in encoded_images:
|
||||
content.append(
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/jpeg;base64,{b64_img}"},
|
||||
}
|
||||
)
|
||||
messages = [{"role": "user", "content": content}]
|
||||
return await self.get_json_completion(messages)
|
||||
|
||||
# #endregion LLMClient._call_llm_for_images
|
||||
|
||||
# #region LLMClient.analyze_dashboard_multimodal [TYPE Function] [C:3]
|
||||
# @BRIEF Path A: send screenshots + logs to multimodal LLM, with chunking support.
|
||||
# @PRE screenshot_paths is a non-empty list of paths.
|
||||
# tab_labels, if provided, must have the same length as screenshot_paths.
|
||||
# @POST Returns dict {status, summary, issues} with optional chunk_count.
|
||||
# @SIDE_EFFECT Compresses images, calls external LLM API (possibly multiple times for chunks).
|
||||
# @RATIONALE Screenshots are split into chunks of max_images to respect provider image limits.
|
||||
# Quality reduction is skipped when chunking — each chunk fits the limit by definition.
|
||||
# Results are merged via _merge_chunk_results.
|
||||
async def analyze_dashboard_multimodal(
|
||||
self,
|
||||
screenshot_paths: list[str],
|
||||
logs: list[str],
|
||||
prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
|
||||
max_width: int = 1024,
|
||||
image_quality: int = 60,
|
||||
max_images: int | None = None,
|
||||
tab_labels: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
with belief_scope("analyze_dashboard_multimodal"):
|
||||
if not screenshot_paths:
|
||||
raise ValueError("screenshot_paths must be a non-empty list")
|
||||
|
||||
# 1. Optimize all images at requested quality
|
||||
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality)
|
||||
|
||||
log_text = "\n".join(logs)
|
||||
tab_list_text = "\n".join(f" Screenshot {i}: {label}" for i, label in enumerate(tab_labels or [])) or "Screenshots are in order."
|
||||
prompt = render_prompt(
|
||||
prompt_template,
|
||||
{
|
||||
"logs": log_text,
|
||||
"tab_list": tab_list_text,
|
||||
"total_chunks": str(len(encoded_images)),
|
||||
},
|
||||
)
|
||||
|
||||
# 2. Determine chunking
|
||||
# Default to 8 images per chunk as a safe fallback when max_images is 0 or None
|
||||
# (0 means probe failed — e.g. Kilo gateway doesn't support OpenAI image format)
|
||||
DEFAULT_CHUNK_SIZE = 8
|
||||
effective_max = max_images if (max_images is not None and max_images > 0) else DEFAULT_CHUNK_SIZE
|
||||
n_total = len(encoded_images)
|
||||
chunk_size = effective_max if effective_max < n_total else n_total
|
||||
is_chunking = chunk_size < n_total
|
||||
|
||||
if is_chunking:
|
||||
logger.reason(
|
||||
"Chunking images for multimodal analysis",
|
||||
payload={"total_images": n_total, "chunk_count": (n_total + chunk_size - 1) // chunk_size, "chunk_size": chunk_size},
|
||||
)
|
||||
# Skip quality reduction: each chunk has ≤ max_images images,
|
||||
# well within the context window at normal quality.
|
||||
else:
|
||||
# Single batch: estimate payload and reduce quality if needed
|
||||
estimate = self._estimate_payload_size(screenshot_paths, len(prompt) + len(log_text))
|
||||
if estimate["exceeds_limit"] and image_quality > 30:
|
||||
logger.reason(
|
||||
"Reducing image quality to fit context window",
|
||||
payload={"pct_of_limit": estimate["pct_of_limit"], "new_quality": 30},
|
||||
)
|
||||
encoded_images = self._optimize_images(screenshot_paths, max_width, image_quality=30)
|
||||
|
||||
# 3. Split into chunks
|
||||
chunks: list[list[str]] = [encoded_images[i : i + chunk_size] for i in range(0, n_total, chunk_size)]
|
||||
|
||||
# 4. Call LLM — parallel for multiple chunks, single for one
|
||||
try:
|
||||
if len(chunks) == 1:
|
||||
result = await self._call_llm_for_images(chunks[0], prompt)
|
||||
else:
|
||||
tasks = [self._call_llm_for_images(chunk, prompt) for chunk in chunks]
|
||||
chunk_results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
valid_results: list[dict] = []
|
||||
for i, cr in enumerate(chunk_results):
|
||||
if isinstance(cr, Exception):
|
||||
# Re-raise permanent provider errors (auth, config) — they apply to all chunks
|
||||
normalized = self._normalize_provider_error(cr)
|
||||
if not normalized.retryable:
|
||||
logger.explore(
|
||||
"Multimodal analysis chunk failed with permanent provider error, aborting",
|
||||
payload={"chunk_index": i + 1, "error_type": type(normalized).__name__},
|
||||
error=str(cr),
|
||||
)
|
||||
raise normalized from cr
|
||||
# Transient chunk failures can be merged as UNKNOWN
|
||||
logger.explore("Multimodal analysis chunk failed (transient)", payload={"chunk_index": i + 1, "total_chunks": len(chunks)}, error=str(cr))
|
||||
valid_results.append(
|
||||
{
|
||||
"status": "UNKNOWN",
|
||||
"summary": f"Chunk {i + 1} failed: {cr!s}",
|
||||
"issues": [],
|
||||
}
|
||||
)
|
||||
else:
|
||||
valid_results.append(cr)
|
||||
|
||||
result = self._merge_chunk_results(valid_results)
|
||||
except (ProviderAuthenticationFailure, ProviderConfigurationFailure):
|
||||
# Permanent provider errors propagate to TaskManager — no UNKNOWN swallow
|
||||
raise
|
||||
except Exception as e:
|
||||
# Transient errors and unexpected failures → UNKNOWN (existing behavior)
|
||||
normalized = self._normalize_provider_error(e)
|
||||
if not normalized.retryable:
|
||||
raise normalized from e
|
||||
logger.explore("Failed to get multimodal analysis from LLM", payload={}, error=str(e))
|
||||
return {
|
||||
"status": "UNKNOWN",
|
||||
"summary": f"Failed to get response from LLM: {e!s}",
|
||||
"issues": [{"severity": "UNKNOWN", "message": "LLM provider returned empty or invalid response"}],
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
# #endregion LLMClient.analyze_dashboard_multimodal
|
||||
|
||||
# #region LLMClient.analyze_dashboard_text_batch [TYPE Function] [C:3]
|
||||
# @BRIEF Path B batch: multiple dashboards in a single text-only LLM call.
|
||||
# @PRE payloads is a non-empty list of {dashboard_id, topology, dataset_health, log_text} dicts.
|
||||
# @POST Returns dict {dashboards: [{dashboard_id, status, summary, issues}]}.
|
||||
# Missing/parse-error dashboard_id -> marked UNKNOWN individually.
|
||||
# @RATIONALE Text-only batch avoids image token costs. Uses per-dashboard sections
|
||||
# with explicit JSON response contract. Fallback ensures partial results survive
|
||||
# single-dashboard parse failures.
|
||||
@retry(
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential(multiplier=2, min=5, max=60),
|
||||
retry=retry_if_exception(_should_retry),
|
||||
reraise=True,
|
||||
)
|
||||
async def analyze_dashboard_text_batch(
|
||||
self,
|
||||
payloads: list[dict],
|
||||
prompt_template: str,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Batch analyze multiple dashboards in one LLM call.
|
||||
|
||||
payloads: list of dicts with keys:
|
||||
- dashboard_id (str)
|
||||
- topology (str) — dashboard structure
|
||||
- dataset_health (str) — health results
|
||||
- log_text (str) — execution logs
|
||||
|
||||
Returns dict like {dashboards: [{dashboard_id, status, summary, issues}]}
|
||||
"""
|
||||
if not payloads:
|
||||
return {"dashboards": []}
|
||||
|
||||
# 1. Build per-dashboard sections
|
||||
sections = []
|
||||
for i, p in enumerate(payloads):
|
||||
did = p.get("dashboard_id", "UNKNOWN")
|
||||
top = p.get("topology", "")
|
||||
first_line = top.split("\n")[0] if top else "(no topology)"
|
||||
section = f'─── Dashboard {i + 1}: "{first_line}" (id: {did}) ───\n{top}\n\nDataset health:\n{p.get("dataset_health", "")}\n\nLogs:\n{p.get("log_text", "")}'
|
||||
sections.append(section)
|
||||
|
||||
full_prompt = prompt_template.replace("{total_dashboards}", str(len(payloads)))
|
||||
full_prompt += '\n\nRespond with a JSON object containing EACH dashboard\'s results:\n{"dashboards": [{"dashboard_id": "...", "status": "...", "summary": "...", "issues": [...]}]}\n\n'
|
||||
full_prompt += "\n---\n".join(sections)
|
||||
|
||||
messages = [{"role": "user", "content": full_prompt}]
|
||||
return await self.get_json_completion(messages)
|
||||
|
||||
# #endregion LLMClient.analyze_dashboard_text_batch
|
||||
# #endregion Plugin.Service.LLMClientAnalysisMixin
|
||||
# #endregion Plugin.Service.LLMClient.AnalysisModule
|
||||
391
backend/src/plugins/llm_analysis/_llm_client_core.py
Normal file
391
backend/src/plugins/llm_analysis/_llm_client_core.py
Normal file
@@ -0,0 +1,391 @@
|
||||
# #region Plugin.Service.LLMClient.CoreModule [C:4] [TYPE Module] [SEMANTICS llm,client,provider,openai,retry]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Core LLM provider transport: client init, SSL, error normalization, JSON completion (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI, AuthenticationError as OpenAIAuthenticationError, RateLimitError
|
||||
from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
|
||||
|
||||
from ...core.logger import belief_scope, logger
|
||||
from .exceptions import (
|
||||
ProviderAuthenticationFailure,
|
||||
ProviderFailure,
|
||||
ProviderRateLimitFailure,
|
||||
ProviderTransportFailure,
|
||||
)
|
||||
from .models import LLMProviderType
|
||||
from typing import Any
|
||||
|
||||
from ._constants import LLM_HTTP_TIMEOUT_S
|
||||
|
||||
|
||||
# #region Plugin.Service.ShouldRetry [C:2] [TYPE Function] [SEMANTICS llm,retry,policy]
|
||||
# @BRIEF Custom retry predicate for Tenacity — excludes non-recoverable errors from LLM retry loops.
|
||||
# @PRE exception is an Exception raised during LLM API call.
|
||||
# @POST Returns True if the error is retryable (transport, rate limit), False for permanent errors (auth, config).
|
||||
# @RELATION CALLED_BY -> [Plugin.Service.LLMClient.get_json_completion]
|
||||
# @RELATION CALLED_BY -> [Plugin.Service.LLMClient.analyze_dashboard_text_batch]
|
||||
# @RATIONALE Extracted to module level because it is referenced by @retry decorators in two separate methods
|
||||
# (get_json_completion and analyze_dashboard_text_batch). A nested function would be fragile —
|
||||
# moving either method would break the other's retry configuration.
|
||||
def _should_retry(exception: Exception) -> bool:
|
||||
"""Custom retry predicate that excludes non-recoverable errors."""
|
||||
# Typed provider failures use their native retryability
|
||||
if isinstance(exception, ProviderFailure):
|
||||
return exception.retryable
|
||||
# Don't retry on OpenAIAuthenticationError
|
||||
if isinstance(exception, OpenAIAuthenticationError):
|
||||
return False
|
||||
# Don't retry on null content / model errors — retrying won't help
|
||||
msg = str(exception).lower()
|
||||
if "null content" in msg or "none" in msg:
|
||||
return False
|
||||
# Retry on rate limit errors
|
||||
if isinstance(exception, RateLimitError):
|
||||
return True
|
||||
# For other exceptions, limit retries
|
||||
return True
|
||||
# #endregion Plugin.Service.ShouldRetry
|
||||
|
||||
|
||||
# #region Plugin.Service.LLMClientCoreMixin [C:4] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: provider client construction, SSL policy, typed error normalization, JSON-mode completion.
|
||||
class LLMClientCoreMixin:
|
||||
# #region LLMClient.__init__ [C:2] [TYPE Function]
|
||||
# @BRIEF Initializes the LLMClient with provider settings.
|
||||
# @PRE api_key, base_url, and default_model are non-empty strings.
|
||||
def __init__(self, provider_type: LLMProviderType, api_key: str, base_url: str, default_model: str):
|
||||
self.provider_type = provider_type
|
||||
normalized_key = (api_key or "").strip()
|
||||
if normalized_key.lower().startswith("bearer "):
|
||||
normalized_key = normalized_key[7:].strip()
|
||||
self.api_key = normalized_key
|
||||
self.base_url = base_url
|
||||
self.default_model = default_model
|
||||
|
||||
# DEBUG: Log initialization parameters (without exposing full API key)
|
||||
logger.reason(
|
||||
"Initializing LLM client",
|
||||
payload={
|
||||
"provider_type": str(provider_type),
|
||||
"base_url": base_url,
|
||||
"default_model": default_model,
|
||||
"api_key_present": bool(self.api_key),
|
||||
"api_key_length": len(self.api_key) if self.api_key else 0,
|
||||
},
|
||||
)
|
||||
|
||||
# Some OpenAI-compatible gateways are strict about auth header naming.
|
||||
default_headers = {"Authorization": f"Bearer {self.api_key}"}
|
||||
if self.provider_type == LLMProviderType.OPENROUTER:
|
||||
default_headers["HTTP-Referer"] = os.getenv("OPENROUTER_SITE_URL", "").strip() or os.getenv("APP_BASE_URL", "").strip()
|
||||
default_headers["X-Title"] = os.getenv("OPENROUTER_APP_NAME", "").strip() or ""
|
||||
if self.provider_type == LLMProviderType.KILO:
|
||||
default_headers["Authentication"] = f"Bearer {self.api_key}"
|
||||
default_headers["X-API-Key"] = self.api_key
|
||||
# LiteLLM proxy uses standard OpenAI-compatible Bearer auth — no special headers needed.
|
||||
# It routes to upstream providers transparently, and the default Authorization header
|
||||
# is sufficient. No additional headers like HTTP-Referer or X-API-Key are required.
|
||||
|
||||
ssl_verify = self._get_ssl_verify()
|
||||
from ...core.ssl import describe_context
|
||||
|
||||
ssl_desc = describe_context(ssl_verify)
|
||||
logger.reason("LLM client SSL verification configured", payload={"ssl_verify": ssl_desc})
|
||||
|
||||
http_client = httpx.AsyncClient(
|
||||
headers=default_headers,
|
||||
timeout=LLM_HTTP_TIMEOUT_S,
|
||||
verify=ssl_verify,
|
||||
)
|
||||
self.client = AsyncOpenAI(
|
||||
api_key=self.api_key,
|
||||
base_url=base_url,
|
||||
default_headers=default_headers,
|
||||
http_client=http_client,
|
||||
)
|
||||
|
||||
# #endregion LLMClient.__init__
|
||||
|
||||
# #region LLMClient._get_ssl_verify [C:3] [TYPE Function]
|
||||
# @BRIEF Resolve SSL verification flag from environment.
|
||||
# @POST Returns SSLContext with system CA dir (never False — centralized SSL).
|
||||
# @RATIONALE Используем capath=/etc/ssl/certs/ вместо cafile, потому что
|
||||
# OpenSSL 3.x не использует intermediate CA сертификаты из cafile для
|
||||
# построения цепочки (verify code 20). capath с хеш-симлинками работает
|
||||
# корректно (verify code 0). Оба пути — cafile и capath — указывают на
|
||||
# один и тот же набор сертификатов, но capath правильно обрабатывает
|
||||
# цепочку Root → Policy → Issuing.
|
||||
# @REJECTED verify=<string> отвергнут — httpx 0.28.x депрекейтит строковый
|
||||
# путь в verify=, требует SSLContext.
|
||||
# @REJECTED cafile отвергнут — OpenSSL 3.x не использует intermediate CA
|
||||
# из единого bundle-файла. Только capath с хеш-симлинками даёт code 0.
|
||||
@staticmethod
|
||||
def _get_ssl_verify() -> ssl.SSLContext | bool:
|
||||
from ...core.ssl import system_ssl_context
|
||||
|
||||
return system_ssl_context()
|
||||
|
||||
# #endregion LLMClient._get_ssl_verify
|
||||
|
||||
# #region LLMClient._format_connection_error [C:2] [TYPE Function]
|
||||
# @BRIEF Format exception chain for diagnostics, extracting httpx cause details.
|
||||
# @POST Returns a human-readable string with the full error chain.
|
||||
@staticmethod
|
||||
def _format_connection_error(exc: Exception) -> str:
|
||||
parts = [f"{type(exc).__name__}: {exc!s}"]
|
||||
cause = exc.__cause__ or exc.__context__
|
||||
while cause:
|
||||
parts.append(f" └─ {type(cause).__name__}: {cause!s}")
|
||||
cause = cause.__cause__ or cause.__context__
|
||||
return "\n".join(parts)
|
||||
|
||||
# #endregion LLMClient._format_connection_error
|
||||
|
||||
# #region LLMClient._supports_json_response_format [C:3] [TYPE Function]
|
||||
# @BRIEF Detect whether provider/model is likely compatible with response_format=json_object.
|
||||
# @PRE Client initialized with base_url and default_model.
|
||||
# @POST Returns False for known-incompatible combinations to avoid avoidable 400 errors.
|
||||
def _supports_json_response_format(self) -> bool:
|
||||
model = (self.default_model or "").lower()
|
||||
|
||||
# Free-tier models from ANY gateway often reject json_object mode
|
||||
# (Nvidia NeMo free via OpenRouter or Kilo, stepfun free, etc.)
|
||||
if ":free" in model:
|
||||
return False
|
||||
# stepfun models (even non-free) don't support json_object mode
|
||||
if "stepfun/" in model or model.startswith("step-"):
|
||||
return False
|
||||
return True
|
||||
|
||||
# #endregion LLMClient._supports_json_response_format
|
||||
|
||||
# #region LLMClient._normalize_provider_error [C:3] [TYPE Function]
|
||||
# @BRIEF Normalize OpenAI/HTTP exceptions into typed ProviderFailure hierarchy.
|
||||
# @PRE exc is a raw exception from the OpenAI SDK or httpx client.
|
||||
# @POST Returns a ProviderFailure subclass that expresses retryability.
|
||||
# @SIDE_EFFECT Logs the normalization decision.
|
||||
# @RATIONALE OpenAI SDK exceptions (OpenAIAuthenticationError, etc.) carry status codes and
|
||||
# messages that map to typed provider errors. httpx transport errors map to
|
||||
# ProviderTransportFailure. Generic exceptions with "401" patterns are caught
|
||||
# via string fallback for non-OpenAI gateways.
|
||||
@staticmethod
|
||||
def _normalize_provider_error(exc: Exception, provider_id: str | None = None) -> ProviderFailure:
|
||||
# Already a typed failure — pass through
|
||||
if isinstance(exc, ProviderFailure):
|
||||
return exc
|
||||
|
||||
# OpenAI SDK auth error
|
||||
if isinstance(exc, OpenAIAuthenticationError):
|
||||
return ProviderAuthenticationFailure(
|
||||
str(exc), provider_id=provider_id, status_code=401, original=exc,
|
||||
)
|
||||
|
||||
# OpenAI rate limit
|
||||
if isinstance(exc, RateLimitError):
|
||||
return ProviderRateLimitFailure(
|
||||
str(exc), provider_id=provider_id, status_code=429, original=exc,
|
||||
)
|
||||
|
||||
# httpx transport errors (connection, timeout, DNS)
|
||||
if isinstance(exc, httpx.TimeoutException):
|
||||
return ProviderTransportFailure(
|
||||
str(exc), provider_id=provider_id, original=exc,
|
||||
)
|
||||
if isinstance(exc, httpx.ConnectError):
|
||||
return ProviderTransportFailure(
|
||||
str(exc), provider_id=provider_id, original=exc,
|
||||
)
|
||||
|
||||
# HTTP status code via status_code attribute (OpenAI-like SDKs)
|
||||
status_code = getattr(exc, "status_code", None) or getattr(exc, "status", None)
|
||||
if status_code is not None:
|
||||
status_code = int(status_code) # type: ignore[arg-type]
|
||||
if status_code in (401, 403):
|
||||
return ProviderAuthenticationFailure(
|
||||
str(exc), provider_id=provider_id, status_code=status_code, original=exc,
|
||||
)
|
||||
if status_code == 429:
|
||||
return ProviderRateLimitFailure(
|
||||
str(exc), provider_id=provider_id, status_code=429, original=exc,
|
||||
)
|
||||
if status_code >= 500:
|
||||
return ProviderTransportFailure(
|
||||
str(exc), provider_id=provider_id, status_code=status_code, original=exc,
|
||||
)
|
||||
|
||||
# String fallback for non-OpenAI gateways that embed an auth status in generic errors.
|
||||
msg = str(exc).lower()
|
||||
if "401" in msg or "403" in msg or "authentication" in msg or "unauthorized" in msg:
|
||||
return ProviderAuthenticationFailure(
|
||||
str(exc),
|
||||
provider_id=provider_id,
|
||||
status_code=401 if "401" in msg else 403 if "403" in msg else None,
|
||||
original=exc,
|
||||
)
|
||||
if "429" in msg or "rate limit" in msg:
|
||||
return ProviderRateLimitFailure(
|
||||
str(exc), provider_id=provider_id, original=exc,
|
||||
)
|
||||
|
||||
# Default: assume transport-level failure (retryable)
|
||||
return ProviderTransportFailure(
|
||||
str(exc), provider_id=provider_id, original=exc,
|
||||
)
|
||||
# #endregion LLMClient._normalize_provider_error
|
||||
|
||||
# #region LLMClient.get_json_completion [C:4] [TYPE Function]
|
||||
# @BRIEF Helper to handle LLM calls with JSON mode and fallback parsing.
|
||||
# @PRE messages is a list of valid message dictionaries.
|
||||
# @POST Returns a parsed JSON dictionary.
|
||||
# @SIDE_EFFECT Calls external LLM API.
|
||||
@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, min=5, max=60), retry=retry_if_exception(_should_retry), reraise=True)
|
||||
async def get_json_completion(self, messages: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
with belief_scope("get_json_completion"):
|
||||
response = None
|
||||
try:
|
||||
use_json_mode = self._supports_json_response_format()
|
||||
try:
|
||||
logger.reason(
|
||||
"Attempting LLM call",
|
||||
payload={
|
||||
"model": self.default_model,
|
||||
"json_mode": "on" if use_json_mode else "off",
|
||||
"base_url": self.base_url,
|
||||
"message_count": len(messages),
|
||||
"api_key_present": bool(self.api_key and len(self.api_key) > 0),
|
||||
},
|
||||
)
|
||||
|
||||
if use_json_mode:
|
||||
response = await self.client.chat.completions.create(model=self.default_model, messages=messages, response_format={"type": "json_object"})
|
||||
else:
|
||||
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
|
||||
except Exception as e:
|
||||
if use_json_mode and ("JSON mode is not enabled" in str(e) or "json_object is not supported" in str(e).lower() or "response_format" in str(e).lower() or "400" in str(e)):
|
||||
logger.explore("JSON mode failed or not supported, falling back to plain text", payload={"model": self.default_model}, error=str(e))
|
||||
response = await self.client.chat.completions.create(model=self.default_model, messages=messages)
|
||||
else:
|
||||
raise e
|
||||
|
||||
logger.reflect("LLM API response received", payload={"response_summary": str(response)[:200]})
|
||||
except RateLimitError as e:
|
||||
logger.explore("Rate limit hit on LLM call, retrying with backoff", payload={}, error=str(e))
|
||||
|
||||
# Extract retry_delay from error metadata if available
|
||||
retry_delay = 5.0 # Default fallback
|
||||
try:
|
||||
# Based on logs, the raw response is in e.body or e.response.json()
|
||||
# The logs show 'metadata': {'raw': '...'} which suggests a proxy or specific client wrapper
|
||||
# Let's try to find the 'retryDelay' in the error message or response
|
||||
import re
|
||||
|
||||
# Try to find "retryDelay": "XXs" in the string representation of the error
|
||||
error_str = str(e)
|
||||
match = re.search(r'"retryDelay":\s*"(\d+)s"', error_str)
|
||||
if match:
|
||||
retry_delay = float(match.group(1))
|
||||
else:
|
||||
# Try to parse from response if it's a standard OpenAI-like error with body
|
||||
if hasattr(e, "body") and isinstance(e.body, dict):
|
||||
# Some providers put it in details
|
||||
details = e.body.get("error", {}).get("details", [])
|
||||
for detail in details:
|
||||
if detail.get("@type") == "type.googleapis.com/google.rpc.RetryInfo":
|
||||
delay_str = detail.get("retryDelay", "5s")
|
||||
retry_delay = float(delay_str.rstrip("s"))
|
||||
break
|
||||
except Exception as parse_e:
|
||||
logger.explore("Failed to parse retry delay from error response", payload={}, error=str(parse_e))
|
||||
|
||||
# Add a small safety margin (0.5s) as requested
|
||||
wait_time = retry_delay + 0.5
|
||||
logger.reason("Waiting before LLM retry", payload={"wait_time_seconds": wait_time})
|
||||
await asyncio.sleep(wait_time)
|
||||
raise
|
||||
except Exception as e:
|
||||
# Normalize into typed provider exception chain
|
||||
provider_exc = self._normalize_provider_error(e, provider_id=self.default_model)
|
||||
logger.explore(
|
||||
"LLM call failed, normalized as",
|
||||
payload={"type": type(provider_exc).__name__, "status_code": provider_exc.status_code},
|
||||
error=str(provider_exc),
|
||||
)
|
||||
raise provider_exc from e
|
||||
|
||||
if not response or not hasattr(response, "choices") or not response.choices:
|
||||
raise RuntimeError(f"Invalid LLM response: {response}")
|
||||
|
||||
content = response.choices[0].message.content
|
||||
logger.reflect("Raw LLM response content received for parsing", payload={"content_length": len(content) if content else 0})
|
||||
|
||||
# LLM returned null content — likely content filter or rate limit
|
||||
if content is None:
|
||||
raise RuntimeError("LLM returned null content (content filter or rate limit)")
|
||||
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
logger.explore("Failed to parse JSON directly, attempting to extract from code blocks", payload={}, error="JSONDecodeError on first parse attempt")
|
||||
if "```json" in content:
|
||||
json_str = content.split("```json")[1].split("```")[0].strip()
|
||||
return json.loads(json_str)
|
||||
elif "```" in content:
|
||||
json_str = content.split("```")[1].split("```")[0].strip()
|
||||
return json.loads(json_str)
|
||||
else:
|
||||
raise
|
||||
|
||||
# #endregion LLMClient.get_json_completion
|
||||
|
||||
# #region LLMClient.test_runtime_connection [C:3] [TYPE Function]
|
||||
# @BRIEF Validate provider credentials using the same chat completions transport as runtime analysis.
|
||||
# @PRE Client is initialized with provider credentials and default_model.
|
||||
# @POST Returns lightweight JSON payload when runtime auth/model path is valid.
|
||||
# @SIDE_EFFECT Calls external LLM API.
|
||||
async def test_runtime_connection(self) -> dict[str, Any]:
|
||||
with belief_scope("test_runtime_connection"):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": 'Return exactly this JSON object and nothing else: {"ok": true}',
|
||||
}
|
||||
]
|
||||
return await self.get_json_completion(messages)
|
||||
|
||||
# #endregion LLMClient.test_runtime_connection
|
||||
|
||||
# #region LLMClient.fetch_models [C:3] [TYPE Function]
|
||||
# @BRIEF Fetch available models from the provider's API.
|
||||
# @PRE Client is initialized with provider credentials.
|
||||
# @POST Returns a list of model ID strings.
|
||||
# @SIDE_EFFECT Calls external LLM API /v1/models endpoint.
|
||||
async def fetch_models(self) -> list[str]:
|
||||
with belief_scope("LLMClient.fetch_models"):
|
||||
try:
|
||||
response = await self.client.models.list()
|
||||
model_ids = [m.id for m in response.data]
|
||||
model_ids.sort()
|
||||
logger.reason(
|
||||
"Fetched available models from provider",
|
||||
payload={"model_count": len(model_ids), "base_url": self.base_url},
|
||||
)
|
||||
return model_ids
|
||||
except Exception as e:
|
||||
logger.explore(
|
||||
"Failed to fetch models from provider",
|
||||
payload={"base_url": self.base_url, "formatted_error": self._format_connection_error(e)},
|
||||
error=str(e),
|
||||
)
|
||||
raise
|
||||
|
||||
# #endregion LLMClient.fetch_models
|
||||
# #endregion Plugin.Service.LLMClientCoreMixin
|
||||
# #endregion Plugin.Service.LLMClient.CoreModule
|
||||
60
backend/src/plugins/llm_analysis/_redaction.py
Normal file
60
backend/src/plugins/llm_analysis/_redaction.py
Normal file
@@ -0,0 +1,60 @@
|
||||
# #region Plugin.Service.RedactionModule [C:2] [TYPE Module] [SEMANTICS llm,redaction,pii,security]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses.
|
||||
# @LAYER Plugin
|
||||
|
||||
import re
|
||||
|
||||
# #region Plugin.Service.RedactionService [C:2] [TYPE Module]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Redacts PII, credentials, and sensitive data from logs and LLM responses.
|
||||
# @LAYER Service
|
||||
# @RATIONALE FR-029: sensitive data must be filtered BEFORE external LLM send and BEFORE persistence.
|
||||
class RedactionService:
|
||||
"""Redacts PII, credentials, and sensitive data."""
|
||||
|
||||
# Common patterns to redact
|
||||
PATTERNS = [
|
||||
(r"password[=:]\s*\S+", "password=***"),
|
||||
(r"secret[=:]\s*\S+", "secret=***"),
|
||||
(r"token[=:]\s*\S+", "token=***"),
|
||||
(r"api_key[=:]\s*\S+", "api_key=***"),
|
||||
(r"apikey[=:]\s*\S+", "apikey=***"),
|
||||
(r"Authorization:\s*\S+", "Authorization: ***"),
|
||||
(r"Bearer\s+\S+\.\S+\.\S+", "Bearer ***"),
|
||||
(r"[A-Za-z0-9+/=]{40,}", "***"), # base64 or long tokens
|
||||
(r"[\w.+-]+@[\w-]+\.[\w.-]+", "***@***"), # emails
|
||||
]
|
||||
|
||||
# #region RedactionService.redact_logs [TYPE Function] [C:2]
|
||||
# @BRIEF Redact PII/credentials from log lines.
|
||||
# @PRE logs is a list of strings.
|
||||
# @POST Returns redacted list preserving structure.
|
||||
@staticmethod
|
||||
def redact_logs(logs: list[str]) -> list[str]:
|
||||
"""Redact PII/credentials from log lines."""
|
||||
redacted = []
|
||||
for line in logs:
|
||||
for pattern, replacement in RedactionService.PATTERNS:
|
||||
line = re.sub(pattern, replacement, line, flags=re.IGNORECASE)
|
||||
redacted.append(line)
|
||||
return redacted
|
||||
|
||||
# #endregion RedactionService.redact_logs
|
||||
|
||||
# #region RedactionService.redact_raw_response [TYPE Function] [C:2]
|
||||
# @BRIEF Redact sensitive data from LLM raw response.
|
||||
# @PRE raw is a string.
|
||||
# @POST Returns redacted string.
|
||||
@staticmethod
|
||||
def redact_raw_response(raw: str) -> str:
|
||||
"""Redact sensitive data from LLM raw response."""
|
||||
for pattern, replacement in RedactionService.PATTERNS:
|
||||
raw = re.sub(pattern, replacement, raw, flags=re.IGNORECASE)
|
||||
return raw
|
||||
|
||||
# #endregion RedactionService.redact_raw_response
|
||||
|
||||
|
||||
# #endregion Plugin.Service.RedactionService
|
||||
# #endregion Plugin.Service.RedactionModule
|
||||
39
backend/src/plugins/llm_analysis/_screenshot.py
Normal file
39
backend/src/plugins/llm_analysis/_screenshot.py
Normal file
@@ -0,0 +1,39 @@
|
||||
# #region Plugin.Service.Screenshot [C:4] [TYPE Module] [SEMANTICS llm,screenshot,facade]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Facade assembling ScreenshotService from login/wait/session/capture/media mixins.
|
||||
# @LAYER Plugin
|
||||
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.LoginModule]
|
||||
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.WaitModule]
|
||||
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.SessionModule]
|
||||
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.CaptureModule]
|
||||
# @RELATION DEPENDS_ON -> [Plugin.Service.Screenshot.MediaModule]
|
||||
|
||||
from ...core.config_models import Environment
|
||||
from ._screenshot_capture import ScreenshotCaptureMixin
|
||||
from ._screenshot_login import ScreenshotLoginMixin
|
||||
from ._screenshot_media import ScreenshotMediaMixin
|
||||
from ._screenshot_session import ScreenshotSessionMixin
|
||||
from ._screenshot_wait import ScreenshotWaitMixin
|
||||
|
||||
__all__ = ["ScreenshotService"]
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotService [C:4] [TYPE Class] [SEMANTICS llm,screenshot,playwright]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Handles capturing screenshots of Superset dashboards.
|
||||
# @SIDE_EFFECT Launches Playwright browser; captures screenshots to disk.
|
||||
class ScreenshotService(
|
||||
ScreenshotLoginMixin,
|
||||
ScreenshotWaitMixin,
|
||||
ScreenshotSessionMixin,
|
||||
ScreenshotCaptureMixin,
|
||||
ScreenshotMediaMixin,
|
||||
):
|
||||
# #region ScreenshotService.__init__ [C:1] [TYPE Function] [SEMANTICS init]
|
||||
# @BRIEF Initializes the ScreenshotService with environment configuration.
|
||||
# @PRE env is a valid Environment object.
|
||||
def __init__(self, env: Environment):
|
||||
self.env = env
|
||||
# #endregion ScreenshotService.__init__
|
||||
# #endregion Plugin.Service.ScreenshotService
|
||||
# #endregion Plugin.Service.Screenshot
|
||||
256
backend/src/plugins/llm_analysis/_screenshot_capture.py
Normal file
256
backend/src/plugins/llm_analysis/_screenshot_capture.py
Normal file
@@ -0,0 +1,256 @@
|
||||
# #region Plugin.Service.Screenshot.CaptureModule [C:4] [TYPE Module] [SEMANTICS llm,screenshot,capture,tabs]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Per-tab CDP screenshot capture and orchestration for ScreenshotService (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ._constants import PLAYWRIGHT_WAIT_TIMEOUT_MS
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotCaptureMixin [C:4] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: browser session launch delegation, per-tab chunked capture, PNG->JPEG/WebP pipeline.
|
||||
class ScreenshotCaptureMixin:
|
||||
# #region ScreenshotService.capture_dashboard_chunks [C:4] [TYPE Function]
|
||||
# @BRIEF Capture per-tab screenshots: login → navigate → switch tabs → per-tab CDP screenshots.
|
||||
# @PRE dashboard_id is valid, browser available.
|
||||
# @POST Returns list of {tab_name, path} dicts — one per tab.
|
||||
# @SIDE_EFFECT Launches browser, logs in, switches tabs, captures screenshots.
|
||||
# @RATIONALE Multi-chunk: one screenshot per tab instead of one full-page.
|
||||
# All screenshots written to output_dir; CDP fallback to Playwright full_page.
|
||||
# @REJECTED Single full-page screenshot rejected for v2 — per-tab captures give LLM
|
||||
# better visibility into individual tab content, especially for dashboards with
|
||||
# many tabs where the full-page capture may miss lazy-loaded tab content.
|
||||
async def capture_dashboard_chunks(
|
||||
self,
|
||||
dashboard_id: str,
|
||||
output_dir: str,
|
||||
parsed_context: dict | None = None,
|
||||
) -> list[dict]:
|
||||
"""Capture per-tab screenshots instead of one full-page.
|
||||
|
||||
Args:
|
||||
dashboard_id: Superset dashboard ID
|
||||
output_dir: Directory to save screenshots
|
||||
parsed_context: Optional parsed context with activeTabs, native_filters from URL parse
|
||||
|
||||
Returns:
|
||||
list of {tab_name, path} — one per tab
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
timestamp = int(_time.time())
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser, context, page = await self._launch_and_login(p, dashboard_id, parsed_context)
|
||||
try:
|
||||
results: list[dict] = []
|
||||
processed_tabs: set[str] = set()
|
||||
|
||||
async def _capture_tabs(depth: int = 0) -> None:
|
||||
if depth > 3:
|
||||
return
|
||||
|
||||
tab_selectors = [
|
||||
".ant-tabs-nav-list .ant-tabs-tab",
|
||||
".dashboard-component-tabs .ant-tabs-tab",
|
||||
'[data-test="dashboard-component-tabs"] .ant-tabs-tab',
|
||||
]
|
||||
|
||||
found_tabs = []
|
||||
for selector in tab_selectors:
|
||||
found_tabs = await page.locator(selector).all()
|
||||
if found_tabs:
|
||||
break
|
||||
|
||||
if not found_tabs:
|
||||
return
|
||||
|
||||
logger.reason("Found tabs at current depth", payload={"tab_count": len(found_tabs), "depth": depth})
|
||||
for i, tab in enumerate(found_tabs):
|
||||
try:
|
||||
tab_text = (await tab.inner_text()).strip()
|
||||
tab_id = f"{depth}_{i}_{tab_text}"
|
||||
|
||||
if tab_id in processed_tabs:
|
||||
continue
|
||||
|
||||
if not await tab.is_visible():
|
||||
continue
|
||||
|
||||
processed_tabs.add(tab_id)
|
||||
logger.reason("Switching to tab", payload={"tab_text": tab_text, "depth": depth, "index": i})
|
||||
|
||||
is_active = "ant-tabs-tab-active" in (await tab.get_attribute("class") or "")
|
||||
if not is_active:
|
||||
await tab.click()
|
||||
try:
|
||||
await page.wait_for_function(
|
||||
"""() => {
|
||||
const activeTab = document.querySelector('.ant-tabs-tab-active');
|
||||
if (!activeTab) return true;
|
||||
const tabPane = activeTab.closest('.ant-tabs')?.querySelector('.ant-tabs-content-holder');
|
||||
if (!tabPane) return true;
|
||||
const charts = tabPane.querySelectorAll('canvas, svg');
|
||||
return charts.length > 0;
|
||||
}""",
|
||||
timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS,
|
||||
)
|
||||
except Exception:
|
||||
logger.explore(
|
||||
"Content verification timed out after tab switch",
|
||||
payload={"tab_text": tab_text},
|
||||
error="Verification wait timed out",
|
||||
)
|
||||
|
||||
# Wait for charts to stabilize
|
||||
await self._wait_for_charts_stabilized(page)
|
||||
|
||||
# Resize viewport to 1920x1200 for consistent screenshots
|
||||
await page.set_viewport_size({"width": 1920, "height": 1200})
|
||||
await self._wait_for_resize_rendered(page, {})
|
||||
|
||||
# CDP screenshot with fallback
|
||||
safe_tab = re.sub(r"[^\w\-_ ]", "", tab_text).strip().replace(" ", "_")[:40]
|
||||
if not safe_tab:
|
||||
safe_tab = f"tab_{depth}_{i}"
|
||||
|
||||
tab_filename = f"{dashboard_id}_{safe_tab}_{timestamp}_d{depth}.png"
|
||||
tab_path = os.path.join(output_dir, tab_filename)
|
||||
|
||||
try:
|
||||
cdp = await page.context.new_cdp_session(page)
|
||||
screenshot_data = await cdp.send(
|
||||
"Page.captureScreenshot",
|
||||
{
|
||||
"format": "png",
|
||||
"fromSurface": True,
|
||||
"captureBeyondViewport": True,
|
||||
},
|
||||
)
|
||||
image_data = base64.b64decode(screenshot_data["data"])
|
||||
with open(tab_path, "wb") as f:
|
||||
f.write(image_data)
|
||||
except Exception as cdp_err:
|
||||
logger.explore(
|
||||
"CDP screenshot failed, falling back to Playwright full_page",
|
||||
payload={"tab_text": tab_text},
|
||||
error=str(cdp_err),
|
||||
)
|
||||
await page.screenshot(path=tab_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
|
||||
|
||||
logger.reason("Saved screenshot for tab", payload={"tab_path": tab_path, "tab_text": tab_text})
|
||||
results.append({"tab_name": tab_text, "path": tab_path})
|
||||
|
||||
# Recurse into nested tabs
|
||||
await _capture_tabs(depth + 1)
|
||||
|
||||
except Exception as tab_e:
|
||||
logger.explore(
|
||||
"Failed to process tab",
|
||||
payload={"tab_index": i, "depth": depth},
|
||||
error=str(tab_e),
|
||||
)
|
||||
|
||||
# Return to first tab
|
||||
try:
|
||||
first_tab = found_tabs[0]
|
||||
if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""):
|
||||
await first_tab.click()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await _capture_tabs()
|
||||
|
||||
# If no tabs found, capture the whole page as a single chunk
|
||||
if not results:
|
||||
logger.reason("No tabs found, capturing full-page as single chunk")
|
||||
await self._wait_for_charts_stabilized(page)
|
||||
await page.set_viewport_size({"width": 1920, "height": 1200})
|
||||
|
||||
tab_path = os.path.join(output_dir, f"{dashboard_id}_full_{timestamp}.png")
|
||||
try:
|
||||
cdp = await page.context.new_cdp_session(page)
|
||||
screenshot_data = await cdp.send(
|
||||
"Page.captureScreenshot",
|
||||
{
|
||||
"format": "png",
|
||||
"fromSurface": True,
|
||||
"captureBeyondViewport": True,
|
||||
},
|
||||
)
|
||||
image_data = base64.b64decode(screenshot_data["data"])
|
||||
with open(tab_path, "wb") as f:
|
||||
f.write(image_data)
|
||||
except Exception as cdp_err:
|
||||
logger.explore("CDP full-page fallback failed, using Playwright full_page", payload={}, error=str(cdp_err))
|
||||
await page.screenshot(path=tab_path, full_page=True, timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS)
|
||||
|
||||
results.append({"tab_name": "full", "path": tab_path})
|
||||
|
||||
return results
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
# #endregion ScreenshotService.capture_dashboard_chunks
|
||||
|
||||
# #region ScreenshotService.capture_dashboard [C:4] [TYPE Function]
|
||||
# @BRIEF Captures multi-chunk screenshots, converts for LLM, archives to WebP.
|
||||
# @PRE dashboard_id is a valid string, output_path is a writable path.
|
||||
# @POST Returns list of {original, webp_path} dicts from WebP archive.
|
||||
# Empty list on complete failure.
|
||||
# @SIDE_EFFECT Launches browser, performs UI login, writes PNG/JPEG/WebP files;
|
||||
# deletes intermediate PNG and JPEG files after conversion.
|
||||
# @RATIONALE Refactored v2: delegates to capture_dashboard_chunks for per-tab PNGs,
|
||||
# then converts for LLM (JPEG) and archive (WebP). Temp intermediates are cleaned up.
|
||||
# @REJECTED Returning bool rejected for v2 — callers need access to archived WebP paths
|
||||
# for persistence and LLM pipeline.
|
||||
async def capture_dashboard(self, dashboard_id: str, output_path: str) -> tuple[list[str], list[dict]]:
|
||||
"""Capture dashboard screenshots (multi-chunk), convert for LLM, archive to WebP.
|
||||
|
||||
Returns (jpeg_paths, archive_results) tuple.
|
||||
jpeg_paths — list of JPEG paths ready for LLM analysis (caller must clean up).
|
||||
archive_results — list of {original, webp_path} dicts from WebP archive.
|
||||
"""
|
||||
output_dir = os.path.dirname(output_path) or "."
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
with belief_scope("capture_dashboard", f"dashboard_id={dashboard_id}"):
|
||||
logger.reason("Capturing dashboard screenshots", payload={"dashboard_id": dashboard_id})
|
||||
|
||||
# 1. Capture per-tab screenshots
|
||||
chunks = await self.capture_dashboard_chunks(dashboard_id, output_dir)
|
||||
png_paths = [c["path"] for c in chunks if c.get("path")]
|
||||
|
||||
if not png_paths:
|
||||
logger.explore("No screenshots captured for dashboard", payload={"dashboard_id": dashboard_id}, error="All capture attempts returned empty")
|
||||
return [], []
|
||||
|
||||
# 2. Convert PNGs to JPEGs for LLM
|
||||
jpeg_paths = self._convert_screenshots_for_llm(png_paths, output_dir)
|
||||
logger.reason(
|
||||
"Converted PNGs to JPEGs for LLM analysis",
|
||||
payload={"converted": len(jpeg_paths), "total": len(png_paths)},
|
||||
)
|
||||
|
||||
# 3. Archive to WebP (deletes PNGs on success)
|
||||
archive_results = self._archive_screenshots_as_webp(png_paths, output_dir)
|
||||
archived_count = sum(1 for r in archive_results if r.get("webp_path"))
|
||||
logger.reason(
|
||||
"Archived screenshots to WebP",
|
||||
payload={"archived": archived_count, "total": len(archive_results)},
|
||||
)
|
||||
|
||||
# 4. Return JPEGs for LLM — caller cleans up after analysis
|
||||
return jpeg_paths, archive_results
|
||||
|
||||
# #endregion ScreenshotService.capture_dashboard
|
||||
# #endregion Plugin.Service.ScreenshotCaptureMixin
|
||||
# #endregion Plugin.Service.Screenshot.CaptureModule
|
||||
264
backend/src/plugins/llm_analysis/_screenshot_login.py
Normal file
264
backend/src/plugins/llm_analysis/_screenshot_login.py
Normal file
@@ -0,0 +1,264 @@
|
||||
# #region Plugin.Service.Screenshot.LoginModule [C:3] [TYPE Module] [SEMANTICS llm,screenshot,login,playwright]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Login and navigation locator helpers for ScreenshotService (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from ...core.logger import logger
|
||||
from ._constants import HTTP_REQUEST_TIMEOUT_MS, PLAYWRIGHT_WAIT_TIMEOUT_MS
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotLoginMixin [C:3] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: login-form discovery, fallback form POST, resilient navigation.
|
||||
class ScreenshotLoginMixin:
|
||||
# #region ScreenshotService._find_first_visible_locator [C:3] [TYPE Function]
|
||||
# @BRIEF Resolve the first visible locator from multiple Playwright locator strategies.
|
||||
# @PRE candidates is a non-empty list of locator-like objects.
|
||||
# @POST Returns a locator ready for interaction or None when nothing matches.
|
||||
async def _find_first_visible_locator(self, candidates) -> Any:
|
||||
for locator in candidates:
|
||||
try:
|
||||
match_count = await locator.count()
|
||||
for index in range(match_count):
|
||||
candidate = locator.nth(index)
|
||||
if await candidate.is_visible():
|
||||
return candidate
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
# #endregion ScreenshotService._find_first_visible_locator
|
||||
|
||||
# #region ScreenshotService._iter_login_roots [C:2] [TYPE Function]
|
||||
# @BRIEF Enumerate page and child frames where login controls may be rendered.
|
||||
# @PRE page is a Playwright page-like object.
|
||||
# @POST Returns ordered roots starting with main page followed by frames.
|
||||
def _iter_login_roots(self, page) -> list[Any]:
|
||||
roots = [page]
|
||||
page_frames = getattr(page, "frames", [])
|
||||
try:
|
||||
for frame in page_frames:
|
||||
if frame not in roots:
|
||||
roots.append(frame)
|
||||
except Exception:
|
||||
pass
|
||||
return roots
|
||||
|
||||
# #endregion ScreenshotService._iter_login_roots
|
||||
|
||||
# #region ScreenshotService._extract_hidden_login_fields [C:2] [TYPE Function]
|
||||
# @BRIEF Collect hidden form fields required for direct login POST fallback.
|
||||
# @PRE Login page is loaded.
|
||||
# @POST Returns hidden input name/value mapping aggregated from page and child frames.
|
||||
async def _extract_hidden_login_fields(self, page) -> dict[str, str]:
|
||||
hidden_fields: dict[str, str] = {}
|
||||
for root in self._iter_login_roots(page):
|
||||
try:
|
||||
locator = root.locator("input[type='hidden'][name]")
|
||||
count = await locator.count()
|
||||
for index in range(count):
|
||||
candidate = locator.nth(index)
|
||||
field_name = str(await candidate.get_attribute("name") or "").strip()
|
||||
if not field_name or field_name in hidden_fields:
|
||||
continue
|
||||
hidden_fields[field_name] = str(await candidate.input_value()).strip()
|
||||
except Exception:
|
||||
continue
|
||||
return hidden_fields
|
||||
|
||||
# #endregion ScreenshotService._extract_hidden_login_fields
|
||||
|
||||
# #region ScreenshotService._extract_csrf_token [C:2] [TYPE Function]
|
||||
# @BRIEF Resolve CSRF token value from main page or embedded login frame.
|
||||
# @PRE Login page is loaded.
|
||||
# @POST Returns first non-empty csrf token or empty string.
|
||||
async def _extract_csrf_token(self, page) -> str:
|
||||
hidden_fields = await self._extract_hidden_login_fields(page)
|
||||
return str(hidden_fields.get("csrf_token") or "").strip()
|
||||
|
||||
# #endregion ScreenshotService._extract_csrf_token
|
||||
|
||||
# #region ScreenshotService._response_looks_like_login_page [C:2] [TYPE Function]
|
||||
# @BRIEF Detect when fallback login POST returned the login form again instead of an authenticated page.
|
||||
# @PRE response_text is normalized HTML or text from login POST response.
|
||||
# @POST Returns True when login-page markers dominate the response body.
|
||||
def _response_looks_like_login_page(self, response_text: str) -> bool:
|
||||
normalized = str(response_text or "").strip().lower()
|
||||
if not normalized:
|
||||
return False
|
||||
|
||||
markers = [
|
||||
"enter your login and password below",
|
||||
"username:",
|
||||
"password:",
|
||||
"sign in",
|
||||
'name="csrf_token"',
|
||||
]
|
||||
return sum(marker in normalized for marker in markers) >= 3
|
||||
|
||||
# #endregion ScreenshotService._response_looks_like_login_page
|
||||
|
||||
# #region ScreenshotService._redirect_looks_authenticated [C:2] [TYPE Function]
|
||||
# @BRIEF Treat non-login redirects after form POST as successful authentication without waiting for redirect target.
|
||||
# @PRE redirect_location may be empty or relative.
|
||||
# @POST Returns True when redirect target does not point back to login flow.
|
||||
def _redirect_looks_authenticated(self, redirect_location: str) -> bool:
|
||||
normalized = str(redirect_location or "").strip().lower()
|
||||
if not normalized:
|
||||
return True
|
||||
return "/login" not in normalized
|
||||
|
||||
# #endregion ScreenshotService._redirect_looks_authenticated
|
||||
|
||||
# #region ScreenshotService._submit_login_via_form_post [C:3] [TYPE Function]
|
||||
# @BRIEF Fallback login path that submits credentials directly with csrf token.
|
||||
# @PRE login_url is same-origin and csrf token can be read from DOM.
|
||||
# @POST Browser context receives authenticated cookies when login succeeds.
|
||||
async def _submit_login_via_form_post(self, page, login_url: str) -> bool:
|
||||
hidden_fields = await self._extract_hidden_login_fields(page)
|
||||
csrf_token = str(hidden_fields.get("csrf_token") or "").strip()
|
||||
if not csrf_token:
|
||||
logger.explore("Direct form login fallback skipped: csrf_token not found", error="Missing csrf_token in login form")
|
||||
return False
|
||||
|
||||
try:
|
||||
request_context = page.context.request
|
||||
except Exception as context_error:
|
||||
logger.explore("Direct form login fallback skipped: request context unavailable", payload={"error": str(context_error)}, error="Browser request context unavailable")
|
||||
return False
|
||||
|
||||
parsed_url = urlsplit(login_url)
|
||||
origin = f"{parsed_url.scheme}://{parsed_url.netloc}" if parsed_url.scheme and parsed_url.netloc else login_url
|
||||
payload = dict(hidden_fields)
|
||||
payload["username"] = self.env.username
|
||||
payload["password"] = self.env.password
|
||||
logger.reason(
|
||||
"Attempting direct form login fallback via browser context request",
|
||||
payload={"hidden_fields": sorted(hidden_fields.keys())},
|
||||
)
|
||||
|
||||
response = await request_context.post(
|
||||
login_url,
|
||||
form=payload,
|
||||
headers={
|
||||
"Origin": origin,
|
||||
"Referer": login_url,
|
||||
},
|
||||
timeout=PLAYWRIGHT_WAIT_TIMEOUT_MS,
|
||||
fail_on_status_code=False,
|
||||
max_redirects=0,
|
||||
)
|
||||
response_url = str(getattr(response, "url", "") or "")
|
||||
response_status = int(getattr(response, "status", 0) or 0)
|
||||
response_headers = dict(getattr(response, "headers", {}) or {})
|
||||
redirect_location = str(response_headers.get("location") or response_headers.get("Location") or "").strip()
|
||||
redirect_statuses = {301, 302, 303, 307, 308}
|
||||
if response_status in redirect_statuses:
|
||||
redirect_authenticated = self._redirect_looks_authenticated(redirect_location)
|
||||
logger.reason(
|
||||
"Direct form login fallback redirect response",
|
||||
payload={"status": response_status, "url": response_url, "location": redirect_location, "authenticated": redirect_authenticated},
|
||||
)
|
||||
return redirect_authenticated
|
||||
|
||||
response_text = await response.text()
|
||||
text_snippet = " ".join(response_text.split())[:200]
|
||||
looks_like_login_page = self._response_looks_like_login_page(response_text)
|
||||
logger.reason(
|
||||
"Direct form login fallback response",
|
||||
payload={"status": response_status, "url": response_url, "login_markup": looks_like_login_page, "snippet": text_snippet},
|
||||
)
|
||||
return not looks_like_login_page
|
||||
|
||||
# #endregion ScreenshotService._submit_login_via_form_post
|
||||
|
||||
# #region ScreenshotService._find_login_field_locator [C:3] [TYPE Function]
|
||||
# @BRIEF Resolve login form input using semantic label text plus generic visible-input fallbacks.
|
||||
# @PRE field_name is `username` or `password`.
|
||||
# @POST Returns a locator for the corresponding input or None.
|
||||
async def _find_login_field_locator(self, page, field_name: str) -> Any:
|
||||
normalized = str(field_name or "").strip().lower()
|
||||
for root in self._iter_login_roots(page):
|
||||
if normalized == "username":
|
||||
input_candidates = [
|
||||
root.get_by_label("Username", exact=False),
|
||||
root.get_by_label("Login", exact=False),
|
||||
root.locator("label:text-matches('Username|Login', 'i')").locator("xpath=following::input[1]"),
|
||||
root.locator("text=/Username|Login/i").locator("xpath=following::input[1]"),
|
||||
root.locator("input[name='username']"),
|
||||
root.locator("input#username"),
|
||||
root.locator("input[placeholder*='Username']"),
|
||||
root.locator("input[type='text']"),
|
||||
root.locator("input:not([type='password'])"),
|
||||
]
|
||||
locator = await self._find_first_visible_locator(input_candidates)
|
||||
if locator:
|
||||
return locator
|
||||
|
||||
if normalized == "password":
|
||||
input_candidates = [
|
||||
root.get_by_label("Password", exact=False),
|
||||
root.locator("label:text-matches('Password', 'i')").locator("xpath=following::input[1]"),
|
||||
root.locator("text=/Password/i").locator("xpath=following::input[1]"),
|
||||
root.locator("input[name='password']"),
|
||||
root.locator("input#password"),
|
||||
root.locator("input[placeholder*='Password']"),
|
||||
root.locator("input[type='password']"),
|
||||
]
|
||||
locator = await self._find_first_visible_locator(input_candidates)
|
||||
if locator:
|
||||
return locator
|
||||
|
||||
return None
|
||||
|
||||
# #endregion ScreenshotService._find_login_field_locator
|
||||
|
||||
# #region ScreenshotService._find_submit_locator [C:3] [TYPE Function]
|
||||
# @BRIEF Resolve login submit button from main page or embedded auth frame.
|
||||
# @PRE page is ready for login interaction.
|
||||
# @POST Returns visible submit locator or None.
|
||||
async def _find_submit_locator(self, page) -> Any:
|
||||
selectors = [
|
||||
lambda root: root.get_by_role("button", name="Sign in", exact=False),
|
||||
lambda root: root.get_by_role("button", name="Login", exact=False),
|
||||
lambda root: root.locator("button[type='submit']"),
|
||||
lambda root: root.locator("button#submit"),
|
||||
lambda root: root.locator(".btn-primary"),
|
||||
lambda root: root.locator("input[type='submit']"),
|
||||
]
|
||||
for root in self._iter_login_roots(page):
|
||||
locator = await self._find_first_visible_locator([factory(root) for factory in selectors])
|
||||
if locator:
|
||||
return locator
|
||||
return None
|
||||
|
||||
# #endregion ScreenshotService._find_submit_locator
|
||||
|
||||
# #region ScreenshotService._goto_resilient [C:3] [TYPE Function]
|
||||
# @BRIEF Navigate without relying on networkidle for pages with long-polling or persistent requests.
|
||||
# @PRE page is a valid Playwright page and url is non-empty.
|
||||
# @POST Returns last navigation response or raises when both primary and fallback waits fail.
|
||||
async def _goto_resilient(
|
||||
self,
|
||||
page,
|
||||
url: str,
|
||||
primary_wait_until: str = "domcontentloaded",
|
||||
fallback_wait_until: str = "load",
|
||||
timeout: int = HTTP_REQUEST_TIMEOUT_MS,
|
||||
):
|
||||
try:
|
||||
return await page.goto(url, wait_until=primary_wait_until, timeout=timeout)
|
||||
except Exception as primary_error:
|
||||
logger.explore(
|
||||
"Primary navigation wait failed, falling back to fallback wait",
|
||||
payload={"primary_wait_until": primary_wait_until, "url": url},
|
||||
error=str(primary_error),
|
||||
)
|
||||
return await page.goto(url, wait_until=fallback_wait_until, timeout=timeout)
|
||||
|
||||
# #endregion ScreenshotService._goto_resilient
|
||||
# #endregion Plugin.Service.ScreenshotLoginMixin
|
||||
# #endregion Plugin.Service.Screenshot.LoginModule
|
||||
102
backend/src/plugins/llm_analysis/_screenshot_media.py
Normal file
102
backend/src/plugins/llm_analysis/_screenshot_media.py
Normal file
@@ -0,0 +1,102 @@
|
||||
# #region Plugin.Service.Screenshot.MediaModule [C:2] [TYPE Module] [SEMANTICS llm,screenshot,jpeg,webp,archive]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Screenshot format conversion and cleanup helpers for ScreenshotService (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotMediaMixin [C:2] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: PNG to JPEG conversion for LLM, WebP archiving, temp-file cleanup.
|
||||
class ScreenshotMediaMixin:
|
||||
# #region ScreenshotService._convert_screenshots_for_llm [TYPE Function] [C:2]
|
||||
# @BRIEF Convert PNG screenshots to JPEG for LLM transmission.
|
||||
# @PRE png_paths is a list of existing PNG file paths.
|
||||
# @POST Returns list of JPEG paths. JPEG files should be deleted after LLM call.
|
||||
@staticmethod
|
||||
def _convert_screenshots_for_llm(png_paths: list[str], output_dir: str) -> list[str]:
|
||||
"""Convert PNG screenshots to JPEG for LLM transmission.
|
||||
|
||||
PNG → Pillow → JPEG quality=60, max 1024px width.
|
||||
Returns list of JPEG paths.
|
||||
"""
|
||||
jpeg_paths: list[str] = []
|
||||
for png_path in png_paths:
|
||||
try:
|
||||
img = Image.open(png_path)
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
|
||||
max_width = 1024
|
||||
if img.width > max_width:
|
||||
scale = max_width / img.width
|
||||
new_w = int(img.width * scale)
|
||||
new_h = int(img.height * scale)
|
||||
img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
|
||||
|
||||
base = os.path.splitext(os.path.basename(png_path))[0]
|
||||
jpeg_path = os.path.join(output_dir, f"{base}_llm.jpg")
|
||||
img.save(jpeg_path, format="JPEG", quality=60, optimize=True)
|
||||
jpeg_paths.append(jpeg_path)
|
||||
except Exception as e:
|
||||
logger.explore("Failed to convert screenshot for LLM", payload={"png_path": png_path}, error=str(e))
|
||||
|
||||
return jpeg_paths
|
||||
|
||||
# #endregion ScreenshotService._convert_screenshots_for_llm
|
||||
|
||||
# #region ScreenshotService._archive_screenshots_as_webp [TYPE Function] [C:2]
|
||||
# @BRIEF Convert PNG screenshots to WebP for archive.
|
||||
# @PRE png_paths is a list of existing PNG file paths.
|
||||
# @POST Returns list of {original, webp_path} dicts. PNG deleted after successful WebP save.
|
||||
@staticmethod
|
||||
def _archive_screenshots_as_webp(png_paths: list[str], archive_dir: str) -> list[dict]:
|
||||
"""Convert PNG screenshots to WebP for archive.
|
||||
|
||||
PNG → Pillow → WebP lossy quality=80.
|
||||
Deletes PNG after WebP saved.
|
||||
Returns list of {original, webp_path} dicts.
|
||||
"""
|
||||
results: list[dict] = []
|
||||
for png_path in png_paths:
|
||||
try:
|
||||
img = Image.open(png_path)
|
||||
if img.mode in ("RGBA", "P"):
|
||||
img = img.convert("RGB")
|
||||
|
||||
base = os.path.splitext(os.path.basename(png_path))[0]
|
||||
webp_path = os.path.join(archive_dir, f"{base}.webp")
|
||||
img.save(webp_path, format="WEBP", quality=80, lossless=False)
|
||||
|
||||
# Delete PNG after successful WebP save
|
||||
os.remove(png_path)
|
||||
|
||||
results.append({"original": png_path, "webp_path": webp_path})
|
||||
except Exception as e:
|
||||
logger.explore("Failed to archive screenshot to WebP, keeping PNG", payload={"png_path": png_path}, error=str(e))
|
||||
results.append({"original": png_path, "webp_path": None})
|
||||
|
||||
return results
|
||||
|
||||
# #endregion ScreenshotService._archive_screenshots_as_webp
|
||||
|
||||
# #region ScreenshotService._cleanup_temp_files [TYPE Function] [C:1]
|
||||
# @BRIEF Delete temporary files (PNG, JPEG intermediates).
|
||||
@staticmethod
|
||||
def _cleanup_temp_files(paths: list[str]) -> None:
|
||||
"""Delete temporary files (PNG, JPEG intermediates)."""
|
||||
for path in paths:
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
except Exception as e:
|
||||
logger.explore("Failed to delete temporary file", payload={"path": path}, error=str(e))
|
||||
|
||||
# #endregion ScreenshotService._cleanup_temp_files
|
||||
# #endregion Plugin.Service.ScreenshotMediaMixin
|
||||
# #endregion Plugin.Service.Screenshot.MediaModule
|
||||
187
backend/src/plugins/llm_analysis/_screenshot_session.py
Normal file
187
backend/src/plugins/llm_analysis/_screenshot_session.py
Normal file
@@ -0,0 +1,187 @@
|
||||
# #region Plugin.Service.Screenshot.SessionModule [C:4] [TYPE Module] [SEMANTICS llm,screenshot,session,login,navigation]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Browser session launch, UI login and dashboard navigation for ScreenshotService (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
from typing import Any
|
||||
|
||||
from ...core.logger import logger
|
||||
from ._constants import HTTP_REQUEST_TIMEOUT_MS, PLAYWRIGHT_NAVIGATION_TIMEOUT_MS
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotSessionMixin [C:4] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: headless Chromium launch with anti-automation flags, login flow, dashboard navigation.
|
||||
class ScreenshotSessionMixin:
|
||||
# #region ScreenshotService._launch_and_login [C:4] [TYPE Function]
|
||||
# @BRIEF Launch browser, log in to Superset, navigate to dashboard URL.
|
||||
# @PRE dashboard_id is valid, playwright instance is provided.
|
||||
# @POST Returns (browser, context, page) tuple with active session.
|
||||
# @SIDE_EFFECT Launches headless Chromium, performs UI login, navigates to dashboard.
|
||||
# @RATIONALE Extracted from capture_dashboard to share login/navigation logic
|
||||
# between capture_dashboard (backward compat) and capture_dashboard_chunks (multi-tab).
|
||||
# @REJECTED Duplicating login logic rejected — any change to auth flow would require
|
||||
# updating two code paths, leading to accidental drift.
|
||||
async def _launch_and_login(
|
||||
self,
|
||||
playwright,
|
||||
dashboard_id: str,
|
||||
parsed_context: dict | None = None,
|
||||
) -> tuple[Any, Any, Any]:
|
||||
"""Launch browser, log in to Superset, and navigate to the dashboard.
|
||||
|
||||
Returns (browser, context, page).
|
||||
"""
|
||||
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
base_ui_url = self.env.url.rstrip("/")
|
||||
if base_ui_url.endswith("/api/v1"):
|
||||
base_ui_url = base_ui_url[: -len("/api/v1")]
|
||||
|
||||
browser = await playwright.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-infobars",
|
||||
"--no-sandbox",
|
||||
],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1280, "height": 720},
|
||||
user_agent=user_agent,
|
||||
extra_http_headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
},
|
||||
)
|
||||
page = await context.new_page()
|
||||
await page.add_init_script("delete Object.getPrototypeOf(navigator).webdriver")
|
||||
|
||||
# 1. Navigate to login page and authenticate
|
||||
login_url = f"{base_ui_url.rstrip('/')}/login/"
|
||||
await self._goto_resilient(
|
||||
page,
|
||||
login_url,
|
||||
primary_wait_until="domcontentloaded",
|
||||
fallback_wait_until="load",
|
||||
timeout=HTTP_REQUEST_TIMEOUT_MS,
|
||||
)
|
||||
await page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
try:
|
||||
used_direct_form_login = False
|
||||
username_locator = await self._find_login_field_locator(page, "username")
|
||||
|
||||
if not username_locator:
|
||||
used_direct_form_login = await self._submit_login_via_form_post(page, login_url)
|
||||
if not used_direct_form_login:
|
||||
raise RuntimeError("Could not find username input field on login page")
|
||||
|
||||
if username_locator is not None:
|
||||
await username_locator.fill(self.env.username)
|
||||
|
||||
password_locator = await self._find_login_field_locator(page, "password") if username_locator is not None else None
|
||||
if username_locator is not None and not password_locator:
|
||||
raise RuntimeError("Could not find password input field on login page")
|
||||
if password_locator is not None:
|
||||
await password_locator.fill(self.env.password)
|
||||
|
||||
submit_locator = await self._find_submit_locator(page) if username_locator is not None else None
|
||||
if username_locator is not None and not submit_locator:
|
||||
raise RuntimeError("Could not find submit button on login page")
|
||||
if submit_locator is not None:
|
||||
await submit_locator.click()
|
||||
|
||||
if not used_direct_form_login:
|
||||
try:
|
||||
await page.wait_for_load_state("load", timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not used_direct_form_login and "/login" in page.url:
|
||||
error_msg = await page.locator(".alert-danger, .error-message").text_content() if await page.locator(".alert-danger, .error-message").count() > 0 else "Unknown error"
|
||||
raise RuntimeError(f"Login failed: {error_msg}")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Login failed: {e!s}")
|
||||
|
||||
# 2. Navigate to dashboard
|
||||
dashboard_url = f"{base_ui_url.rstrip('/')}/superset/dashboard/{dashboard_id}/?standalone=true"
|
||||
if base_ui_url.startswith("https://") and dashboard_url.startswith("http://"):
|
||||
dashboard_url = dashboard_url.replace("http://", "https://")
|
||||
|
||||
if parsed_context:
|
||||
native_filters = parsed_context.get("native_filters")
|
||||
active_tabs = parsed_context.get("activeTabs")
|
||||
if native_filters:
|
||||
dashboard_url += f"&native_filters={native_filters}"
|
||||
if active_tabs:
|
||||
dashboard_url += f"&activeTabs={active_tabs}"
|
||||
|
||||
await self._goto_resilient(
|
||||
page,
|
||||
dashboard_url,
|
||||
primary_wait_until="domcontentloaded",
|
||||
fallback_wait_until="load",
|
||||
timeout=HTTP_REQUEST_TIMEOUT_MS,
|
||||
)
|
||||
|
||||
if "/login" in page.url:
|
||||
raise RuntimeError("Dashboard navigation redirected to login page after authentication")
|
||||
|
||||
# 3. Wait for dashboard content
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
'.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]',
|
||||
timeout=PLAYWRIGHT_NAVIGATION_TIMEOUT_MS,
|
||||
)
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
".loading, .ant-skeleton, .spinner",
|
||||
state="hidden",
|
||||
timeout=HTTP_REQUEST_TIMEOUT_MS,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.wait_for_selector(
|
||||
".chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container",
|
||||
timeout=HTTP_REQUEST_TIMEOUT_MS,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
await page.wait_for_function(
|
||||
"""() => {
|
||||
const charts = document.querySelectorAll('.chart-container, .slice_container');
|
||||
if (charts.length === 0) return true;
|
||||
return Array.from(charts).every(chart => {
|
||||
const hasCanvas = chart.querySelector('canvas') !== null;
|
||||
const hasSvg = chart.querySelector('svg') !== null;
|
||||
const hasContent = chart.innerText.trim().length > 0 || chart.children.length > 0;
|
||||
return hasCanvas || hasSvg || hasContent;
|
||||
});
|
||||
}""",
|
||||
timeout=HTTP_REQUEST_TIMEOUT_MS,
|
||||
)
|
||||
await page.evaluate("""async () => {
|
||||
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
for (let i = 0; i < document.body.scrollHeight; i += 500) {
|
||||
window.scrollTo(0, i);
|
||||
await delay(100);
|
||||
}
|
||||
window.scrollTo(0, 0);
|
||||
await delay(500);
|
||||
}""")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await self._wait_for_charts_stabilized(page)
|
||||
logger.reflect("Login and navigation to dashboard successful", payload={"dashboard_id": dashboard_id})
|
||||
return browser, context, page
|
||||
|
||||
# #endregion ScreenshotService._launch_and_login
|
||||
# #endregion Plugin.Service.ScreenshotSessionMixin
|
||||
# #endregion Plugin.Service.Screenshot.SessionModule
|
||||
88
backend/src/plugins/llm_analysis/_screenshot_wait.py
Normal file
88
backend/src/plugins/llm_analysis/_screenshot_wait.py
Normal file
@@ -0,0 +1,88 @@
|
||||
# #region Plugin.Service.Screenshot.WaitModule [C:2] [TYPE Module] [SEMANTICS llm,screenshot,wait,polling]
|
||||
# @defgroup LLMAnalysis Module group.
|
||||
# @BRIEF Chart stabilization and debug-screenshot helpers for ScreenshotService (mixin).
|
||||
# @LAYER Plugin
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
from ...core.logger import logger
|
||||
|
||||
|
||||
# #region Plugin.Service.ScreenshotWaitMixin [C:2] [TYPE Class]
|
||||
# @ingroup LLMAnalysis
|
||||
# @BRIEF Mixin: chart render polling, post-resize re-render wait, debug screenshots.
|
||||
class ScreenshotWaitMixin:
|
||||
# #region ScreenshotService._wait_for_charts_stabilized [TYPE Function] [C:2]
|
||||
# @BRIEF Wait until chart elements have non-zero dimensions, with polling.
|
||||
# @PRE page is a valid Playwright page.
|
||||
# @POST Waits for chart stabilization or raises on timeout (handled internally).
|
||||
# @RATIONALE Polls for actual chart rendering dimensions rather than using a fixed delay — charts may load at different speeds depending on dashboard complexity and network conditions.
|
||||
# @REJECTED Fixed sleep-based wait rejected — would either waste time (too long) or produce blank screenshots (too short); polling for actual canvas/svg dimensions is more reliable.
|
||||
async def _wait_for_charts_stabilized(self, page, timeout_ms: int = 15000):
|
||||
"""Wait until chart elements have non-zero dimensions, with polling."""
|
||||
# Short initial delay for rendering pipeline to start
|
||||
await asyncio.sleep(0.5)
|
||||
try:
|
||||
await page.wait_for_function(
|
||||
"""() => {
|
||||
const charts = document.querySelectorAll('.chart-container canvas, .slice_container svg, .grid-content canvas');
|
||||
if (charts.length === 0) return true;
|
||||
return Array.from(charts).some(c => {
|
||||
if (c.tagName === 'CANVAS') return c.width > 10 && c.height > 10;
|
||||
if (c.tagName === 'svg') {
|
||||
const bbox = c.getBoundingClientRect();
|
||||
return bbox.width > 10 && bbox.height > 10;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}""",
|
||||
timeout=timeout_ms,
|
||||
)
|
||||
except Exception:
|
||||
logger.explore("Chart stabilization wait timed out, proceeding anyway", error="Timed out waiting for charts to render")
|
||||
|
||||
# #endregion ScreenshotService._wait_for_charts_stabilized
|
||||
|
||||
# #region ScreenshotService._wait_for_resize_rendered [TYPE Function] [C:2]
|
||||
# @BRIEF Wait for charts to re-render after viewport resize.
|
||||
# @PRE page is a valid Playwright page; chart_count_before contains pre-resize element counts.
|
||||
# @POST Waits for chart content to return or timeout.
|
||||
# @RATIONALE After viewport resize, Superset triggers lazy chart re-rendering — this function polls for chart elements to reappear before taking the screenshot.
|
||||
# @REJECTED Single fixed wait after resize rejected — some dashboards re-render instantly while others take seconds; fixed wait is brittle across dashboard types.
|
||||
async def _wait_for_resize_rendered(self, page, chart_count_before: dict, timeout_ms: int = 10000):
|
||||
"""Wait for charts to re-render after viewport resize, with polling."""
|
||||
try:
|
||||
await page.wait_for_function(
|
||||
"""(preCounts) => {
|
||||
const currentCharts = document.querySelectorAll('.chart-container, .slice_container').length;
|
||||
const currentCanvases = document.querySelectorAll('canvas').length;
|
||||
const currentSvgs = document.querySelectorAll('.chart-container svg, .slice_container svg').length;
|
||||
// At least one chart element must be present
|
||||
return currentCharts > 0 && (currentCanvases > 0 || currentSvgs > 0);
|
||||
}""",
|
||||
arg=chart_count_before,
|
||||
timeout=timeout_ms,
|
||||
)
|
||||
except Exception:
|
||||
logger.explore("Re-render wait timed out after viewport resize, proceeding anyway", error="Timed out waiting for charts to re-render after resize")
|
||||
|
||||
# #endregion ScreenshotService._wait_for_resize_rendered
|
||||
|
||||
# #region ScreenshotService._save_debug_screenshot [TYPE Function] [C:1]
|
||||
# @BRIEF Save a debug screenshot to a temp directory for diagnostic purposes.
|
||||
# @PRE debug_dir exists and is writable.
|
||||
# @POST Returns the debug path or None on failure.
|
||||
# @RATIONALE Uses tempfile.mkdtemp() to avoid accumulating .png files in production storage. Temp dir is cleaned up on success (rmtree) or preserved on failure for debugging.
|
||||
# @REJECTED Old approach saved debug .png next to output path — accumulated _debug_failed_login.png, _preresize.png permanently in screenshots dir (F1). In-memory-only debug logging rejected — screenshot state is visual and cannot be captured in logs.
|
||||
async def _save_debug_screenshot(self, page, debug_dir: str, suffix: str) -> str | None:
|
||||
debug_path = os.path.join(debug_dir, suffix)
|
||||
try:
|
||||
await page.screenshot(path=debug_path)
|
||||
return debug_path
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# #endregion ScreenshotService._save_debug_screenshot
|
||||
# #endregion Plugin.Service.ScreenshotWaitMixin
|
||||
# #endregion Plugin.Service.Screenshot.WaitModule
|
||||
@@ -1,4 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# #region Plugin.LlmAnalysis.AuthDiag [C:3] [TYPE Module] [SEMANTICS superset,auth,diagnostics]
|
||||
# @defgroup LlmAnalysis Module group.
|
||||
# @BRIEF Superset auth diagnostics — full-cycle verification of JWT/form/session auth paths.
|
||||
"""
|
||||
Superset Auth Diagnostics Script — Extended v3
|
||||
|
||||
@@ -31,6 +34,9 @@ except ImportError:
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.PrintHelpers [C:1] [TYPE Block]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Console output primitives for the diagnostic report.
|
||||
def banner(msg):
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {msg}")
|
||||
@@ -47,13 +53,21 @@ def warn(msg):
|
||||
|
||||
def section(msg):
|
||||
print(f"\n --- {msg} ---")
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.PrintHelpers
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.HttpClient [C:2] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Build a shared httpx client with optional SSL verification bypass.
|
||||
def http_client(verify: bool = True) -> httpx.Client:
|
||||
import warnings
|
||||
if not verify:
|
||||
warnings.filterwarnings("ignore", message=".*verify=False.*", category=UserWarning)
|
||||
return httpx.Client(verify=verify, timeout=30, follow_redirects=False)
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.HttpClient
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.ExtractSessionCookie [C:2] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Extract session cookie dict from a Set-Cookie header.
|
||||
def extract_session_cookie(set_cookie_header: str) -> dict | None:
|
||||
"""Извлекает session=... из Set-Cookie заголовка."""
|
||||
if not set_cookie_header:
|
||||
@@ -64,10 +78,14 @@ def extract_session_cookie(set_cookie_header: str) -> dict | None:
|
||||
value = part.split("=", 1)[1]
|
||||
return {"name": "session", "value": value}
|
||||
return None
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.ExtractSessionCookie
|
||||
|
||||
|
||||
# ── 1. Reachability ──────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckReachability [C:3] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Probe Superset reachability via /health/ with base-URL fallback.
|
||||
def check_reachability(base_url: str, verify: bool) -> bool:
|
||||
banner("1. Доступность Superset")
|
||||
with http_client(verify) as c:
|
||||
@@ -84,10 +102,14 @@ def check_reachability(base_url: str, verify: bool) -> bool:
|
||||
except httpx.RequestError as e:
|
||||
fail(f"Недоступен: {e}")
|
||||
return False
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckReachability
|
||||
|
||||
|
||||
# ── 2. JWT Login ─────────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtLogin [C:3] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Attempt JWT API login across db/ldap providers.
|
||||
def check_jwt_login(base_url: str, username: str, password: str, verify: bool) -> str | None:
|
||||
banner("2. JWT API Login")
|
||||
with http_client(verify) as c:
|
||||
@@ -107,10 +129,14 @@ def check_jwt_login(base_url: str, username: str, password: str, verify: bool) -
|
||||
except Exception as e:
|
||||
fail(f"provider='{provider}' → ошибка: {e}")
|
||||
return None
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtLogin
|
||||
|
||||
|
||||
# ── 3. JWT endpoints ─────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtEndpoints [C:2] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Verify JWT bearer access to core API endpoints.
|
||||
def check_jwt_endpoints(base_url: str, token: str, verify: bool):
|
||||
banner("3. JWT — доступ к API endpoints")
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
@@ -122,10 +148,14 @@ def check_jwt_endpoints(base_url: str, token: str, verify: bool):
|
||||
else fail(f"GET {path} → HTTP {r.status_code}")
|
||||
except Exception as e:
|
||||
fail(f"GET {path} → ошибка: {e}")
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtEndpoints
|
||||
|
||||
|
||||
# ── 3b. CSRF token + session cookie ─────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckCsrfSession [C:4] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Fetch CSRF token and probe anonymous/authenticated Set-Cookie session behavior.
|
||||
def check_csrf_session(base_url: str, token: str, verify: bool) -> tuple[str | None, dict | None]:
|
||||
"""
|
||||
GET /api/v1/security/csrf_token/ с JWT.
|
||||
@@ -173,10 +203,14 @@ def check_csrf_session(base_url: str, token: str, verify: bool) -> tuple[str | N
|
||||
warn(f"GET / ошибка: {e}")
|
||||
|
||||
return csrf_token, csrf_session_cookie
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckCsrfSession
|
||||
|
||||
|
||||
# ── 4. Guest Token ───────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckGuestToken [C:4] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Try guest-token issuance via three strategies: bare JWT, CSRF header, JWT+CSRF+session.
|
||||
def check_guest_token(base_url: str, token: str,
|
||||
csrf_token: str | None, csrf_session: dict | None,
|
||||
verify: bool, dashboard_ids: list[str] | None) -> list[str] | None:
|
||||
@@ -256,10 +290,14 @@ def check_guest_token(base_url: str, token: str,
|
||||
warn(f"dashboard #{did}: пропускаем JWT+CSRF+sess — нет csrf_token или csrf_session")
|
||||
|
||||
return dashboard_ids
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckGuestToken
|
||||
|
||||
|
||||
# ── 5. Form Login ────────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckFormLogin [C:3] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Attempt classic form login and capture the raw Set-Cookie header.
|
||||
def check_form_login(base_url: str, username: str, password: str, verify: bool) -> str | None:
|
||||
"""POST /login/, возвращает сырой Set-Cookie или None."""
|
||||
banner("5. Form Login (POST /login/)")
|
||||
@@ -301,10 +339,14 @@ def check_form_login(base_url: str, username: str, password: str, verify: bool)
|
||||
except Exception as e:
|
||||
fail(f"POST /login/ ошибка: {e}")
|
||||
return None
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckFormLogin
|
||||
|
||||
|
||||
# ── 6. Session Cookie → Dashboard ────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckSessionToDashboard [C:3] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Full-cycle check: session cookie against main page and dashboard load.
|
||||
def check_session_to_dashboard(base_url: str, set_cookie_header: str | None,
|
||||
verify: bool, dashboard_ids: list[str] | None):
|
||||
banner("6. Session Cookie → Dashboard (ПОЛНЫЙ ЦИКЛ)")
|
||||
@@ -358,10 +400,14 @@ def check_session_to_dashboard(base_url: str, set_cookie_header: str | None,
|
||||
warn(f"dashboard #{did} → HTTP {r.status_code}")
|
||||
except Exception as e:
|
||||
fail(f"dashboard #{did} → ошибка: {e}")
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckSessionToDashboard
|
||||
|
||||
|
||||
# ── 7. JWT → UI dashboard ────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.CheckJwtUi [C:2] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Check whether JWT bearer alone can load the dashboard UI.
|
||||
def check_jwt_ui(base_url: str, token: str, verify: bool, dashboard_ids: list[str] | None):
|
||||
banner("7. JWT Bearer → Dashboard UI (без session cookie)")
|
||||
if not dashboard_ids:
|
||||
@@ -381,10 +427,14 @@ def check_jwt_ui(base_url: str, token: str, verify: bool, dashboard_ids: list[st
|
||||
warn(f"dashboard #{did} → HTTP {r.status_code}")
|
||||
except Exception as e:
|
||||
warn(f"dashboard #{did} → ошибка: {e}")
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.CheckJwtUi
|
||||
|
||||
|
||||
# ── MAIN ──────────────────────────────────────────────────────
|
||||
|
||||
# #region Plugin.LlmAnalysis.AuthDiag.Main [C:4] [TYPE Function]
|
||||
# @ingroup LlmAnalysis
|
||||
# @BRIEF Orchestrate all auth-path checks and print the summary verdict.
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Superset Auth Diagnostics (Extended v3)")
|
||||
parser.add_argument("--url", required=True, help="Superset base URL")
|
||||
@@ -466,3 +516,5 @@ def main():
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag.Main
|
||||
# #endregion Plugin.LlmAnalysis.AuthDiag
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ from ..models.maintenance import MaintenanceSettings
|
||||
from ..services.maintenance import end_all_maintenance, end_maintenance, start_maintenance
|
||||
|
||||
|
||||
# #region Plugin.MaintenanceBanner.MaintenanceBannerPlugin [C:4] [TYPE Class]
|
||||
# #region Plugin.MaintenanceBanner.MaintenanceBannerPlugin.Class [C:4] [TYPE Class]
|
||||
# @defgroup Plugin Module group.
|
||||
# @BRIEF Plugin for executing maintenance banner operations asynchronously via TaskManager.
|
||||
# @RELATION CALLS -> [start_maintenance]
|
||||
@@ -159,7 +159,7 @@ class MaintenanceBannerPlugin(PluginBase):
|
||||
finally:
|
||||
db.close()
|
||||
# #endregion Plugin.MaintenanceBanner.Execute
|
||||
# #endregion Plugin.MaintenanceBanner.MaintenanceBannerPlugin
|
||||
# #endregion Plugin.MaintenanceBanner.MaintenanceBannerPlugin.Class
|
||||
|
||||
|
||||
# #region Plugin.MaintenanceBanner.ReportProgress [C:2] [TYPE Function]
|
||||
|
||||
@@ -23,7 +23,7 @@ from ...models.storage import FileCategory, StoredFile
|
||||
from ...services.storage_service import StorageService
|
||||
|
||||
|
||||
# #region Plugin.StoragePlugin [TYPE Class]
|
||||
# #region Plugin.StoragePlugin.Class [TYPE Class]
|
||||
# @defgroup Storage Module group.
|
||||
# @BRIEF Implementation of the storage management plugin — thin REST adapter over StorageService.
|
||||
class StoragePlugin(PluginBase):
|
||||
@@ -274,5 +274,5 @@ class StoragePlugin(PluginBase):
|
||||
return fp
|
||||
# endregion Plugin.StoragePlugin.GetFilePath
|
||||
|
||||
# #endregion Plugin.StoragePlugin
|
||||
# #endregion Plugin.StoragePlugin.Class
|
||||
# #endregion Plugin.StoragePlugin
|
||||
|
||||
@@ -34,7 +34,7 @@ from .dictionary import DictionaryManager
|
||||
from .events import TranslationEventLog
|
||||
|
||||
|
||||
# #region Plugin.BatchProc.BatchProcessingService [C:4] [TYPE Class]
|
||||
# #region Plugin.BatchProc.BatchProcessingService.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Create batch records, classify rows, process LLM calls, persist results.
|
||||
class BatchProcessingService:
|
||||
@@ -411,5 +411,5 @@ class BatchProcessingService:
|
||||
async def insert_batch_to_target(self, job: TranslationJob, batch_id: str, run_id: str) -> None:
|
||||
await insert_batch_to_target(self.db, self.config_manager, job, batch_id, run_id)
|
||||
# #endregion Plugin.BatchProc.InsertBatchToTarget
|
||||
# #endregion Plugin.BatchProc.BatchProcessingService
|
||||
# #endregion Plugin.BatchProc.BatchProcessingService.Class
|
||||
# #endregion Plugin.BatchProc.BatchProcessingService
|
||||
|
||||
@@ -29,7 +29,7 @@ from ._token_budget import (
|
||||
from ._utils import estimate_row_tokens
|
||||
|
||||
|
||||
# #region Plugin.BatchSizer.AdaptiveBatchSizer [C:3] [TYPE Class]
|
||||
# #region Plugin.BatchSizer.AdaptiveBatchSizer.Class [C:3] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Split source rows into auto-sized batches based on token budget estimates.
|
||||
class AdaptiveBatchSizer:
|
||||
@@ -226,5 +226,5 @@ class AdaptiveBatchSizer:
|
||||
|
||||
return batches
|
||||
# #endregion Plugin.BatchSizer.AutoSizeBatches
|
||||
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer
|
||||
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer.Class
|
||||
# #endregion Plugin.BatchSizer.AdaptiveBatchSizer
|
||||
|
||||
@@ -38,7 +38,7 @@ from .prompt_builder import ContextAwarePromptBuilder
|
||||
MAX_RETRIES_PER_BATCH = 3
|
||||
|
||||
|
||||
# #region Plugin.LlmCall.LLMTranslationService [C:4] [TYPE Class]
|
||||
# #region Plugin.LlmCall.LLMTranslationService.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Call LLM, handle retry/truncation, parse response, persist records.
|
||||
class LLMTranslationService:
|
||||
@@ -868,5 +868,5 @@ class LLMTranslationService:
|
||||
def _parse_llm_response(*a, **kw):
|
||||
return parse_llm_response(*a, **kw)
|
||||
# #endregion Plugin.LlmCall.ParseLlmResponse
|
||||
# #endregion Plugin.LlmCall.LLMTranslationService
|
||||
# #endregion Plugin.LlmCall.LLMTranslationService.Class
|
||||
# #endregion Plugin.LlmCall.LLMTranslationService
|
||||
|
||||
@@ -23,7 +23,7 @@ from ...models.translate import TranslationJob, TranslationRecord, TranslationRu
|
||||
from ._run_source import _extract_chart_data_rows, fetch_source_rows
|
||||
|
||||
|
||||
# #region Plugin.RunService.RunExecutionService [C:4] [TYPE Class]
|
||||
# #region Plugin.RunService.RunExecutionService.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Orchestrate full translation run: fetch data, process batches, finalize.
|
||||
class RunExecutionService:
|
||||
@@ -187,5 +187,5 @@ class RunExecutionService:
|
||||
lang_stat.token_count = total_tokens // num_langs
|
||||
lang_stat.estimated_cost = round((lang_stat.token_count / 1000) * cost_per_token, 6)
|
||||
self.db.flush()
|
||||
# #endregion Plugin.RunService.RunExecutionService
|
||||
# #endregion Plugin.RunService.RunExecutionService.Class
|
||||
# #endregion Plugin.RunService.RunExecutionService
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
# CJK ratio 1.5 — too optimistic for Qwen/DeepSeek tokenizers (actual ~1.0-1.2).
|
||||
# Input-only batch sizing — output budget is the primary truncation cause (finish_reason=length).
|
||||
|
||||
# #endregion Plugin.TokenBudget.EstimateTokenBudget
|
||||
# #region Plugin.TokenBudget.DEFAULTCONTEXTWINDOW [TYPE Constant]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Conservative default when provider has no context_window. Local LM Studio often
|
||||
|
||||
@@ -41,7 +41,7 @@ VALID_EVENT_TYPES = TERMINAL_EVENT_TYPES | {
|
||||
DEFAULT_RETENTION_DAYS = 90
|
||||
|
||||
|
||||
# #region Plugin.Events.TranslationEventLog [C:5] [TYPE Class]
|
||||
# #region Plugin.Events.TranslationEventLog.Class [C:5] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Structured event logging for translation operations with terminal event invariant enforcement.
|
||||
# @PRE Database session is available.
|
||||
@@ -275,5 +275,5 @@ class TranslationEventLog:
|
||||
# endregion Plugin.Events.GetRunEventSummary
|
||||
|
||||
|
||||
# #endregion Plugin.Events.TranslationEventLog
|
||||
# #endregion Plugin.Events.TranslationEventLog.Class
|
||||
# #endregion Plugin.Events.TranslationEventLog
|
||||
|
||||
@@ -47,7 +47,7 @@ __all__ = [
|
||||
]
|
||||
|
||||
# #endregion Plugin.Executor.TranslationExecutor
|
||||
# #region Plugin.Executor.TranslationExecutor [C:4] [TYPE Class] [SEMANTICS translate,executor,orchestrator]
|
||||
# #region Plugin.Executor.TranslationExecutor.Class [C:4] [TYPE Class] [SEMANTICS translate,executor,orchestrator]
|
||||
# @ingroup TranslationExecutor
|
||||
# @BRIEF Thin orchestrator for translation execution. Delegates heavy logic to sub-services for INV_7 compliance.
|
||||
# @PRE DB session and ConfigManager provided.
|
||||
@@ -570,7 +570,6 @@ class TranslationExecutor:
|
||||
# @RATIONALE The previous elif s == 0 unconditionally marked any zero-success run as FAILED, even when all work was legitimately skipped (new_key_only scheduled runs with 8 skipped rows). This exactly matched the prod symptom "status: FAILED, successful:0, failed:0, skipped:8".
|
||||
# Separately, f>0 with s>0 (e.g. LLM down but some cache hits) was silently COMPLETED — that hides LLM unavailability from operators.
|
||||
# @REJECTED Treating sk>0 + s==0 as failure — it would turn healthy "nothing to do" scheduled runs into red alerts and pollute history.
|
||||
# @TEST_INVARIANT Pure-skip scheduled translation must end COMPLETED (see Test.Executor.FinalizeRun).
|
||||
# @RELATION BINDS_TO -> [Test.Executor.FinalizeRun]
|
||||
@staticmethod
|
||||
def _finalize_run(run: TranslationRun, s: int, f: int, sk: int) -> TranslationRun:
|
||||
@@ -778,4 +777,4 @@ class TranslationExecutor:
|
||||
from ._llm_call import LLMTranslationService
|
||||
return LLMTranslationService._parse_llm_response(response_text, expected_count, target_languages, finish_reason)
|
||||
# #endregion TranslationExecutor._parse_llm_response
|
||||
# #endregion Plugin.Executor.TranslationExecutor
|
||||
# #endregion Plugin.Executor.TranslationExecutor.Class
|
||||
|
||||
@@ -29,7 +29,7 @@ def _optional_int(value: Any) -> int | None:
|
||||
# #endregion Plugin.Metrics.optional_int
|
||||
|
||||
|
||||
# #region Plugin.Metrics.TranslationMetrics [C:3] [TYPE Class]
|
||||
# #region Plugin.Metrics.TranslationMetrics.Class [C:3] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Aggregate translation metrics from live events and MetricSnapshot.
|
||||
class TranslationMetrics:
|
||||
@@ -37,7 +37,7 @@ class TranslationMetrics:
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
|
||||
#region Plugin.Metrics.GetJobMetrics [C:3] [TYPE Function]
|
||||
# #region Plugin.Metrics.GetJobMetrics [C:3] [TYPE Function]
|
||||
# @BRIEF Get aggregated metrics for a specific job.
|
||||
# @PRE job_id exists.
|
||||
# @POST Returns dict with metrics from events + latest snapshot.
|
||||
@@ -205,9 +205,9 @@ class TranslationMetrics:
|
||||
"next_scheduled_run": next_schedule.last_run_at.isoformat() if next_schedule and next_schedule.last_run_at else None,
|
||||
"per_language_metrics": per_language,
|
||||
}
|
||||
# endregion Plugin.Metrics.GetJobMetrics
|
||||
# #endregion Plugin.Metrics.GetJobMetrics
|
||||
|
||||
# region Plugin.Metrics.GetAllMetrics [C:2] [TYPE Function]
|
||||
# #region Plugin.Metrics.GetAllMetrics [C:2] [TYPE Function]
|
||||
# @BRIEF Get aggregated metrics for all jobs by iterating get_job_metrics.
|
||||
# @POST Returns list of per-job metrics.
|
||||
def get_all_metrics(self) -> list[dict[str, Any]]:
|
||||
@@ -218,8 +218,8 @@ class TranslationMetrics:
|
||||
.all()
|
||||
)
|
||||
return [self.get_job_metrics(jid[0]) for jid in job_ids if jid[0]]
|
||||
# endregion Plugin.Metrics.GetAllMetrics
|
||||
# #endregion Plugin.Metrics.GetAllMetrics
|
||||
|
||||
|
||||
# #endregion Plugin.Metrics.TranslationMetrics
|
||||
# #endregion Plugin.Metrics.TranslationMetrics.Class
|
||||
# #endregion Plugin.Metrics.TranslationMetrics
|
||||
|
||||
@@ -37,7 +37,7 @@ from .orchestrator_runner import TranslationStageRunner
|
||||
|
||||
|
||||
# #endregion Plugin.Orchestrator.TranslationOrchestrator
|
||||
# #region Plugin.Orchestrator.TranslationOrchestrator [C:5] [TYPE Class]
|
||||
# #region Plugin.Orchestrator.TranslationOrchestrator.Class [C:5] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Coordinates full translation run lifecycle: validation, execution, SQL generation, Superset submission, event logging.
|
||||
# @PRE DB session and config manager are available.
|
||||
@@ -266,4 +266,4 @@ class TranslationOrchestrator:
|
||||
return self._aggregator.get_run_history(job_id, page, page_size)
|
||||
# endregion Plugin.Orchestrator.GetRunHistory
|
||||
|
||||
# #endregion Plugin.Orchestrator.TranslationOrchestrator
|
||||
# #endregion Plugin.Orchestrator.TranslationOrchestrator.Class
|
||||
|
||||
@@ -30,7 +30,7 @@ from .orchestrator_cancel import cancel_run as _cancel_run, retry_insert as _ret
|
||||
|
||||
|
||||
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager
|
||||
# #region Plugin.OrchestratorRetry.TranslationRunRetryManager [C:4] [TYPE Class] [SEMANTICS translate,retry,manager]
|
||||
# #region Plugin.OrchestratorRetry.TranslationRunRetryManager.Class [C:4] [TYPE Class] [SEMANTICS translate,retry,manager]
|
||||
# @ingroup Translate
|
||||
# @BRIEF Manages retry of failed batches and delegates cancel/insert.
|
||||
# @PRE db, config, event_log provided.
|
||||
@@ -189,4 +189,4 @@ class TranslationRunRetryManager:
|
||||
return _cancel_run(self.db, self.event_log, self.current_user, run_id)
|
||||
# #endregion TranslationRunRetryManager.cancel_run
|
||||
|
||||
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager
|
||||
# #endregion Plugin.OrchestratorRetry.TranslationRunRetryManager.Class
|
||||
|
||||
@@ -81,11 +81,6 @@ INSERT_RESULT_SKIPPED = "skipped"
|
||||
# PostgreSQL conflict_target, leading to "ON CONFLICT DO UPDATE cannot affect row a second time"
|
||||
# errors at the DB level. The safe approach is to fail fast and require target_language_column
|
||||
# to be included in target_key_cols for MERGE multi-language configurations.
|
||||
# @TEST_INVARIANT dedup_merge_key_matches_conflict_target -> VERIFIED_BY: [
|
||||
# test_merge_dedup_preserves_language_rows,
|
||||
# test_merge_dedup_collapses_single_lang,
|
||||
# test_merge_precondition_fails_without_lang_in_key
|
||||
# ]
|
||||
def dedup_rows_for_merge(
|
||||
rows: list[dict[str, object]],
|
||||
key_cols: list[str] | None,
|
||||
|
||||
@@ -43,7 +43,7 @@ from .preview_response_parser import (
|
||||
from .preview_review import PreviewSessionManager
|
||||
|
||||
|
||||
# #region Plugin.Preview.TranslationPreview [C:4] [TYPE Class]
|
||||
# #region Plugin.Preview.TranslationPreview.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Manages preview lifecycle: fetch sample rows, call LLM, manage row-level approve/edit/reject, accept gate.
|
||||
# @PRE Database session and config manager are available.
|
||||
@@ -292,7 +292,7 @@ class TranslationPreview:
|
||||
def get_preview_session(self, job_id: str) -> dict[str, Any]:
|
||||
return self._session_mgr.get_preview_session(job_id)
|
||||
|
||||
# #endregion Plugin.Preview.TranslationPreview
|
||||
# #endregion Plugin.Preview.TranslationPreview.Class
|
||||
|
||||
# Re-export for backward compatibility
|
||||
# #endregion Plugin.Preview.TranslationPreview
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
# #region Plugin.PromptBuilder.ContextAwarePromptBuilder [C:2] [TYPE Class]
|
||||
# #region Plugin.PromptBuilder.ContextAwarePromptBuilder.Class [C:2] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Build LLM prompts with context-aware dictionary entries and similarity-based priority.
|
||||
|
||||
@@ -149,7 +149,7 @@ class ContextAwarePromptBuilder:
|
||||
ContextAwarePromptBuilder.render_entry(entry, priority, row_context)
|
||||
for entry, priority in results
|
||||
]
|
||||
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder
|
||||
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder.Class
|
||||
|
||||
|
||||
# #endregion Plugin.PromptBuilder.ContextAwarePromptBuilder
|
||||
|
||||
@@ -76,7 +76,7 @@ def _ensure_aware(dt: datetime | None) -> datetime | None:
|
||||
# #endregion Plugin.Scheduler.EnsureAware
|
||||
|
||||
|
||||
# #region Plugin.Scheduler.TranslationScheduler [C:4] [TYPE Class]
|
||||
# #region Plugin.Scheduler.TranslationScheduler.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF CRUD for TranslationSchedule rows + APScheduler registration wrappers.
|
||||
class TranslationScheduler:
|
||||
@@ -289,7 +289,7 @@ class TranslationScheduler:
|
||||
# endregion Plugin.Scheduler.GetNextExecutions
|
||||
|
||||
|
||||
# #endregion Plugin.Scheduler.TranslationScheduler
|
||||
# #endregion Plugin.Scheduler.TranslationScheduler.Class
|
||||
|
||||
|
||||
# #region Plugin.Scheduler.ExecuteScheduledTranslation [C:4] [TYPE Function]
|
||||
|
||||
@@ -26,7 +26,7 @@ from .service_datasource import fetch_datasource_metadata
|
||||
from .service_utils import _extract_dialect
|
||||
|
||||
|
||||
# #region Plugin.Service.TranslateJobService [TYPE Class]
|
||||
# #region Plugin.Service.TranslateJobService.Class [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Service for translation job CRUD with validation and Superset integration.
|
||||
class TranslateJobService:
|
||||
@@ -313,7 +313,7 @@ class TranslateJobService:
|
||||
})
|
||||
return result
|
||||
# endregion Plugin.Service.FetchAvailableDatasources
|
||||
# #endregion Plugin.Service.TranslateJobService
|
||||
# #endregion Plugin.Service.TranslateJobService.Class
|
||||
|
||||
|
||||
# Re-exports for backward compatibility
|
||||
|
||||
@@ -249,7 +249,7 @@ def generate_upsert_sql(
|
||||
# #endregion Plugin.SqlGenerator.GenerateUpsertSql
|
||||
|
||||
|
||||
# #region Plugin.SqlGenerator.SQLGenerator [C:3] [TYPE Class]
|
||||
# #region Plugin.SqlGenerator.SQLGenerator.Class [C:3] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Generate safe, dialect-appropriate SQL INSERT/UPSERT statements.
|
||||
# @PRE Job has target_schema, target_table, key columns configured.
|
||||
@@ -427,5 +427,5 @@ class SQLGenerator:
|
||||
# endregion SQLGenerator.generate_batch
|
||||
|
||||
|
||||
# #endregion Plugin.SqlGenerator.SQLGenerator
|
||||
# #endregion Plugin.SqlGenerator.SQLGenerator.Class
|
||||
# #endregion Plugin.SqlGenerator.SQLGenerator
|
||||
|
||||
@@ -21,7 +21,7 @@ from ...core.superset_client import SupersetClient
|
||||
from ...core.utils.client_registry import get_superset_client
|
||||
|
||||
|
||||
# #region Plugin.SupersetExecutor.SupersetSqlLabExecutor [C:4] [TYPE Class]
|
||||
# #region Plugin.SupersetExecutor.SupersetSqlLabExecutor.Class [C:4] [TYPE Class]
|
||||
# @defgroup Translate Module group.
|
||||
# @BRIEF Submit SQL to Superset SQL Lab API with polling and status tracking.
|
||||
# @PRE Valid environment ID and ConfigManager.
|
||||
@@ -436,5 +436,5 @@ class SupersetSqlLabExecutor:
|
||||
# endregion Plugin.SupersetExecutor.GetQueryResults
|
||||
|
||||
|
||||
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor
|
||||
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor.Class
|
||||
# #endregion Plugin.SupersetExecutor.SupersetSqlLabExecutor
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas [C:5] [TYPE Module] [SEMANTICS baseline,dashboard-testing,dto,schema]
|
||||
# #region DashboardTesting.Schemas [C:5] [TYPE Module] [SEMANTICS baseline,dashboard-testing,dto,schema]
|
||||
# @defgroup DashboardTesting Pydantic DTOs for Superset-native dashboard query inspection, filter normalization, execution, comparison, and baseline lifecycle.
|
||||
# @LAYER DTO
|
||||
# @RELATION DEPENDS_ON -> [SupersetBaselineEngine.DataModel]
|
||||
@@ -146,4 +146,4 @@ from .verification import (
|
||||
VerificationRunRequest as VerificationRunRequest,
|
||||
)
|
||||
|
||||
#endregion DashboardTesting.Schemas
|
||||
# #endregion DashboardTesting.Schemas
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Candidates [C:3] [TYPE Module] [SEMANTICS baseline,candidates,dto]
|
||||
# #region DashboardTesting.Schemas.Candidates [C:3] [TYPE Module] [SEMANTICS baseline,candidates,dto]
|
||||
# @defgroup DashboardTesting.Candidates Candidate request, response, and approval gate schemas.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -269,5 +269,5 @@ class ApprovalConsumeResponse(BaseModel):
|
||||
release_commit_hash: str = Field(description="40-char git commit hash")
|
||||
# #endregion DashboardTesting.Schemas.ApprovalConsumeResponse
|
||||
|
||||
#endregion DashboardTesting.Schemas.Candidates
|
||||
# #endregion DashboardTesting.Schemas.Candidates
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Capture [C:2] [TYPE Module] [SEMANTICS baseline,capture,dto]
|
||||
# #region DashboardTesting.Schemas.Capture [C:2] [TYPE Module] [SEMANTICS baseline,capture,dto]
|
||||
# @defgroup DashboardTesting.Capture Server-side capture request schema for metric candidates.
|
||||
# @LAYER DTO
|
||||
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Candidates]
|
||||
@@ -69,4 +69,4 @@ class CaptureCandidateResponse(BaseModel):
|
||||
source_response_hash: str = Field(description="SHA-256 of the raw Superset response bytes")
|
||||
# #endregion DashboardTesting.Schemas.CaptureCandidateResponse
|
||||
|
||||
#endregion DashboardTesting.Schemas.Capture
|
||||
# #endregion DashboardTesting.Schemas.Capture
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Catalog [C:3] [TYPE Module] [SEMANTICS baseline,catalog,dto]
|
||||
# #region DashboardTesting.Schemas.Catalog [C:3] [TYPE Module] [SEMANTICS baseline,catalog,dto]
|
||||
# @defgroup DashboardTesting.Catalog Baseline entry, immutability block, and catalog container schemas.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -175,4 +175,4 @@ class BaselineCatalog(BaseModel):
|
||||
warnings: list[Warning] = Field(default_factory=list)
|
||||
# #endregion DashboardTesting.Schemas.BaselineCatalog
|
||||
|
||||
#endregion DashboardTesting.Schemas.Catalog
|
||||
# #endregion DashboardTesting.Schemas.Catalog
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Common [C:1] [TYPE Module] [SEMANTICS baseline,common,dto]
|
||||
# #region DashboardTesting.Schemas.Common [C:1] [TYPE Module] [SEMANTICS baseline,common,dto]
|
||||
# @defgroup DashboardTesting.Common Shared DTOs — Warning, Provenance.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -38,4 +38,4 @@ class ApprovalInfo(BaseModel):
|
||||
at: datetime = Field(description="Timestamp of approval")
|
||||
# #endregion DashboardTesting.Schemas.ApprovalInfo
|
||||
|
||||
#endregion DashboardTesting.Schemas.Common
|
||||
# #endregion DashboardTesting.Schemas.Common
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Enums [C:1] [TYPE Module] [SEMANTICS baseline,enum]
|
||||
# #region DashboardTesting.Schemas.Enums [C:1] [TYPE Module] [SEMANTICS baseline,enum]
|
||||
# @defgroup DashboardTesting.Enums Core enums for dashboard testing schemas.
|
||||
# @LAYER DTO
|
||||
# @INVARIANT All enums are StrEnum for JSON serialization compatibility.
|
||||
@@ -112,4 +112,4 @@ class DiffKind(StrEnum):
|
||||
TIME_GRAIN_CHANGED = "time_grain_changed"
|
||||
# #endregion DashboardTesting.Schemas.DiffKind
|
||||
|
||||
#endregion DashboardTesting.Schemas.Enums
|
||||
# #endregion DashboardTesting.Schemas.Enums
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Execution [C:1] [TYPE Module] [SEMANTICS baseline,execution,dto]
|
||||
# #region DashboardTesting.Schemas.Execution [C:1] [TYPE Module] [SEMANTICS baseline,execution,dto]
|
||||
# @defgroup DashboardTesting.Execution Query execution request schema.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -26,4 +26,4 @@ class ExecuteQueryRequest(BaseModel):
|
||||
max_rows: int = Field(default=10000, le=10000, description="Bounded result limit")
|
||||
# #endregion DashboardTesting.Schemas.ExecuteQueryRequest
|
||||
|
||||
#endregion DashboardTesting.Schemas.Execution
|
||||
# #endregion DashboardTesting.Schemas.Execution
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Filters [C:2] [TYPE Module] [SEMANTICS baseline,filter,dto]
|
||||
# #region DashboardTesting.Schemas.Filters [C:2] [TYPE Module] [SEMANTICS baseline,filter,dto]
|
||||
# @defgroup DashboardTesting.Filters Filter normalization schemas — FilterValue, NormalizedFilter, NormalizedFilterContext.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -56,4 +56,4 @@ class NormalizeFiltersRequest(BaseModel):
|
||||
query_model_fingerprint: str | None = None
|
||||
# #endregion DashboardTesting.Schemas.NormalizeFiltersRequest
|
||||
|
||||
#endregion DashboardTesting.Schemas.Filters
|
||||
# #endregion DashboardTesting.Schemas.Filters
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Inheritance [C:3] [TYPE Module] [SEMANTICS baseline,inheritance,dto,plan]
|
||||
# #region DashboardTesting.Schemas.Inheritance [C:3] [TYPE Module] [SEMANTICS baseline,inheritance,dto,plan]
|
||||
# @defgroup DashboardTesting.Inheritance Request/response DTOs for baseline inheritance planning and execution.
|
||||
# @LAYER DTO
|
||||
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Catalog]
|
||||
@@ -99,4 +99,4 @@ class InheritancePlan(BaseModel):
|
||||
new_entries: list[dict[str, Any]] = Field(default_factory=list, description="New entries needing fresh capture")
|
||||
# #endregion DashboardTesting.Schemas.InheritancePlan
|
||||
|
||||
#endregion DashboardTesting.Schemas.Inheritance
|
||||
# #endregion DashboardTesting.Schemas.Inheritance
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.QueryModel [C:3] [TYPE Module] [SEMANTICS baseline,query-model,dto]
|
||||
# #region DashboardTesting.Schemas.QueryModel [C:3] [TYPE Module] [SEMANTICS baseline,query-model,dto]
|
||||
# @defgroup DashboardTesting.QueryModel Structured chart/dashboard/filter metadata extracted from Superset.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -134,4 +134,4 @@ class DashboardQueryModel(BaseModel):
|
||||
query_model_fingerprint: str = Field(description="Deterministic hash of the query model structure")
|
||||
# #endregion DashboardTesting.Schemas.DashboardQueryModel
|
||||
|
||||
#endregion DashboardTesting.Schemas.QueryModel
|
||||
# #endregion DashboardTesting.Schemas.QueryModel
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.Results [C:2] [TYPE Module] [SEMANTICS baseline,result,comparison,dto]
|
||||
# #region DashboardTesting.Schemas.Results [C:2] [TYPE Module] [SEMANTICS baseline,result,comparison,dto]
|
||||
# @defgroup DashboardTesting.Results Normalized values, comparison policies, and comparison results.
|
||||
# @LAYER DTO
|
||||
|
||||
@@ -100,7 +100,7 @@ class ComparisonResult(BaseModel):
|
||||
evidence_refs: list[str] = Field(default_factory=list, description="036 evidence artifact references")
|
||||
# #endregion DashboardTesting.Schemas.ComparisonResult
|
||||
|
||||
#endregion DashboardTesting.Schemas.Results
|
||||
# #endregion DashboardTesting.Schemas.Results
|
||||
|
||||
# Resolve forward reference for ComparisonPolicy.per_column
|
||||
ComparisonPolicy.model_rebuild()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region ScenarioRegistry.Schemas [C:2] [TYPE Module] [SEMANTICS scenario,registry,dto,response,schema]
|
||||
# #region ScenarioRegistry.Schemas [C:2] [TYPE Module] [SEMANTICS scenario,registry,dto,response,schema]
|
||||
# @defgroup ScenarioRegistry Pydantic response DTOs for the Scenario Registry (042).
|
||||
# @LAYER DTO
|
||||
# @RELATION DEPENDS_ON -> [ScenarioRegistry.DataModel]
|
||||
@@ -218,4 +218,4 @@ class ScenarioEditorProposalSaveRequest(BaseModel):
|
||||
agent_action_id: str | None = None
|
||||
# #endregion ScenarioEditor.Schemas.ProposalSaveRequest
|
||||
|
||||
#endregion ScenarioRegistry.Schemas
|
||||
# #endregion ScenarioRegistry.Schemas
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#region DashboardTesting.Schemas.StructureDiff [C:2] [TYPE Module] [SEMANTICS baseline,structure-diff,dto]
|
||||
# #region DashboardTesting.Schemas.StructureDiff [C:2] [TYPE Module] [SEMANTICS baseline,structure-diff,dto]
|
||||
# @defgroup DashboardTesting.StructureDiff Dashboard structure diff schemas — request + response.
|
||||
# @LAYER DTO
|
||||
# @RELATION DEPENDS_ON -> [DashboardTesting.Schemas.Enums]
|
||||
@@ -53,7 +53,7 @@ class StructureChange(BaseModel):
|
||||
# #endregion DashboardTesting.Schemas.StructureChange
|
||||
|
||||
|
||||
# #region DashboardTesting.Schemas.StructureDiff [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,release]
|
||||
# #region DashboardTesting.Schemas.StructureDiff.Class [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,release]
|
||||
class StructureDiff(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@@ -64,7 +64,7 @@ class StructureDiff(BaseModel):
|
||||
changes: list[StructureChange] = Field(default_factory=list)
|
||||
summary: dict[str, int] = Field(default_factory=dict, description="Counts by severity (critical, warning, info, pass)")
|
||||
blocked: bool = Field(default=False, description="True if critical changes block progression")
|
||||
# #endregion DashboardTesting.Schemas.StructureDiff
|
||||
# #endregion DashboardTesting.Schemas.StructureDiff.Class
|
||||
|
||||
|
||||
# #region DashboardTesting.Schemas.SnapshotCaptureResponse [C:3] [TYPE Class] [SEMANTICS baseline,structure-diff,capture,response,provenance]
|
||||
@@ -87,4 +87,4 @@ class SnapshotCaptureResponse(BaseModel):
|
||||
warnings: int = Field(default=0, description="Number of warnings from inspection")
|
||||
# #endregion DashboardTesting.Schemas.SnapshotCaptureResponse
|
||||
|
||||
#endregion DashboardTesting.Schemas.StructureDiff
|
||||
# #endregion DashboardTesting.Schemas.StructureDiff
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user