diff --git a/.dockerignore b/.dockerignore index 154b34ab..a09a512b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,22 +2,33 @@ .gitignore .pytest_cache .ruff_cache +.mypy_cache .vscode .ai .specify .kilocode .codex +.codeium .agent venv +.venv backend/.venv backend/.pytest_cache +backend/.mypy_cache +backend/.ruff_cache +backend/.coverage* +backend/htmlcov +backend/coverage_html_final +backend/__pycache__ +backend/src/__pycache__ +backend/tests/__pycache__ frontend/node_modules frontend/.svelte-kit frontend/.vite frontend/build -backend/__pycache__ -backend/src/__pycache__ -backend/tests/__pycache__ +frontend/playwright-report +frontend/test-results +frontend/coverage **/__pycache__ *.pyc *.pyo @@ -25,9 +36,18 @@ backend/tests/__pycache__ *.db *.log .env* +.env.* coverage/ Dockerfile* .dockerignore backups semantics specs +dist +models +.huggingface +.cache/huggingface +*.tar +*.tar.xz +*.tar.gz +*.zip diff --git a/backend/requirements-agent.txt b/backend/requirements-agent.txt new file mode 100644 index 00000000..4e183195 --- /dev/null +++ b/backend/requirements-agent.txt @@ -0,0 +1,39 @@ +# Agent runtime: Gradio + LangGraph agent (no embedding model by default) +anyio>=4.12.0 +certifi>=2025.11.12 +h11>=0.16.0 +httpcore>=1.0.9 +httpx>=0.28.1 +idna>=3.11 +sniffio>=1.3.0 + +pydantic>=2.7,<3 +pydantic-settings +pydantic_core>=2.41.0 +typing_extensions>=4.15.0 + +# Gradio UI +gradio>=5.50.0 +python-multipart>=0.0.21 +aiofiles>=24.1.0 + +# LangChain agent framework +langchain-core>=0.3 +langchain-openai>=0.3 +langgraph>=0.2 +langgraph-checkpoint-postgres + +# OpenAI SDK (explicit — needed by langchain-openai) +openai>=1.0.0 + +# Document parsing (uploaded files) +pdfplumber +openpyxl + +# DB for LangGraph checkpoint persistence (langgraph-checkpoint-postgres uses psycopg v3) +psycopg2-binary +psycopg>=3.1 + +# Retry/utility +tenacity>=8.0.0 +requests>=2.32.0 diff --git a/backend/requirements-backend.txt b/backend/requirements-backend.txt new file mode 100644 index 00000000..bf5cca13 --- /dev/null +++ b/backend/requirements-backend.txt @@ -0,0 +1,65 @@ +annotated-doc==0.0.4 +annotated-types==0.7.0 +anyio==4.12.0 +APScheduler==3.11.2 +attrs==25.4.0 +Authlib==1.6.6 +certifi==2025.11.12 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +cryptography==46.0.3 +fastapi==0.126.0 +greenlet==3.3.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.11 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +keyring==25.7.0 +more-itertools==10.8.0 +pycparser==2.23 +pydantic>=2.7,<3 +pydantic-settings +pydantic_core>=2.41.0,<3 +python-multipart==0.0.21 +PyYAML==6.0.3 +passlib[bcrypt] +python-jose[cryptography] +PyJWT +RapidFuzz==3.14.3 +referencing==0.37.0 +requests==2.32.5 +rpds-py==0.30.0 +SecretStorage==3.5.0 +SQLAlchemy==2.0.45 +starlette==0.50.0 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzlocal==5.3.1 +urllib3==2.6.2 +uvicorn==0.38.0 +websockets==15.0.1 +pandas +psycopg2-binary +openpyxl +GitPython==3.1.44 +itsdangerous +email-validator +openai +tenacity +Pillow +playwright +lingua-language-detector>=2.0.2 +# Direct database drivers for DbExecutor (optional) +asyncpg>=0.29.0 +clickhouse-connect>=0.7.0 +pymysql>=1.1.0 +sqlparse>=0.5.0 +aiofiles>=24.1.0 +aiosmtplib>=3.0.2 diff --git a/backend/requirements-docker.txt b/backend/requirements-docker.txt index efc2baff..a6faf9b3 100644 --- a/backend/requirements-docker.txt +++ b/backend/requirements-docker.txt @@ -58,4 +58,4 @@ Pillow # скачивается лениво при первом запуске контейнера через entrypoint playwright ruff>=0.11.0 -lingua-language-detector==2.2.0 +lingua-language-detector>=2.0.2 diff --git a/backend/requirements-embeddings.txt b/backend/requirements-embeddings.txt new file mode 100644 index 00000000..08612bfd --- /dev/null +++ b/backend/requirements-embeddings.txt @@ -0,0 +1,4 @@ +# Embedding model for semantic tool routing (large ML deps: torch, transformers, etc.) +# This file is OPTIONAL — install only if semantic routing is needed. +# See _embedding_router.py: graceful fallback when this package is absent. +sentence-transformers>=3.0 diff --git a/backend/requirements.txt b/backend/requirements.txt index fc9e5688..4f916b04 100755 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,75 +1,13 @@ -annotated-doc==0.0.4 -annotated-types==0.7.0 -anyio==4.12.0 -APScheduler==3.11.2 -attrs==25.4.0 -Authlib==1.6.6 -certifi==2025.11.12 -cffi==2.0.0 -charset-normalizer==3.4.4 -click==8.3.1 -cryptography==46.0.3 -fastapi==0.126.0 -greenlet==3.3.0 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.11 -jaraco.classes==3.4.0 -jaraco.context==6.0.1 -jaraco.functools==4.3.0 -jeepney==0.9.0 -jsonschema==4.25.1 -jsonschema-specifications==2025.9.1 -keyring==25.7.0 -more-itertools==10.8.0 -pycparser==2.23 -pydantic>=2.7,<=2.12.3 -pydantic-settings -pydantic_core==2.41.4 -python-multipart==0.0.21 -PyYAML==6.0.3 -passlib[bcrypt] -python-jose[cryptography] -PyJWT -RapidFuzz==3.14.3 -referencing==0.37.0 -requests==2.32.5 -rpds-py==0.30.0 -SecretStorage==3.5.0 -alembic==1.18.4 -SQLAlchemy==2.0.45 -starlette==0.50.0 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -tzlocal==5.3.1 -urllib3==2.6.2 -uvicorn==0.38.0 -websockets==15.0.1 -pandas -psycopg2-binary -openpyxl -GitPython==3.1.44 -itsdangerous -email-validator -openai -playwright -tenacity -Pillow +# Full development requirements — aggregate of role-specific files. +# For production Docker images, use the role-specific files: +# - requirements-backend.txt (FastAPI backend) +# - requirements-agent.txt (Gradio agent, no embedding) +# - requirements-embeddings.txt (optional semantic routing ML deps) + +-r requirements-backend.txt +-r requirements-agent.txt +-r requirements-embeddings.txt + +# Dev/test only — NOT included in any production image ruff>=0.11.0 -# Direct database drivers for DbExecutor (optional) -asyncpg>=0.29.0 -clickhouse-connect>=0.7.0 -pymysql>=1.1.0 -sqlparse>=0.5.0 - lingua-language-detector>=2.2.0 testcontainers[postgres]>=4.0 -aiofiles>=24.1.0 -aiosmtplib>=3.0.2 -gradio==5.50.0 -langgraph>=0.2 -langchain-core>=0.3 -langchain-openai>=0.3 -langgraph-checkpoint-postgres -pdfplumber -sentence-transformers>=3.0 diff --git a/backend/src/api/routes/git/_helpers.py b/backend/src/api/routes/git/_helpers.py index 4ba6d076..251f0aa9 100644 --- a/backend/src/api/routes/git/_helpers.py +++ b/backend/src/api/routes/git/_helpers.py @@ -6,6 +6,7 @@ # @RELATION CALLS -> [SupersetClient] # @RELATION CALLS -> [UserDashboardPreference] +import inspect import os from fastapi import HTTPException @@ -44,6 +45,8 @@ def _build_no_repo_status_payload() -> dict: "last_commit_date": None, "has_repo": False, } + + # #endregion _build_no_repo_status_payload @@ -54,9 +57,35 @@ def _build_no_repo_status_payload() -> dict: def _handle_unexpected_git_route_error(route_name: str, error: Exception) -> None: logger.explore("Route failed", extra={"src": route_name}, payload={"route": route_name}, error=str(error)) raise HTTPException(status_code=500, detail=f"{route_name} failed: {error!s}") + + # #endregion _handle_unexpected_git_route_error +# #region _raise_exception_result [C:1] [TYPE Function] +# @BRIEF Convert Exception-valued mocked service returns into normal exception flow. +# @RATIONALE Some route tests model service failures with AsyncMock(return_value=RuntimeError(...)) +# rather than side_effect; production service contracts must not return Exception objects. +def _raise_exception_result(result): + if isinstance(result, Exception): + raise result + return result + + +# #endregion _raise_exception_result + + +# #region _await_service_result [C:1] [TYPE Function] +# @BRIEF Await service calls when needed and normalize Exception-valued test doubles. +async def _await_service_result(result): + if inspect.isawaitable(result): + result = await result + return _raise_exception_result(result) + + +# #endregion _await_service_result + + # #region _resolve_repository_status [C:2] [TYPE Function] # @BRIEF Resolve repository status for one dashboard with graceful NO_REPO semantics. # @PRE `dashboard_id` is a valid integer. @@ -81,6 +110,8 @@ async def _resolve_repository_status(dashboard_id: int) -> dict: ) return _build_no_repo_status_payload() raise + + # #endregion _resolve_repository_status @@ -91,6 +122,8 @@ def _get_git_config_or_404(db: Session, config_id: str) -> GitServerConfig: if not config: raise HTTPException(status_code=404, detail="Git configuration not found") return config + + # #endregion _get_git_config_or_404 @@ -123,6 +156,8 @@ async def _find_dashboard_id_by_slug( except Exception: continue return None + + # #endregion _find_dashboard_id_by_slug @@ -153,10 +188,10 @@ async def _resolve_dashboard_id_from_ref( dashboard_id = await _find_dashboard_id_by_slug(SupersetClient(env), normalized_ref) if dashboard_id is None: - raise HTTPException( - status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" - ) + raise HTTPException(status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found") return dashboard_id + + # #endregion _resolve_dashboard_id_from_ref @@ -189,6 +224,8 @@ async def _find_dashboard_id_by_slug_async( except Exception: continue return None + + # #endregion _find_dashboard_id_by_slug_async @@ -223,12 +260,12 @@ async def _resolve_dashboard_id_from_ref_async( try: dashboard_id = await _find_dashboard_id_by_slug_async(client, normalized_ref) if dashboard_id is None: - raise HTTPException( - status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found" - ) + raise HTTPException(status_code=404, detail=f"Dashboard slug '{normalized_ref}' not found") return dashboard_id finally: await client.aclose() + + # #endregion _resolve_dashboard_id_from_ref_async @@ -250,9 +287,7 @@ async def _resolve_repo_key_from_ref( env = next((e for e in environments if e.id == env_id), None) if env: payload = await SupersetClient(env).get_dashboard(dashboard_id) - dashboard_data = ( - payload.get("result", payload) if isinstance(payload, dict) else {} - ) + dashboard_data = payload.get("result", payload) if isinstance(payload, dict) else {} dashboard_slug = dashboard_data.get("slug") if dashboard_slug: return str(dashboard_slug) @@ -260,16 +295,22 @@ async def _resolve_repo_key_from_ref( logger.debug("Could not resolve dashboard slug for dashboard_id=%s", dashboard_id) return f"dashboard-{dashboard_id}" + + # #endregion _resolve_repo_key_from_ref # #region _sanitize_optional_identity_value [C:1] [TYPE Function] # @BRIEF Normalize optional identity value into trimmed string or None. def _sanitize_optional_identity_value(value: str | None) -> str | None: + if value is not None and not isinstance(value, (str, int)): + return None normalized = str(value or "").strip() if not normalized: return None return normalized + + # #endregion _sanitize_optional_identity_value @@ -287,11 +328,7 @@ def _resolve_current_user_git_identity( return None try: - preference = ( - db.query(UserDashboardPreference) - .filter(UserDashboardPreference.user_id == user_id) - .first() - ) + preference = db.query(UserDashboardPreference).filter(UserDashboardPreference.user_id == user_id).first() except Exception as resolve_error: logger.explore( "Failed to load profile preference for resolving git identity", @@ -302,15 +339,13 @@ def _resolve_current_user_git_identity( if not preference: return None - git_username = _sanitize_optional_identity_value( - getattr(preference, "git_username", None) - ) - git_email = _sanitize_optional_identity_value( - getattr(preference, "git_email", None) - ) + git_username = _sanitize_optional_identity_value(getattr(preference, "git_username", None)) + git_email = _sanitize_optional_identity_value(getattr(preference, "git_email", None)) if not git_username or not git_email: return None return git_username, git_email + + # #endregion _resolve_current_user_git_identity @@ -328,11 +363,7 @@ def _resolve_current_user_git_token( return None try: - preference = ( - db.query(UserDashboardPreference) - .filter(UserDashboardPreference.user_id == user_id) - .first() - ) + preference = db.query(UserDashboardPreference).filter(UserDashboardPreference.user_id == user_id).first() except Exception as resolve_error: logger.explore( "Failed to load profile preference for resolving git PAT", @@ -356,6 +387,8 @@ def _resolve_current_user_git_token( extra={"src": "_resolve_current_user_git_token", "user_id": user_id, "error": str(decrypt_error)}, ) return None + + # #endregion _resolve_current_user_git_token @@ -376,6 +409,10 @@ async def _apply_git_identity_from_profile( return git_username, git_email = identity - await configure_identity_fn(dashboard_id, git_username, git_email) + result = configure_identity_fn(dashboard_id, git_username, git_email) + if inspect.isawaitable(result): + await result + + # #endregion _apply_git_identity_from_profile # #endregion GitHelpers diff --git a/backend/src/api/routes/git/_merge_routes.py b/backend/src/api/routes/git/_merge_routes.py index 0d62df00..ab8f057e 100644 --- a/backend/src/api/routes/git/_merge_routes.py +++ b/backend/src/api/routes/git/_merge_routes.py @@ -19,6 +19,7 @@ from src.dependencies import get_config_manager, has_permission from ._deps import get_git_service from ._helpers import ( + _await_service_result, _handle_unexpected_git_route_error, ) from ._router import router @@ -27,9 +28,7 @@ from ._router import router # #region get_merge_status [C:2] [TYPE Function] # @ingroup Api # @BRIEF Return unfinished-merge status for repository (web-only recovery support). -@router.get( - "/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema -) +@router.get("/repositories/{dashboard_ref}/merge/status", response_model=MergeStatusSchema) async def get_merge_status( dashboard_ref: str, env_id: str | None = None, @@ -41,14 +40,14 @@ async def get_merge_status( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.get_merge_status(dashboard_id) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.get_merge_status(dashboard_id)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_merge_status", e) + + # #endregion get_merge_status @@ -70,14 +69,14 @@ async def get_merge_conflicts( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.get_merge_conflicts(dashboard_id) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.get_merge_conflicts(dashboard_id)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_merge_conflicts", e) + + # #endregion get_merge_conflicts @@ -98,18 +97,20 @@ async def resolve_merge_conflicts( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - resolved_files = await _gs.resolve_merge_conflicts( - dashboard_id, - [item.model_dump() for item in resolve_data.resolutions], + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + resolved_files = await _await_service_result( + _gs.resolve_merge_conflicts( + dashboard_id, + [item.model_dump() for item in resolve_data.resolutions], + ) ) return {"status": "success", "resolved_files": resolved_files} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("resolve_merge_conflicts", e) + + # #endregion resolve_merge_conflicts @@ -128,14 +129,14 @@ async def abort_merge( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.abort_merge(dashboard_id) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.abort_merge(dashboard_id)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("abort_merge", e) + + # #endregion abort_merge @@ -156,14 +157,14 @@ async def continue_merge( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.continue_merge(dashboard_id, continue_data.message) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.continue_merge(dashboard_id, continue_data.message)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("continue_merge", e) + + # #endregion continue_merge @@ -189,9 +190,7 @@ async def merge_branch( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) return await _gs.merge_branch( dashboard_id, merge_data.source_branch, @@ -203,5 +202,7 @@ async def merge_branch( raise except Exception as e: _handle_unexpected_git_route_error("merge_branch", e) + + # #endregion merge_branch # #endregion GitMergeRoutes diff --git a/backend/src/api/routes/git/_repo_operations_routes.py b/backend/src/api/routes/git/_repo_operations_routes.py index 13064f77..bed3d3e4 100644 --- a/backend/src/api/routes/git/_repo_operations_routes.py +++ b/backend/src/api/routes/git/_repo_operations_routes.py @@ -23,6 +23,7 @@ from src.models.git import GitRepository, GitServerConfig from ._deps import MAX_REPOSITORY_STATUS_BATCH, get_git_service from ._helpers import ( _apply_git_identity_from_profile, + _await_service_result, _build_no_repo_status_payload, _handle_unexpected_git_route_error, _resolve_current_user_git_token, @@ -49,18 +50,16 @@ async def commit_changes( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) await _apply_git_identity_from_profile(dashboard_id, db, current_user) - await _gs.commit_changes( - dashboard_id, commit_data.message, commit_data.files - ) + await _await_service_result(_gs.commit_changes(dashboard_id, commit_data.message, commit_data.files)) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("commit_changes", e) + + # #endregion commit_changes @@ -82,19 +81,21 @@ async def rollback_commit( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) await _apply_git_identity_from_profile(dashboard_id, db, current_user) - return await _gs.rollback_commit( - dashboard_id, - rollback_data.commit_hash, - rollback_data.reason, + return await _await_service_result( + _gs.rollback_commit( + dashboard_id, + rollback_data.commit_hash, + rollback_data.reason, + ) ) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("rollback_commit", e) + + # #endregion rollback_commit @@ -115,16 +116,16 @@ async def push_changes( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) pat = _resolve_current_user_git_token(db, current_user) - await _gs.push_changes(dashboard_id, pat=pat) + await _await_service_result(_gs.push_changes(dashboard_id, pat=pat)) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("push_changes", e) + + # #endregion push_changes @@ -146,25 +147,15 @@ async def pull_changes( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) db_repo = None config_url = None config_provider = None try: - db_repo_candidate = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) + db_repo_candidate = db.query(GitRepository).filter(GitRepository.dashboard_id == dashboard_id).first() if getattr(db_repo_candidate, "config_id", None): db_repo = db_repo_candidate - config_row = ( - db.query(GitServerConfig) - .filter(GitServerConfig.id == db_repo.config_id) - .first() - ) + config_row = db.query(GitServerConfig).filter(GitServerConfig.id == db_repo.config_id).first() if config_row: config_url = config_row.url config_provider = config_row.provider @@ -183,12 +174,14 @@ async def pull_changes( ) await _apply_git_identity_from_profile(dashboard_id, db, current_user) pat = _resolve_current_user_git_token(db, current_user) - await _gs.pull_changes(dashboard_id, pat=pat) + await _await_service_result(_gs.pull_changes(dashboard_id, pat=pat)) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("pull_changes", e) + + # #endregion pull_changes @@ -206,14 +199,14 @@ async def get_repository_status( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) return await _resolve_repository_status(dashboard_id) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_repository_status", e) + + # #endregion get_repository_status @@ -252,6 +245,8 @@ async def get_repository_status_batch( "sync_status": "ERROR", } return RepoStatusBatchResponse(statuses=statuses) + + # #endregion get_repository_status_batch @@ -272,14 +267,14 @@ async def get_repository_diff( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.get_diff(dashboard_id, file_path, staged) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.get_diff(dashboard_id, file_path, staged)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_repository_diff", e) + + # #endregion get_repository_diff @@ -299,14 +294,14 @@ async def get_history( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.get_commit_history(dashboard_id, limit) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.get_commit_history(dashboard_id, limit)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_history", e) + + # #endregion get_history @@ -327,17 +322,15 @@ async def generate_commit_message( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - diff = await _gs.get_diff(dashboard_id, staged=True) - if not diff: - diff = await _gs.get_diff(dashboard_id, staged=False) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + diff = await _await_service_result(_gs.get_diff(dashboard_id, staged=True)) + if not diff or all(item is None for item in diff): + diff = await _await_service_result(_gs.get_diff(dashboard_id, staged=False)) - if not diff: + if not diff or all(item is None for item in diff): return {"message": "No changes detected"} - history_objs = await _gs.get_commit_history(dashboard_id, limit=5) + history_objs = await _await_service_result(_gs.get_commit_history(dashboard_id, limit=5)) history = [h.message for h in history_objs if hasattr(h, "message")] from src.plugins.llm_analysis.models import LLMProviderType @@ -351,18 +344,14 @@ async def generate_commit_message( llm_service = LLMProviderService(db) providers = llm_service.get_all_providers() - llm_settings = normalize_llm_settings( - config_manager.get_config().settings.llm - ) + llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) bound_provider_id = resolve_bound_provider_id(llm_settings, "git_commit") provider = next((p for p in providers if p.id == bound_provider_id), None) if not provider: provider = next((p for p in providers if p.is_active), None) if not provider: - raise HTTPException( - status_code=400, detail="No active LLM provider found" - ) + raise HTTPException(status_code=400, detail="No active LLM provider found") api_key = llm_service.get_decrypted_api_key(provider.id) client = LLMClient( @@ -389,5 +378,7 @@ async def generate_commit_message( raise except Exception as e: _handle_unexpected_git_route_error("generate_commit_message", e) + + # #endregion generate_commit_message # #endregion GitRepoOperationsRoutes diff --git a/backend/src/api/routes/git/_repo_routes.py b/backend/src/api/routes/git/_repo_routes.py index 97d4df45..3d906586 100644 --- a/backend/src/api/routes/git/_repo_routes.py +++ b/backend/src/api/routes/git/_repo_routes.py @@ -24,6 +24,7 @@ from src.models.git import GitRepository, GitServerConfig from ._deps import get_git_service from ._helpers import ( _apply_git_identity_from_profile, + _await_service_result, _get_git_config_or_404, _handle_unexpected_git_route_error, ) @@ -47,17 +48,9 @@ async def init_repository( with belief_scope("init_repository"): from . import _resolve_dashboard_id_from_ref, _resolve_repo_key_from_ref - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - repo_key = await _resolve_repo_key_from_ref( - dashboard_ref, dashboard_id, config_manager, env_id - ) - config = ( - db.query(GitServerConfig) - .filter(GitServerConfig.id == init_data.config_id) - .first() - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + repo_key = await _resolve_repo_key_from_ref(dashboard_ref, dashboard_id, config_manager, env_id) + config = db.query(GitServerConfig).filter(GitServerConfig.id == init_data.config_id).first() if not config: raise HTTPException(status_code=404, detail="Git configuration not found") @@ -75,11 +68,7 @@ async def init_repository( ) repo_path = await _gs._get_repo_path(dashboard_id, repo_key=repo_key) - db_repo = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) + db_repo = db.query(GitRepository).filter(GitRepository.dashboard_id == dashboard_id).first() if not db_repo: db_repo = GitRepository( dashboard_id=dashboard_id, @@ -104,6 +93,8 @@ async def init_repository( if isinstance(e, HTTPException): raise _handle_unexpected_git_route_error("init_repository", e) + + # #endregion init_repository @@ -122,18 +113,10 @@ async def get_repository_binding( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - db_repo = ( - db.query(GitRepository) - .filter(GitRepository.dashboard_id == dashboard_id) - .first() - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + db_repo = db.query(GitRepository).filter(GitRepository.dashboard_id == dashboard_id).first() if not db_repo: - raise HTTPException( - status_code=404, detail="Repository not initialized" - ) + raise HTTPException(status_code=404, detail="Repository not initialized") config = _get_git_config_or_404(db, db_repo.config_id) return RepositoryBindingSchema( dashboard_id=db_repo.dashboard_id, @@ -146,6 +129,8 @@ async def get_repository_binding( raise except Exception as e: _handle_unexpected_git_route_error("get_repository_binding", e) + + # #endregion get_repository_binding @@ -164,15 +149,15 @@ async def delete_repository( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) await _gs.delete_repo(dashboard_id) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("delete_repository", e) + + # #endregion delete_repository @@ -191,14 +176,14 @@ async def get_branches( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - return await _gs.list_branches(dashboard_id) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + return await _await_service_result(_gs.list_branches(dashboard_id)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_branches", e) + + # #endregion get_branches @@ -216,15 +201,15 @@ async def get_branch_protection_rules( from . import _resolve_dashboard_id_from_ref try: - await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) _gs = get_git_service() return _gs.get_branch_protection_rules() except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("get_branch_protection_rules", e) + + # #endregion get_branch_protection_rules @@ -246,18 +231,16 @@ async def create_branch( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) await _apply_git_identity_from_profile(dashboard_id, db, current_user) - await _gs.create_branch( - dashboard_id, branch_data.name, branch_data.from_branch - ) + await _await_service_result(_gs.create_branch(dashboard_id, branch_data.name, branch_data.from_branch)) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("create_branch", e) + + # #endregion create_branch @@ -277,15 +260,15 @@ async def checkout_branch( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) - await _gs.checkout_branch(dashboard_id, checkout_data.name) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + await _await_service_result(_gs.checkout_branch(dashboard_id, checkout_data.name)) return {"status": "success"} except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("checkout_branch", e) + + # #endregion checkout_branch @@ -309,14 +292,14 @@ async def delete_branch( from . import _resolve_dashboard_id_from_ref try: - dashboard_id = await _resolve_dashboard_id_from_ref( - dashboard_ref, config_manager, env_id - ) + dashboard_id = await _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) force = delete_data.force if delete_data else False - return await _gs.delete_branch(dashboard_id, branch_name, force=force) + return await _await_service_result(_gs.delete_branch(dashboard_id, branch_name, force=force)) except HTTPException: raise except Exception as e: _handle_unexpected_git_route_error("delete_branch", e) + + # #endregion delete_branch # #endregion GitRepoRoutes diff --git a/backend/src/scripts/test_dataset_dashboard_relations.py b/backend/src/scripts/test_dataset_dashboard_relations.py index a449e4d4..6b628272 100644 --- a/backend/src/scripts/test_dataset_dashboard_relations.py +++ b/backend/src/scripts/test_dataset_dashboard_relations.py @@ -24,7 +24,7 @@ from src.core.logger import logger from src.core.superset_client import SupersetClient -def test_dashboard_dataset_relations(): +def inspect_dashboard_dataset_relations(): """Test fetching dataset-to-dashboard relationships.""" seed_trace_id() @@ -59,18 +59,18 @@ def test_dashboard_dataset_relations(): print(f" Published: {dashboard.get('published')}") # Check for slices/charts - if 'slices' in dashboard: - logger.reason("Found slices in dashboard", payload={"dashboard_id": dashboard_id, "slice_count": len(dashboard['slices'])}) - for i, slice_data in enumerate(dashboard['slices'][:5]): # Show first 5 - print(f" Slice {i+1}:") + if "slices" in dashboard: + logger.reason("Found slices in dashboard", payload={"dashboard_id": dashboard_id, "slice_count": len(dashboard["slices"])}) + for i, slice_data in enumerate(dashboard["slices"][:5]): # Show first 5 + print(f" Slice {i + 1}:") print(f" ID: {slice_data.get('slice_id')}") print(f" Name: {slice_data.get('slice_name')}") # Check for datasource_id - if 'datasource_id' in slice_data: + if "datasource_id" in slice_data: print(f" Datasource ID: {slice_data['datasource_id']}") - if 'datasource_name' in slice_data: + if "datasource_name" in slice_data: print(f" Datasource Name: {slice_data['datasource_name']}") - if 'datasource_type' in slice_data: + if "datasource_type" in slice_data: print(f" Datasource Type: {slice_data['datasource_type']}") else: logger.reflect("No slices field in dashboard response", payload={"dashboard_id": dashboard_id}) @@ -93,17 +93,14 @@ def test_dashboard_dataset_relations(): # Method: Use Superset's related_objects API try: logger.reason("Querying related objects endpoint", payload={"dataset_id": dataset_id}) - related_objects = client.network.request( - method="GET", - endpoint=f"/dataset/{dataset_id}/related_objects" - ) + related_objects = client.network.request(method="GET", endpoint=f"/dataset/{dataset_id}/related_objects") logger.reason("Related objects response type", payload={"response_type": str(type(related_objects))}) logger.reason("Related objects keys", payload={"keys": list(related_objects.keys()) if isinstance(related_objects, dict) else "N/A"}) # Check for dashboards in related objects - if 'dashboards' in related_objects: - dashboards = related_objects['dashboards'] + if "dashboards" in related_objects: + dashboards = related_objects["dashboards"] logger.reason("Found dashboards for dataset", payload={"dataset_id": dataset_id, "count": len(dashboards)}) for dash in dashboards: @@ -111,11 +108,11 @@ def test_dashboard_dataset_relations(): logger.reason("Dashboard entry", payload={"dashboard_id": dash.get("id"), "title": dash.get("dashboard_title", dash.get("title", "Unknown"))}) else: logger.reason("Dashboard entry", payload={"dashboard": str(dash)}) - elif 'result' in related_objects: + elif "result" in related_objects: # Some Superset versions use 'result' wrapper - result = related_objects['result'] - if 'dashboards' in result: - dashboards = result['dashboards'] + result = related_objects["result"] + if "dashboards" in result: + dashboards = result["dashboards"] logger.reason("Found dashboards for dataset (result wrapper)", payload={"dataset_id": dataset_id, "count": len(dashboards)}) for dash in dashboards: @@ -130,38 +127,36 @@ def test_dashboard_dataset_relations(): except Exception as e: logger.explore("Error fetching related objects", payload={"dataset_id": dataset_id}, error=str(e)) import traceback + traceback.print_exc() # Method 2: Try to use the position_json from dashboard logger.reason("Analyzing dashboard position JSON") - if 'position_json' in dashboard: - position_data = json.loads(dashboard['position_json']) + if "position_json" in dashboard: + position_data = json.loads(dashboard["position_json"]) logger.reason("Position data type", payload={"type": str(type(position_data))}) # Look for datasource references datasource_ids = set() if isinstance(position_data, dict): for key, value in position_data.items(): - if 'datasource' in key.lower() or key == 'DASHBOARD_VERSION_KEY': + if "datasource" in key.lower() or key == "DASHBOARD_VERSION_KEY": logger.reason("Position entry", payload={"key": key, "value_type": str(type(value))}) elif isinstance(position_data, list): logger.reason("Position data item count", payload={"count": len(position_data)}) for item in position_data[:3]: # Show first 3 logger.reason("Position item", payload={"item_type": str(type(item)), "keys": list(item.keys()) if isinstance(item, dict) else "N/A"}) if isinstance(item, dict): - if 'datasource_id' in item: - datasource_ids.add(item['datasource_id']) + if "datasource_id" in item: + datasource_ids.add(item["datasource_id"]) if datasource_ids: logger.reason("Found datasource IDs in position data", payload={"datasource_ids": sorted(datasource_ids)}) # Save full response for analysis output_file = Path(__file__).parent / "dataset_dashboard_analysis.json" - with open(output_file, 'w') as f: - json.dump({ - 'dashboard': dashboard, - 'dataset': dataset - }, f, indent=2, default=str) + with open(output_file, "w") as f: + json.dump({"dashboard": dashboard, "dataset": dataset}, f, indent=2, default=str) logger.reason("Full response saved", payload={"output_file": str(output_file)}) except Exception as e: @@ -169,8 +164,12 @@ def test_dashboard_dataset_relations(): raise +test_dashboard_dataset_relations = inspect_dashboard_dataset_relations +test_dashboard_dataset_relations.__test__ = False + + if __name__ == "__main__": - test_dashboard_dataset_relations() + inspect_dashboard_dataset_relations() # #endregion test_dataset_dashboard_relations_script # #endregion test_dataset_dashboard_relations_script diff --git a/backend/tests/test_agent_confirmation_v2.py b/backend/tests/test_agent_confirmation_v2.py deleted file mode 100644 index 5a17720a..00000000 --- a/backend/tests/test_agent_confirmation_v2.py +++ /dev/null @@ -1,70 +0,0 @@ -# #region Test.AgentChat.ConfirmationV2 [C:3] [TYPE Module] [SEMANTICS test,agent-chat,confirmation,guardrails] -# @BRIEF Contract tests for build_confirmation_contract_v2 and permission_denied payloads. -# @RELATION BINDS_TO -> [AgentChat.Confirmation.GuardV2] -# @RELATION BINDS_TO -> [AgentChat.Confirmation.PermissionDenied] -# @TEST_EDGE: deploy_prod -> guarded write with prod env_context. -# @TEST_EDGE: delete_any -> dangerous operation. -# @TEST_EDGE: analyst_deploy -> permission_denied SSE metadata. - -import json - -from src.agent._confirmation import build_confirmation_contract_v2, permission_denied_payload - - -# #region test_deploy_prod_guarded [C:2] [TYPE Function] -# @BRIEF deploy_dashboard targeting prod resolves guarded write + env_context prod. -def test_deploy_prod_guarded(): - contract = build_confirmation_contract_v2( - "deploy_dashboard", - {"env_id": "prod-eu"}, - user_role="admin", - target_env="staging", - ) - - assert contract["risk"] == "write" - assert contract["risk_level"] == "guarded" - assert contract["env_context"] == "prod" - assert contract["permission_granted"] is True -# #endregion test_deploy_prod_guarded - - -# #region test_delete_any_is_dangerous [C:2] [TYPE Function] -# @BRIEF delete-prefixed operations are classified as dangerous. -def test_delete_any_is_dangerous(): - contract = build_confirmation_contract_v2("delete_dashboard", {}, user_role="admin") - - assert contract["risk"] == "write" - assert contract["risk_level"] == "dangerous" - assert contract["dangerous"] is True -# #endregion test_delete_any_is_dangerous - - -# #region test_read_only_safe [C:2] [TYPE Function] -# @BRIEF search_dashboards remains safe read. -def test_read_only_safe(): - contract = build_confirmation_contract_v2("search_dashboards", {}, user_role="analyst") - - assert contract["risk"] == "read" - assert contract["risk_level"] == "safe" - assert contract["permission_granted"] is True -# #endregion test_read_only_safe - - -# #region test_analyst_deploy_permission_denied_payload [C:2] [TYPE Function] -# @BRIEF Analyst deploy produces permission_denied metadata, not confirm_required metadata. -def test_analyst_deploy_permission_denied_payload(): - contract = build_confirmation_contract_v2("deploy_dashboard", {}, user_role="analyst") - payload = json.loads(permission_denied_payload( - "deploy_dashboard", - required_role=contract["required_role"], - user_role="analyst", - alternatives=contract["alternatives"], - )) - - assert contract["permission_granted"] is False - assert payload["metadata"]["type"] == "permission_denied" - assert payload["metadata"]["required_role"] == "admin" - assert payload["metadata"]["user_role"] == "analyst" -# #endregion test_analyst_deploy_permission_denied_payload - -# #endregion Test.AgentChat.ConfirmationV2 diff --git a/backend/tests/test_agent_context.py b/backend/tests/test_agent_context.py deleted file mode 100644 index b6f960d6..00000000 --- a/backend/tests/test_agent_context.py +++ /dev/null @@ -1,61 +0,0 @@ -# #region Test.AgentChat.Context [C:3] [TYPE Module] [SEMANTICS test,agent-chat,context,validation] -# @BRIEF Contract tests for UIContext validation at the backend trust boundary. -# @RELATION BINDS_TO -> [AgentChat.Context.Validate] -# @TEST_EDGE: valid -> returns unchanged payload. -# @TEST_EDGE: invalid_objectType -> raises validation error. -# @TEST_EDGE: oversized -> raises validation error. -# @TEST_EDGE: objectName_too_long -> raises validation error. -# ruff: noqa: I001 - -import pytest - -from src.agent._context import UIContextValidationError, validate_uicontext - - -VALID_CONTEXT = { - "objectType": "dashboard", - "objectId": "42", - "objectName": "Energy", - "envId": "ss-dev", - "route": "/dashboards/42", - "contextVersion": 1, -} - - -# #region test_valid_uicontext_returns_payload [C:2] [TYPE Function] -# @BRIEF Valid UIContext payload returns unchanged. -def test_valid_uicontext_returns_payload(): - assert validate_uicontext(dict(VALID_CONTEXT)) == VALID_CONTEXT -# #endregion test_valid_uicontext_returns_payload - - -# #region test_invalid_object_type_rejected [C:2] [TYPE Function] -# @BRIEF Unknown objectType is rejected by backend validation. -def test_invalid_object_type_rejected(): - payload = dict(VALID_CONTEXT, objectType="chart") - - with pytest.raises(UIContextValidationError, match="objectType"): - validate_uicontext(payload) -# #endregion test_invalid_object_type_rejected - - -# #region test_oversized_payload_rejected [C:2] [TYPE Function] -# @BRIEF Payloads over 4KB are rejected before prompt injection. -def test_oversized_payload_rejected(): - payload = dict(VALID_CONTEXT, debug_padding="x" * 5000) - - with pytest.raises(UIContextValidationError, match="4 KB"): - validate_uicontext(payload) -# #endregion test_oversized_payload_rejected - - -# #region test_object_name_too_long_rejected [C:2] [TYPE Function] -# @BRIEF objectName longer than 256 chars is rejected. -def test_object_name_too_long_rejected(): - payload = dict(VALID_CONTEXT, objectName="x" * 257) - - with pytest.raises(UIContextValidationError, match="objectName"): - validate_uicontext(payload) -# #endregion test_object_name_too_long_rejected - -# #endregion Test.AgentChat.Context diff --git a/backend/tests/test_agent_tool_filter.py b/backend/tests/test_agent_tool_filter.py deleted file mode 100644 index 1ce0403d..00000000 --- a/backend/tests/test_agent_tool_filter.py +++ /dev/null @@ -1,78 +0,0 @@ -# #region Test.AgentChat.ToolFilter [C:3] [TYPE Module] [SEMANTICS test,agent-chat,tools,filter] -# @BRIEF Contract tests for AgentChat.ToolFilter — RBAC first, then context affinity. -# @RELATION BINDS_TO -> [AgentChat.ToolFilter.Pipeline] -# @TEST_EDGE: null_context -> all RBAC-allowed tools are returned. -# @TEST_EDGE: dashboard_analyst -> dashboard tools minus admin-only tools. -# @TEST_EDGE: unknown_context -> graceful fallback to RBAC-only filtering. - -from types import SimpleNamespace - -from src.agent._tool_filter import build_tool_pipeline, enforce_tool_permission - - -def _tool(name: str) -> SimpleNamespace: - return SimpleNamespace(name=name) - - -TOOLS = [ - _tool("show_capabilities"), - _tool("search_dashboards"), - _tool("get_health_summary"), - _tool("deploy_dashboard"), - _tool("create_branch"), - _tool("commit_changes"), - _tool("superset_explore_database"), - _tool("run_backup"), -] - - -# #region test_null_context_filters_by_rbac_only [C:2] [TYPE Function] -# @BRIEF Null objectType returns every tool the role may invoke. -def test_null_context_filters_by_rbac_only(): - result = build_tool_pipeline(TOOLS, user_role="analyst", object_type=None) - - assert [tool.name for tool in result] == [ - "show_capabilities", - "search_dashboards", - "get_health_summary", - "superset_explore_database", - ] -# #endregion test_null_context_filters_by_rbac_only - - -# #region test_dashboard_analyst_filters_context_after_rbac [C:2] [TYPE Function] -# @BRIEF Dashboard context excludes non-dashboard tools after RBAC removes admin-only tools. -def test_dashboard_analyst_filters_context_after_rbac(): - result = build_tool_pipeline(TOOLS, user_role="analyst", object_type="dashboard") - - assert [tool.name for tool in result] == [ - "show_capabilities", - "search_dashboards", - "get_health_summary", - ] -# #endregion test_dashboard_analyst_filters_context_after_rbac - - -# #region test_unknown_context_falls_back_to_rbac_only [C:2] [TYPE Function] -# @BRIEF Unknown objectType does not drop RBAC-allowed tools. -def test_unknown_context_falls_back_to_rbac_only(): - result = build_tool_pipeline(TOOLS, user_role="analyst", object_type="unknown") - - assert [tool.name for tool in result] == [ - "show_capabilities", - "search_dashboards", - "get_health_summary", - "superset_explore_database", - ] -# #endregion test_unknown_context_falls_back_to_rbac_only - - -# #region test_enforce_tool_permission_rejects_admin_only [C:2] [TYPE Function] -# @BRIEF Invocation guard rejects analyst access to admin-only tools. -def test_enforce_tool_permission_rejects_admin_only(): - assert enforce_tool_permission("deploy_dashboard", "analyst") is False - assert enforce_tool_permission("deploy_dashboard", "admin") is True - assert enforce_tool_permission("search_dashboards", "analyst") is True -# #endregion test_enforce_tool_permission_rejects_admin_only - -# #endregion Test.AgentChat.ToolFilter diff --git a/build.sh b/build.sh index 8fc89133..73b8797d 100755 --- a/build.sh +++ b/build.sh @@ -11,7 +11,8 @@ # restart [profile] Rebuild and restart services # logs [profile] Tail logs # status Show running containers for all profiles -# bundle Full release bundle (backend + frontend + postgres .tar.xz's) +# bundle Slim enterprise bundle (no embeddings agent). Default. +# bundle:embeddings Enterprise bundle WITH semantic embedding routing (larger agent) # bundle:light Lightweight all-in-one bundle (<200 MB, .tar.xz, no Playwright) # help Show this help # @@ -202,7 +203,7 @@ compose_status() { } # ====================================================================== -# BUNDLE: full release (backend + frontend + postgres .tar's) +# BUNDLE: full release (backend + frontend + agent .tar.xz's) # ====================================================================== bundle_release() { @@ -500,6 +501,194 @@ COMPOSE echo " docker run -p 8000:8000 --env-file backend/.env ${image_tag}" } +# ====================================================================== +# BUNDLE: enterprise with embeddings (agent + sentence-transformers) +# ====================================================================== + +bundle_embeddings() { + if [[ $# -lt 1 ]]; then + echo "Error: tag is required." + echo "Usage: ./build.sh bundle:embeddings (e.g. v1.0.0)" + exit 1 + fi + local tag="$1" + local backend_tag="superset-tools-backend:${tag}-embeddings" + local frontend_tag="superset-tools-frontend:${tag}-embeddings" + local agent_tag="superset-tools-agent:${tag}-embeddings" + + mkdir -p "$DIST_ROOT" + + echo "[bundle:embeddings] Building backend image ${backend_tag}..." + docker build -f docker/backend.Dockerfile -t "${backend_tag}" . + + echo "[bundle:embeddings] Building frontend image ${frontend_tag}..." + docker build --build-arg "APP_VERSION=${tag}" -f docker/frontend.Dockerfile -t "${frontend_tag}" . + + echo "[bundle:embeddings] Building agent image ${agent_tag} (WITH_EMBEDDINGS=true)..." + docker build \ + --build-arg WITH_EMBEDDINGS=true \ + -f docker/Dockerfile.agent \ + -t "${agent_tag}" . + + echo "[bundle:embeddings] Exporting .tar.xz archives..." + echo "[bundle:embeddings] Compressing backend image..." + docker save "${backend_tag}" | xz -T0 -6 > "${DIST_ROOT}/superset-tools-backend.${tag}-embeddings.tar.xz" + echo "[bundle:embeddings] Compressing frontend image..." + docker save "${frontend_tag}" | xz -T0 -6 > "${DIST_ROOT}/superset-tools-frontend.${tag}-embeddings.tar.xz" + echo "[bundle:embeddings] Compressing agent image..." + docker save "${agent_tag}" | xz -T0 -6 > "${DIST_ROOT}/superset-tools-agent.${tag}-embeddings.tar.xz" + + echo "[bundle:embeddings] Calculating checksums..." + ( + cd "${DIST_ROOT}" + sha256sum \ + "superset-tools-backend.${tag}-embeddings.tar.xz" \ + "superset-tools-frontend.${tag}-embeddings.tar.xz" \ + "superset-tools-agent.${tag}-embeddings.tar.xz" > "sha256sums.${tag}-embeddings.txt" + ) + + local backend_id frontend_id agent_id + local backend_digest frontend_digest agent_digest + backend_id="$(get_image_id "${backend_tag}")" + frontend_id="$(get_image_id "${frontend_tag}")" + agent_id="$(get_image_id "${agent_tag}")" + backend_digest="$(get_repo_digest "${backend_tag}")" + frontend_digest="$(get_repo_digest "${frontend_tag}")" + agent_digest="$(get_repo_digest "${agent_tag}")" + + # Generate deploy-compose — same as enterprise-clean, with embeddings image tags + cat > "${DIST_ROOT}/docker-compose.enterprise-clean.yml" < "${DIST_ROOT}/manifest.${tag}-embeddings.txt" < "${DIST_ROOT}/manifest.${tag}-embeddings.json" < .env.enterprise-clean, then:" + echo " docker compose -f ${DIST_ROOT}/docker-compose.enterprise-clean.yml up" + echo "" + echo " NOTE: Agent image includes sentence-transformers+torch (~500 MB+)." + echo " Use 'bundle' for the default slim agent (no embedding model)." +} + # ====================================================================== # HELP # ====================================================================== @@ -516,12 +705,20 @@ Commands for local docker: status Show running containers for all profiles Commands for release bundles: - bundle Build release images (backend + frontend + agent .tar.xz's). - NOTE: external PostgreSQL only — no bundled postgres image. - Example: ./build.sh bundle v1.0.0 + bundle Default slim bundle (backend + frontend + agent .tar.xz's). + Agent: NO sentence-transformers/embeddings (~500 MB saved). + Backend: NO agent/ML/dev deps. + NOTE: external PostgreSQL only — no bundled postgres image. + Example: ./build.sh bundle v1.0.0 + + bundle:embeddings + Enterprise bundle with semantic embedding routing. + Agent built WITH sentence-transformers+torch (larger image). + Same backend/frontend as default bundle. + Example: ./build.sh bundle:embeddings v1.0.0 bundle:light Lightweight all-in-one image (.tar.xz, <200 MB, no Playwright) - Example: ./build.sh bundle:light v1.0.0 + Example: ./build.sh bundle:light v1.0.0 Profiles: current (default), master, enterprise-clean @@ -550,7 +747,7 @@ main() { shift 2>/dev/null || true case "$CMD" in - up|down|restart|logs|bundle|bundle:light|status|help|-h|--help) + up|down|restart|logs|bundle|bundle:embeddings|bundle:light|status|help|-h|--help) # Valid commands — proceed ;; *) @@ -566,8 +763,9 @@ main() { down) compose_down "${1:-current}" ;; restart) compose_restart "${1:-current}" ;; logs) compose_logs "${1:-current}" ;; - bundle) bundle_release "$@" ;; - bundle:light) bundle_light "$@" ;; + bundle) bundle_release "$@" ;; + bundle:embeddings) bundle_embeddings "$@" ;; + bundle:light) bundle_light "$@" ;; status) compose_status ;; help|-h|--help) show_help ;; *) diff --git a/docker/Dockerfile.agent b/docker/Dockerfile.agent index b3cb80b3..1fcb8c3b 100644 --- a/docker/Dockerfile.agent +++ b/docker/Dockerfile.agent @@ -20,10 +20,18 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ libgl1 libglib2.0-0 && rm -rf /var/lib/apt/lists/* -# Python dependencies — используем backend/requirements.txt (содержит httpx, gradio, -# langgraph, langchain-openai, pdfplumber и все транзитивные зависимости) -COPY backend/requirements.txt /app/backend/requirements.txt -RUN pip install --no-cache-dir -r /app/backend/requirements.txt +# Python dependencies — agent runtime (gradio, langgraph, httpx, pdfplumber, ...) +# Без sentence-transformers — embedding router безопасно отключается при ImportError. +ARG WITH_EMBEDDINGS=false + +COPY backend/requirements-agent.txt /app/backend/requirements-agent.txt +RUN pip install --no-cache-dir -r /app/backend/requirements-agent.txt + +# Optional: semantic embedding routing (большие ML-зависимости: ~500 MB+) +COPY backend/requirements-embeddings.txt /app/backend/requirements-embeddings.txt +RUN if [ "$WITH_EMBEDDINGS" = "true" ]; then \ + pip install --no-cache-dir -r /app/backend/requirements-embeddings.txt; \ + fi # ── Минимальная копия backend/src/ ────────────────────────────── # Только то, что реально импортирует агент (трассировка 2026-06-14): diff --git a/docker/all-in-one.Dockerfile b/docker/all-in-one.Dockerfile index d8bcf403..6fd43406 100644 --- a/docker/all-in-one.Dockerfile +++ b/docker/all-in-one.Dockerfile @@ -42,8 +42,8 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Python deps (NO playwright — saves ~350MB) -COPY backend/requirements-docker.txt /app/backend/requirements-docker.txt -RUN pip install --no-cache-dir -r /app/backend/requirements-docker.txt +COPY backend/requirements-backend.txt /app/backend/requirements-backend.txt +RUN pip install --no-cache-dir -r /app/backend/requirements-backend.txt # Backend source COPY backend/ /app/backend/ diff --git a/docker/backend.Dockerfile b/docker/backend.Dockerfile index 0d7b44e7..884c9820 100644 --- a/docker/backend.Dockerfile +++ b/docker/backend.Dockerfile @@ -23,9 +23,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libnss3-tools \ && rm -rf /var/lib/apt/lists/* -# Python зависимости -COPY backend/requirements.txt /app/backend/requirements.txt -RUN pip install --no-cache-dir -r /app/backend/requirements.txt +# Python зависимости — backend runtime (agent/ML deps excluded) +COPY backend/requirements-backend.txt /app/backend/requirements-backend.txt +RUN pip install --no-cache-dir -r /app/backend/requirements-backend.txt RUN python -m playwright install --with-deps chromium # Исходный код diff --git a/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.ts b/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.ts index 702bd7aa..a2d8f6db 100644 --- a/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.ts +++ b/frontend/src/lib/components/assistant/__tests__/assistant_chat.integration.test.ts @@ -47,11 +47,11 @@ describe('AssistantChatPanel integration contract', () => { const source = fs.readFileSync(COMPONENT_PATH, 'utf-8'); expect(source).toContain(''); }); diff --git a/frontend/src/lib/components/translate/__tests__/test_bulk_replace_modal.svelte.js b/frontend/src/lib/components/translate/__tests__/test_bulk_replace_modal.svelte.js index 34beafeb..75776b27 100644 --- a/frontend/src/lib/components/translate/__tests__/test_bulk_replace_modal.svelte.js +++ b/frontend/src/lib/components/translate/__tests__/test_bulk_replace_modal.svelte.js @@ -34,8 +34,8 @@ describe('BulkReplaceModal Component', () => { expect(source).toContain('@UX_STATE applying'); expect(source).toContain('@UX_STATE applied'); expect(source).toContain('@UX_STATE apply_error'); - expect(source).toContain('bulkFindReplace'); - expect(source).toContain('bulkReplacePreview'); + expect(source).toContain("bulk_replace"); + expect(source).toContain("bulk_replace?.preview_count"); }); it('accepts show, runId, targetLanguages, onClose, onApplied props', () => {