diff --git a/agent/src/ss_tools/agent/_config.py b/agent/src/ss_tools/agent/_config.py index 01eb29ed..77ccdb7c 100644 --- a/agent/src/ss_tools/agent/_config.py +++ b/agent/src/ss_tools/agent/_config.py @@ -11,6 +11,7 @@ FASTAPI_URL: str = os.getenv("FASTAPI_URL", "http://localhost:8000") SERVICE_JWT: str = os.getenv("SERVICE_JWT", "") GRADIO_SERVER_NAME: str = os.getenv("GRADIO_SERVER_NAME", "0.0.0.0") GRADIO_SERVER_PORT: int = int(os.getenv("GRADIO_SERVER_PORT", "7860")) +GRADIO_ROOT_PATH: str = os.getenv("GRADIO_ROOT_PATH", "/api/agent/gradio") GRADIO_ALLOW_PORT_FALLBACK: bool = os.getenv("GRADIO_ALLOW_PORT_FALLBACK", "").strip().lower() in {"1", "true", "yes"} STORAGE_ROOT: str = os.getenv("STORAGE_ROOT", "/app/storage") AGENT_PREFETCH_DASHBOARD_LIMIT: int = int(os.getenv("AGENT_PREFETCH_DASHBOARD_LIMIT", "25")) diff --git a/agent/src/ss_tools/agent/app.py b/agent/src/ss_tools/agent/app.py index bef64c5e..531aa718 100644 --- a/agent/src/ss_tools/agent/app.py +++ b/agent/src/ss_tools/agent/app.py @@ -33,7 +33,7 @@ from langchain_core.messages import HumanMessage from langchain_openai import ChatOpenAI from openai import APIConnectionError, APITimeoutError, AuthenticationError, RateLimitError -from ss_tools.agent._config import GRADIO_SERVER_NAME, GRADIO_SERVER_PORT, STORAGE_ROOT as _STORAGE_ROOT +from ss_tools.agent._config import GRADIO_ROOT_PATH, GRADIO_SERVER_NAME, GRADIO_SERVER_PORT, STORAGE_ROOT as _STORAGE_ROOT from ss_tools.agent._confirmation import ( _pending_confirmations, confirmation_payload, @@ -817,5 +817,6 @@ if __name__ == "__main__": demo.launch( server_name=GRADIO_SERVER_NAME, server_port=GRADIO_SERVER_PORT, + root_path=GRADIO_ROOT_PATH, ) # #endregion AgentChat.GradioApp diff --git a/agent/src/ss_tools/agent/run.py b/agent/src/ss_tools/agent/run.py index 464ccdab..2bfeb457 100644 --- a/agent/src/ss_tools/agent/run.py +++ b/agent/src/ss_tools/agent/run.py @@ -12,7 +12,14 @@ import socket import httpx -from ss_tools.agent._config import FASTAPI_URL, GRADIO_ALLOW_PORT_FALLBACK, GRADIO_SERVER_NAME, GRADIO_SERVER_PORT, SERVICE_JWT +from ss_tools.agent._config import ( + FASTAPI_URL, + GRADIO_ALLOW_PORT_FALLBACK, + GRADIO_ROOT_PATH, + GRADIO_SERVER_NAME, + GRADIO_SERVER_PORT, + SERVICE_JWT, +) from ss_tools.shared.cot_logger import seed_trace_id from ss_tools.shared.logger import logger @@ -128,5 +135,6 @@ if __name__ == "__main__": demo.launch( server_name=GRADIO_SERVER_NAME, server_port=port, + root_path=GRADIO_ROOT_PATH, ) # #endregion AgentChat.Run diff --git a/agent/tests/test_agent/test_run.py b/agent/tests/test_agent/test_run.py index 128cf2ee..f1b721fa 100644 --- a/agent/tests/test_agent/test_run.py +++ b/agent/tests/test_agent/test_run.py @@ -160,6 +160,7 @@ class TestMainBlock: patch('ss_tools.agent.langgraph_setup.init_checkpointer'), \ patch('ss_tools.agent.run.SERVICE_JWT', svc_jwt), \ patch('ss_tools.agent.run.GRADIO_SERVER_PORT', gradio_port), \ + patch('ss_tools.agent.run.GRADIO_ROOT_PATH', env_overrides.get("GRADIO_ROOT_PATH", "/api/agent/gradio")), \ patch('ss_tools.agent.run.GRADIO_ALLOW_PORT_FALLBACK', gradio_fallback): mock_asyncio_run.side_effect = lambda coro: coro.close() if hasattr(coro, "close") else None @@ -206,6 +207,7 @@ class TestMainBlock: result['set_jwt'].assert_not_called() result['configure'].assert_not_called() result['demo'].launch.assert_called_once() + assert result['demo'].launch.call_args.kwargs["root_path"] == "/api/agent/gradio" def test_main_block_with_service_jwt(self, monkeypatch): """Main block sets service JWT via ContextVar.""" diff --git a/backend/src/plugins/translate/_batch_proc.py b/backend/src/plugins/translate/_batch_proc.py index e53efe66..a48e757a 100644 --- a/backend/src/plugins/translate/_batch_proc.py +++ b/backend/src/plugins/translate/_batch_proc.py @@ -13,6 +13,7 @@ # @RATIONALE Extracted from TranslationExecutor. Batch insert delegated to _batch_insert.py. # @REJECTED Keeping batch processing inside TranslationExecutor — caused class to exceed INV_7. +import asyncio from datetime import UTC, datetime import time from typing import Any @@ -66,7 +67,7 @@ class BatchProcessingService: tls = [str(tls)] if not isinstance(tls, list) else tls # ★ Run local language detection on all rows (heuristic, no LLM) - self._detect_languages(batch_rows, tls) + await self._detect_languages(batch_rows, tls) source_texts = [r.get("source_text", "") for r in batch_rows if r.get("source_text")] rc = batch_rows[0].get("source_data") if batch_rows else None @@ -135,14 +136,15 @@ class BatchProcessingService: # #region _detect_languages [C:2] [TYPE Function] [SEMANTICS translate,language,detect,lingua] # @ingroup Translate # @BRIEF Run local language detection on all batch rows (no LLM). - def _detect_languages(self, batch_rows: list[dict], target_languages: list[str]) -> None: + async def _detect_languages(self, batch_rows: list[dict], target_languages: list[str]) -> None: """Run local language detection on all batch rows (no LLM). Attaches '_detected_lang' (BCP-47 code or 'und') to each row dict. - Uses batch_detect() for efficient multi-text processing. + Uses batch_detect() for efficient multi-text processing, offloaded + to a thread to avoid blocking the event loop. """ texts = [row.get("source_text", "") for row in batch_rows] - results = batch_detect(texts, target_languages) + results = await asyncio.to_thread(batch_detect, texts, target_languages) for row, lang in zip(batch_rows, results): row["_detected_lang"] = lang # #endregion _detect_languages diff --git a/backend/src/plugins/translate/executor.py b/backend/src/plugins/translate/executor.py index fec5975f..83c43e03 100644 --- a/backend/src/plugins/translate/executor.py +++ b/backend/src/plugins/translate/executor.py @@ -18,6 +18,7 @@ # @REJECTED Keeping monolithic executor.py at 1974 lines — violates INV_7 by +1574 lines. # Single monolithic LLM call — would lose all progress on any failure. +import asyncio from collections.abc import Callable from datetime import UTC, datetime from typing import Any @@ -145,6 +146,7 @@ class TranslationExecutor: run.skipped_records = skipped_records run.cache_hits = cache_hits self.db.commit() + await asyncio.sleep(0) # yield event loop to prevent starvation (499 on health probes) batch_id = batch_result.get("batch_id") if batch_id and batch_result.get("successful", 0) > 0: diff --git a/backend/tests/plugins/translate/test_batch_proc.py b/backend/tests/plugins/translate/test_batch_proc.py index d7c2363e..d52d537d 100644 --- a/backend/tests/plugins/translate/test_batch_proc.py +++ b/backend/tests/plugins/translate/test_batch_proc.py @@ -83,16 +83,37 @@ class TestCreateBatch: class TestDetectLanguages: """Verify _detect_languages.""" - def test_detects_languages(self, db_session): + @pytest.mark.asyncio + async def test_detects_languages(self, db_session): config = MagicMock() svc = BatchProcessingService(db_session, config) rows = _batch_rows(2) with patch('src.plugins.translate._batch_proc.batch_detect', return_value=["en", "fr"]): - svc._detect_languages(rows, ["fr", "de"]) + await svc._detect_languages(rows, ["fr", "de"]) assert rows[0]["_detected_lang"] == "en" assert rows[1]["_detected_lang"] == "fr" + @pytest.mark.asyncio + async def test_detect_languages_offloads_to_thread(self, db_session): + """Regression: CPU-bound lingua batch detection must not run on event loop.""" + config = MagicMock() + svc = BatchProcessingService(db_session, config) + rows = _batch_rows(2) + + async def fake_to_thread(fn, texts, target_languages): + assert fn.__name__ == "batch_detect" + assert texts == ["Hello world 0", "Hello world 1"] + assert target_languages == ["fr"] + return ["en", "en"] + + with patch('src.plugins.translate._batch_proc.asyncio.to_thread', + new=AsyncMock(side_effect=fake_to_thread)) as mock_to_thread: + await svc._detect_languages(rows, ["fr"]) + + mock_to_thread.assert_awaited_once() + assert [row["_detected_lang"] for row in rows] == ["en", "en"] + class TestCheckCache: """Verify _check_cache.""" diff --git a/backend/tests/plugins/translate/test_executor.py b/backend/tests/plugins/translate/test_executor.py index 31c07229..d99abb46 100644 --- a/backend/tests/plugins/translate/test_executor.py +++ b/backend/tests/plugins/translate/test_executor.py @@ -230,6 +230,32 @@ class TestProcessBatches: await executor._process_batches(run, job, batches, ["fr"]) callback.assert_called_once() + @pytest.mark.asyncio + async def test_process_batches_yields_after_commit(self, db_with_run): + """Regression: batch loop yields to event loop after sync DB commit.""" + session, run_id = db_with_run + config = MagicMock() + executor = TranslationExecutor(session, config) + job = _make_job(session) + run = session.query(TranslationRun).filter(TranslationRun.id == run_id).first() + + batches = [ + [{"row_index": "0", "source_text": "Hello", "source_data": {"table": "t"}}], + [{"row_index": "1", "source_text": "World", "source_data": {"table": "t"}}], + ] + + with patch.object(executor, '_process_batch', new=AsyncMock(side_effect=[ + {"successful": 1, "failed": 0, "skipped": 0, "retries": 0, "cache_hits": 0, "batch_id": "batch-1"}, + {"successful": 1, "failed": 0, "skipped": 0, "retries": 0, "cache_hits": 0, "batch_id": "batch-2"}, + ])): + with patch.object(executor, '_insert_batch_to_target', new=AsyncMock(return_value=None)): + with patch('src.plugins.translate.executor.asyncio.sleep', new=AsyncMock()) as mock_sleep: + result = await executor._process_batches(run, job, batches, ["fr"]) + + assert result.status == "COMPLETED" + assert mock_sleep.await_count == 2 + mock_sleep.assert_any_await(0) + class TestFinalizeRun: """Verify _finalize_run.""" diff --git a/backend/tests/test_gradio_proxy_config.py b/backend/tests/test_gradio_proxy_config.py new file mode 100644 index 00000000..de89c051 --- /dev/null +++ b/backend/tests/test_gradio_proxy_config.py @@ -0,0 +1,70 @@ +# #region Test.Infrastructure.GradioProxy [C:3] [TYPE Module] [SEMANTICS test,nginx,docker,agent,gradio] +# @BRIEF Verify Gradio agent proxy contracts across nginx, compose, and agent launch config. +# @RELATION BINDS_TO -> [docker.nginx.conf] +# @RELATION BINDS_TO -> [docker.nginx.ssl.conf] +# @RELATION BINDS_TO -> [AgentChat.Config] +# @TEST_EDGE: docker_dns_name -> Proxy targets compose service name `agent`. +# @TEST_EDGE: prefix_forwarding -> Proxy preserves `/api/agent/gradio` for Gradio root_path. +# @TEST_EDGE: auth_forwarding -> Browser Authorization header is forwarded. +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] + + +def _read(relative_path: str) -> str: + return (PROJECT_ROOT / relative_path).read_text(encoding="utf-8") + + +def test_nginx_http_gradio_proxy_targets_agent_service_and_preserves_prefix(): + """HTTP nginx routes Gradio traffic to compose service `agent` without stripping root_path.""" + text = _read("docker/nginx.conf") + assert "location /api/agent/gradio/" in text + assert "set $agent_api http://agent:7860;" in text + assert "superset-tools-agent" not in text + assert "rewrite ^/api/agent/gradio" not in text + assert "proxy_set_header Authorization $http_authorization;" in text + assert "proxy_set_header Connection $connection_upgrade;" in text + assert "proxy_redirect http://agent:7860/ /api/agent/gradio/;" in text + + +def test_nginx_ssl_gradio_proxy_matches_http_contract(): + """SSL nginx keeps the same Gradio proxy invariants as HTTP nginx.""" + text = _read("docker/nginx.ssl.conf") + assert "map $http_upgrade $connection_upgrade" in text + assert "location /api/agent/gradio/" in text + assert "set $agent_api http://agent:7860;" in text + assert "superset-tools-agent" not in text + assert "rewrite ^/api/agent/gradio" not in text + assert "proxy_set_header Authorization $http_authorization;" in text + assert "proxy_set_header X-Forwarded-Proto https;" in text + + +def test_compose_frontend_depends_on_agent_and_agent_uses_root_path(): + """Compose starts agent before frontend and configures Gradio root_path.""" + for compose_path in ("docker-compose.yml", "docker-compose.enterprise-clean.yml"): + text = _read(compose_path) + assert " frontend:" in text + frontend_block = text.split(" frontend:", 1)[1].split("\n\n", 1)[0] + assert " - backend" in frontend_block + assert " - agent" in frontend_block + agent_block = text.split(" agent:", 1)[1] + assert "GRADIO_ROOT_PATH: /api/agent/gradio" in agent_block + + +def test_build_script_generated_enterprise_compose_keeps_agent_proxy_contract(): + """Release bundle compose heredocs preserve frontend->agent dependency and root_path.""" + text = _read("build.sh") + assert text.count(" - agent") >= 2 + assert text.count("GRADIO_ROOT_PATH: /api/agent/gradio") >= 2 + + +def test_agent_launch_uses_default_gradio_root_path(): + """Agent runtime exports `/api/agent/gradio` as Gradio root_path for @gradio/client.""" + config_text = _read("agent/src/ss_tools/agent/_config.py") + run_text = _read("agent/src/ss_tools/agent/run.py") + app_text = _read("agent/src/ss_tools/agent/app.py") + assert 'GRADIO_ROOT_PATH: str = os.getenv("GRADIO_ROOT_PATH", "/api/agent/gradio")' in config_text + assert "root_path=GRADIO_ROOT_PATH" in run_text + assert "root_path=GRADIO_ROOT_PATH" in app_text +# #endregion Test.Infrastructure.GradioProxy diff --git a/build.sh b/build.sh index 50912dd0..6104fa72 100755 --- a/build.sh +++ b/build.sh @@ -294,6 +294,7 @@ services: restart: unless-stopped depends_on: - backend + - agent environment: SSL_KEY_PASSPHRASE: \${SSL_KEY_PASSPHRASE:-} ports: @@ -317,6 +318,7 @@ services: SERVICE_JWT: \${SERVICE_JWT:?Set SERVICE_JWT in .env} DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools} GRADIO_SERVER_PORT: 7860 + GRADIO_ROOT_PATH: /api/agent/gradio ports: - "\${AGENT_HOST_PORT:-7860}:7860" DEPLOY @@ -610,6 +612,7 @@ services: restart: unless-stopped depends_on: - backend + - agent environment: SSL_KEY_PASSPHRASE: \${SSL_KEY_PASSPHRASE:-} ports: @@ -633,6 +636,7 @@ services: SERVICE_JWT: \${SERVICE_JWT:?Set SERVICE_JWT in .env} DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools} GRADIO_SERVER_PORT: 7860 + GRADIO_ROOT_PATH: /api/agent/gradio ports: - "\${AGENT_HOST_PORT:-7860}:7860" DEPLOY diff --git a/docker-compose.enterprise-clean.yml b/docker-compose.enterprise-clean.yml index 78156045..1e3ad9c8 100644 --- a/docker-compose.enterprise-clean.yml +++ b/docker-compose.enterprise-clean.yml @@ -74,6 +74,7 @@ services: restart: unless-stopped depends_on: - backend + - agent environment: # Пароль от приватного ключа/PKCS#12 для SSL терминации nginx. # Опционально: если server.key не зашифрован или используется HTTP — не требуется. @@ -105,6 +106,7 @@ services: SERVICE_JWT: ${SERVICE_JWT:-agent-service-secret} DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools} GRADIO_SERVER_PORT: 7860 + GRADIO_ROOT_PATH: /api/agent/gradio LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "${AGENT_HOST_PORT:-7860}:7860" diff --git a/docker-compose.yml b/docker-compose.yml index 72735c41..1ca22bc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,6 +64,7 @@ services: SERVICE_JWT: ${SERVICE_JWT:-agent-service-secret} DATABASE_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools GRADIO_SERVER_PORT: 7860 + GRADIO_ROOT_PATH: /api/agent/gradio LLM_CA_CERT_URLS: ${LLM_CA_CERT_URLS:-} ports: - "7860" # internal only, proxied through nginx @@ -77,6 +78,7 @@ services: restart: unless-stopped depends_on: - backend + - agent ports: - "${FRONTEND_HOST_PORT:-8000}:80" diff --git a/docker/nginx.conf b/docker/nginx.conf index d22442b9..637eb2b3 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -1,3 +1,8 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 80; server_name _; @@ -10,6 +15,22 @@ server { try_files $uri $uri/ /index.html; } + # Gradio agent proxy — must be before /api/ (longest prefix match wins) + location /api/agent/gradio/ { + set $agent_api http://agent:7860; + proxy_pass $agent_api; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect http://agent:7860/ /api/agent/gradio/; + proxy_read_timeout 300s; + } + location /api/ { set $backend_api http://backend:8000; proxy_pass $backend_api; diff --git a/docker/nginx.ssl.conf b/docker/nginx.ssl.conf index 74789350..862965b9 100644 --- a/docker/nginx.ssl.conf +++ b/docker/nginx.ssl.conf @@ -13,6 +13,11 @@ # @REJECTED Прокси на HTTPS внутри compose-сети отвергнут — избыточно. # #endregion docker.nginx.ssl.conf +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + server { listen 80 default_server; server_name _; @@ -39,6 +44,22 @@ server { try_files $uri $uri/ /index.html; } + # Gradio agent proxy — must be before /api/ (longest prefix match wins) + location /api/agent/gradio/ { + set $agent_api http://agent:7860; + proxy_pass $agent_api; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header Authorization $http_authorization; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_redirect http://agent:7860/ /api/agent/gradio/; + proxy_read_timeout 300s; + } + location /api/ { set $backend_api http://backend:8000; proxy_pass $backend_api;