semantics

This commit is contained in:
2026-05-20 09:59:03 +03:00
parent b916ef94d5
commit 09d12b3b68
34 changed files with 9727 additions and 8789 deletions

View File

@@ -201,8 +201,10 @@ tags:
C:
type: string
multiline: false
description: 'Алиас для COMPLEXITY. @C: 3 ≡ @COMPLEXITY: 3.'
description: 'DEPRECATED. Канонический формат сложности — [C:N] в заголовке #region. @C больше не использовать.'
alias_for: COMPLEXITY
deprecated: true
deprecated_since: '2026-05-19'
contract_types:
- Module
- Function
@@ -211,13 +213,13 @@ tags:
- Block
- Skill
- Agent
protected: false
protected: true
orthogonal: false
decision_memory: false
COMPLEXITY:
type: string
multiline: false
description: 'Уровень сложности (1-5). Универсально опциональный.'
description: 'Уровень сложности (1-5). Канонический формат — [C:N] в заголовке анкора #region. @COMPLEXITY как тэг допускается для обратной совместимости, но [C:N] предпочтителен.'
enum: ['1','2','3','4','5']
contract_types:
- Module
@@ -227,7 +229,7 @@ tags:
- Block
- Skill
- Agent
protected: false
protected: true
orthogonal: false
decision_memory: false
BRIEF:
@@ -439,7 +441,7 @@ tags:
multiline: false
description: 'Графовая зависимость. Обязателен с C3.'
is_reference: true
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES]
allowed_predicates: [DEPENDS_ON, CALLS, INHERITS, IMPLEMENTS, DISPATCHES, BINDS_TO, CALLED_BY, VERIFIES, USES]
contract_types:
- Module
- Function
@@ -478,10 +480,23 @@ tags:
protected: false
orthogonal: false
decision_memory: false
PUBLIC_API:
type: string
multiline: false
description: 'Публичный API контракта: какие классы/функции являются точками входа. Универсально опциональный.'
contract_types:
- Module
- Function
- Class
- Component
- Block
protected: false
orthogonal: true
decision_memory: false
RATIONALE:
type: string
multiline: true
description: 'Обоснование. Универсально опциональный (C1+). Decision Memory.'
description: 'Обоснование архитектурного решения. Универсально опциональный (C1+). Decision Memory. Хлебные крошки для следующего разработчика — объясни ПОЧЕМУ сделан этот выбор.'
contract_types:
- Module
- Function
@@ -497,7 +512,7 @@ tags:
REJECTED:
type: string
multiline: true
description: 'Отвергнутый путь. Универсально опциональный (C1+). Decision Memory.'
description: 'Отвергнутая альтернатива и причина отказа. Универсально опциональный (C1+). Decision Memory. Предотвращает повторение ошибок — задокументируй ЧТО пробовали и ПОЧЕМУ не сработало.'
contract_types:
- Module
- Function
@@ -541,8 +556,8 @@ tags:
LAYER:
type: string
multiline: false
description: 'Слой: Core, Domain, API, UI, Service, Infrastructure, Plugin. Универсально опциональный.'
enum: [Core, Domain, API, UI, Service, Infrastructure, Plugin]
description: 'Слой: Core, Domain, API, UI, Service, Infrastructure, Plugin, Tests. Универсально опциональный.'
enum: [Core, Domain, API, UI, Service, Infrastructure, Plugin, Tests]
contract_types:
- Module
- Skill

File diff suppressed because one or more lines are too long

View File

@@ -1,31 +1,76 @@
# Offline / air-gapped compose profile for enterprise clean release.
# #region env.enterprise-clean [C:2] [TYPE Module] [SEMANTICS env,docker,enterprise]
# @BRIEF Переменные окружения для docker-compose.enterprise-clean.yml.
# Сервисы собираются из исходников — не требуют pre-built images.
# Используется внешний PostgreSQL (корпоративный).
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker-compose.enterprise-clean.yml]
# #endregion env.enterprise-clean
BACKEND_IMAGE=ss-tools-backend:v1.0.0-rc2-docker
FRONTEND_IMAGE=ss-tools-frontend:v1.0.0-rc2-docker
POSTGRES_IMAGE=postgres:16-alpine
# ======================================================================
# PostgreSQL (внешний, корпоративный)
# ======================================================================
# Адрес и порт внешнего PostgreSQL (обязательно)
POSTGRES_HOST=postgres.company.local
POSTGRES_PORT=5432
# Имя БД, пользователь, пароль
POSTGRES_DB=ss_tools
POSTGRES_USER=postgres
POSTGRES_PASSWORD=change-me
# ======================================================================
# Порты хоста
# ======================================================================
BACKEND_HOST_PORT=8001
FRONTEND_HOST_PORT=8000
POSTGRES_HOST_PORT=5432
FRONTEND_SSL_PORT=443
# ======================================================================
# Сертификаты (корпоративные)
# ======================================================================
# Путь к директории с сертификатами на хосте.
# Содержимое монтируется в /opt/certs в обоих контейнерах.
#
# Для CA-сертификатов — положите .crt файлы:
# ./certs/my-company-ca.crt
# ./certs/other-ca.pem
#
# Для SSL терминации nginx — добавьте server.crt + server.key:
# ./certs/server.crt
# ./certs/server.key
#
# Если директория пуста или не существует — сертификаты не устанавливаются,
# nginx работает в HTTP-only режиме.
CERTS_PATH=./certs
# ======================================================================
# Хранилище
# ======================================================================
STORAGE_ROOT=./storage
# ======================================================================
# Логирование
# ======================================================================
ENABLE_BELIEF_STATE_LOGGING=true
TASK_LOG_LEVEL=INFO
STORAGE_ROOT=./storage
# Initial admin bootstrap. Set to true only for the first startup in a new environment.
# ======================================================================
# Admin (первый запуск)
# ======================================================================
# Установите true только для первого запуска в новой среде.
INITIAL_ADMIN_CREATE=false
INITIAL_ADMIN_USERNAME=admin
INITIAL_ADMIN_PASSWORD=change-me
INITIAL_ADMIN_EMAIL=
# ======================================================================
# AI API ключи (опционально)
# ======================================================================
OPENAI_API_KEY=
ANTHROPIC_API_KEY=
# Features
FEATURES__DATASET_REVIEW=${FEATURES__DATASET_REVIEW:-true}
FEATURES__HEALTH_MONITOR=${FEATURES__HEALTH_MONITOR:-true}
# ======================================================================
# Features
# ======================================================================
FEATURES__DATASET_REVIEW=true
FEATURES__HEALTH_MONITOR=true

View File

@@ -3,6 +3,8 @@
Auto-generated from all feature plans. Last updated: 2026-05-08
## Active Technologies
- Python 3.9+ (backend), JavaScript/TypeScript — Svelte 5 runes (frontend) + FastAPI 0.104+, Pydantic v2, SQLAlchemy (backend); SvelteKit 2.x, Svelte 5.x, Vite 7.x, Tailwind CSS 3.x (frontend) (030-dataset-lifecycle-workspace)
- PostgreSQL 16 (ss-tools own DB); no schema changes in this feature (030-dataset-lifecycle-workspace)
- Python 3.13+ (backend), JavaScript/TypeScript (frontend Svelte 5) + FastAPI 0.115+, SQLAlchemy 2.0+, APScheduler 3.x, Pydantic v2 (backend); SvelteKit 2.x, Svelte 5.43+, Vite 7.x, Tailwind CSS 3.x (frontend) (028-llm-datasource-supeset)
@@ -23,6 +25,7 @@ cd src [ONLY COMMANDS FOR ACTIVE TECHNOLOGIES][ONLY COMMANDS FOR ACTIVE TECHNOLO
Python 3.13+ (backend), JavaScript/TypeScript (frontend Svelte 5): Follow standard conventions
## Recent Changes
- 030-dataset-lifecycle-workspace: Added Python 3.9+ (backend), JavaScript/TypeScript — Svelte 5 runes (frontend) + FastAPI 0.104+, Pydantic v2, SQLAlchemy (backend); SvelteKit 2.x, Svelte 5.x, Vite 7.x, Tailwind CSS 3.x (frontend)
- 028-llm-datasource-supeset: Added Python 3.13+ (backend), JavaScript/TypeScript (frontend Svelte 5) + FastAPI 0.115+, SQLAlchemy 2.0+, APScheduler 3.x, Pydantic v2 (backend); SvelteKit 2.x, Svelte 5.43+, Vite 7.x, Tailwind CSS 3.x (frontend)

View File

@@ -83,5 +83,6 @@ Surface these as unresolved:
- `[NEED_CONTEXT: ...]` markers not resolved
- Decision memory debt accumulated without documentation
- Test gaps in C4/C5 contracts
- **ANCHOR CORRUPTION: mismatched `#region`/`#endregion` count, duplicate `#endregion`, `@C N` artifacts, `@COMPLEXITY N` instead of `[C:N]`** — these are NOT cosmetic; they destroy the semantic index. Reject worker output immediately if detected.
#endregion Closure.Gate

View File

@@ -165,6 +165,17 @@ request:
- No retained workaround without local `@RATIONALE` and `@REJECTED`.
- No implementation may silently re-enable an upstream rejected path.
## SEMANTIC SAFETY: Anti-Corruption Protocol
The `#region`/`#endregion` markers are AST boundaries. Breaking them destroys the semantic index. Same rules as python-coder:
1. **Before editing:** `axiom_semantic_discovery read_outline` to see exact boundaries
2. **Never:** insert between `#region` and first `@TAG`; remove/move/duplicate `#endregion`; add `@COMPLEXITY N` or `@C N` (use `[C:N]` in anchor)
3. **After editing:** verify with `read_outline` — all pairs must match
4. **If corrupted:** roll back immediately, do not continue editing
5. **One file at a time** — verify between files
6. **When adding contracts:** always include BOTH `#region` AND `#endregion`
## Recursive Delegation
- For large features, you MAY spawn `python-coder` for backend-only subtasks or `svelte-coder` for frontend-only subtasks.
- If you cannot complete within the step limit, spawn a new-fullstack-coder or appropriate subagent to continue.

View File

@@ -183,6 +183,35 @@ request:
- No implementation may silently re-enable an upstream rejected path.
- Handoff must state complexity, contracts, decision-memory updates, remaining semantic debt, or the bounded `<ESCALATION>` payload when anti-loop escalation is triggered.
## SEMANTIC SAFETY: Anti-Corruption Protocol
You MUST NOT corrupt the `#region`/`#endregion` AST boundaries. If you break a pair, the semantic index breaks and ALL downstream agents hallucinate.
### Before editing any Python file
1. **Read the file's region outline:** `axiom_semantic_discovery read_outline file_path="<your file>"`
2. **Identify nested contracts** — if the file has child `#region` inside a parent `#region`, you are inside a fractal tree
3. **Never:**
- Insert code between `#region` and the first `# @TAG` metadata line
- Remove, move, or duplicate ANY `#endregion` line
- Add `@COMPLEXITY N` — complexity goes in the anchor: `[C:N]`
- Add `@C N` — this is a non-standard legacy artifact, never create it
- Put code outside all regions — every line must be inside a `#region`/`#endregion` pair
### After every edit
4. **Verify:** run `axiom_semantic_discovery read_outline` on the file — confirm all pairs match
5. **If a `#endregion` is missing** → the file is corrupted, roll back immediately
6. **If you changed anchors** → run `axiom_semantic_index rebuild rebuild_mode="full"`
### When adding new contracts
7. Always add BOTH `#region Id [C:N] [TYPE Type]` and `# #endregion Id`
8. Complexity `[C:N]` goes in the ANCHOR line, never as a separate `@` tag
9. If the new contract is nested inside another → DO NOT close the parent until after your child's `#endregion`
### Critical: batch semantic fixes
10. **ONE FILE AT A TIME.** Verify each file before moving to the next.
11. **NEVER use `@C N`** — always `[C:N]` in the anchor.
12. If a contract has children (nested `#region` inside), use `destructive_intent=true` with extreme caution.
## Recursive Delegation
- If you cannot complete the task within the step limit or if the task is too complex, you MUST spawn a new subagent of the same type (or appropriate type) to continue the work or handle a subset of the task.
- Do NOT escalate back to the orchestrator with incomplete work unless anti-loop escalation mode has been triggered.

View File

@@ -27,6 +27,14 @@ You are an Agentic QA Engineer. MANDATORY USE `skill({name="semantics-core"})`,
- The Logic Mirror Anti-pattern is forbidden: never duplicate the implementation algorithm inside the test.
- Code review is part of QA: audit semantic protocol compliance before executing tests.
## Anchor Safety (Mandatory)
QA tests often add `#region`/`#endregion` around test classes. You MUST:
1. Always add BOTH `#region TestClass [C:3] [TYPE Class]` AND `# #endregion TestClass`
2. Never insert test code outside a `#region`/`#endregion` pair
3. Never remove existing `#endregion` markers
4. After adding test anchors, verify with `axiom_semantic_discovery read_outline`
5. Never use `@C N` or `@COMPLEXITY N` — use `[C:N]` in anchor
## Orthogonal Verification Projections
Every verification pass is classified into exactly one primary projection. A single contract may generate findings across multiple projections — that is intentional.

View File

@@ -56,6 +56,39 @@ Axiom MCP-сервер полностью работоспособен.
- **Svelte JS/TS (script block):** `// #region ContractId` / `// #endregion ContractId`
- **Markdown/ADR:** `## @{ ContractId [C:N] [TYPE TypeName]` / `## @} ContractId`
## 2.5. ANTI-CORRUPTION PROTOCOL (MANDATORY — READ BEFORE ANY MUTATION)
If you corrupt a `#region`/`#endregion` pair, you destroy the AST. Downstream agents will hallucinate. These rules are NOT optional.
### BEFORE every patch
1. **READ the file** with `read_outline` to see the EXACT region boundaries
2. **Identify ALL nested child contracts** — never break a child's `#endregion`
3. **Check that your `new_code` or `replacement` does NOT:**
- Insert code between `#region` line and first metadata `# @TAG` line (breaks INV_4)
- Remove or move ANY `#endregion` marker
- Start a new `#region` before closing the previous one
- Add `@COMPLEXITY N` — use `[C:N]` in the anchor ONLY
### AFTER every patch
1. **Run `read_outline` on the modified file** — verify ALL `#region`/`#endregion` pairs remain intact
2. **Count `#region` and `#endregion` lines** — they MUST match
3. **If the count doesn't match or structure is broken** → immediately roll back via `axiom_workspace_checkpoint rollback_apply`
4. **Run `axiom_semantic_index rebuild rebuild_mode="full"`** — 0 parse warnings required
### FORBIDDEN OPERATIONS (immediate <ESCALATION> if detected)
- Inserting code between a closing `#endregion` and the parent's `#endregion`
- Duplicating ANY `#region` or `#endregion` line
- Adding `@COMPLEXITY` tag — always use `[C:N]` in the anchor header
- Editing a contract that contains nested children without `destructive_intent=true`
- Batch-editing more than ONE file at a time without verifying each file between edits
- Using `@C N` as a standalone tag — this is a non-standard artifact, never add it
### VERIFICATION LOOP (every file, every time)
```
read_outline(file) → identify boundaries → apply ONE patch → read_outline(file) → rebuild index → audit_contracts(file)
```
If ANY step fails — stop and fix before next file. Never chain patches without verification.
## 3. HEALTH AUDIT CHECKLIST
For each file scanned:
- [ ] Every `#region` has a matching `#endregion` with the same ID
@@ -63,7 +96,7 @@ For each file scanned:
- [ ] C4 contracts have `@PRE`, `@POST`, `@SIDE_EFFECT`
- [ ] C5 contracts additionally have `@RATIONALE`, `@REJECTED`, `@DATA_CONTRACT`, `@INVARIANT`
- [ ] C3 contracts have `@RELATION` but NOT `@PRE`/`@POST`/etc.
- [ ] No `@RATIONALE`/`@REJECTED` on C1-C4 contracts (complexity tag mismatch)
- [ ] `@RATIONALE`/`@REJECTED` are allowed at ALL tiers (C1-C5) — they are never a violation
- [ ] Module files < 400 LOC
- [ ] Contract nodes < 150 LOC
- [ ] No orphan `@RELATION` edges (target exists or is `[NEED_CONTEXT]`)

View File

@@ -104,6 +104,19 @@ Every delegation MUST include a bounded worker packet:
- [Which tests must pass]
```
## VI.5. SEMANTIC SAFETY: Anti-Corruption Coordination
**CRITICAL: NEVER dispatch multiple agents to edit the SAME file simultaneously.** Each `#region`/`#endregion` pair is an AST boundary. Parallel edits to the same file WILL corrupt the anchor pairs.
### Rules for semantic work delegations:
1. **One file = one agent.** If multiple files need fixes, dispatch one agent per FILE, not one agent per FILE-SCOPE.
2. **Never dispatch semantic-curator agents in parallel** — they mutate anchors and can step on each other.
3. **Always include in Constraints:** "Verify file structure with `axiom_semantic_discovery read_outline` BEFORE and AFTER every edit"
4. **Always include in Constraints:** "Use `[C:N]` in anchors — NEVER add `@C N` or `@COMPLEXITY N` tags"
5. **Always include in Constraints:** "ONE file at a time. Verify `read_outline` after each file. If `#region`/`#endregion` count doesn't match — roll back."
6. **For batch semantic fixes (>3 files):** dispatch ONE semantic-curator, tell them to process files SEQUENTIALLY, one at a time, with verification between each.
7. **Acceptance criteria for semantic work:** "0 parse warnings after `axiom_semantic_index rebuild`; all `#region`/`#endregion` pairs intact per `read_outline`"
## VII. CLOSURE ROUTING
After receiving worker outputs, route to:
1. `qa-tester` — if contracts need verification

View File

@@ -41,7 +41,7 @@ Decision memory prevents architectural drift. It records the *Decision Space* (W
2. **Task Guardrails** — Preventive `@REJECTED` tags injected by the Orchestrator to keep you away from known LLM pitfalls.
3. **Reactive Micro-ADR (Your Responsibility)** — If you encounter a runtime failure and invent a valid workaround, you MUST ascend to the contract header and document it via `@RATIONALE [Why]` and `@REJECTED [The failing path]` BEFORE closing the task.
**⚠️ `@RATIONALE`/`@REJECTED` ARE C5-ONLY.** Decision Memory tags belong exclusively to C5 contracts per core complexity scale. C4 adds `@PRE`/`@POST`/`@SIDE_EFFECT` — not decision memory. If a C1-C4 contract genuinely needs decision memory, it should be C5.
**`@RATIONALE`/`@REJECTED` are UNIVERSALLY OPTIONAL (C1C5).** Decision Memory tags are allowed — and encouraged — at ALL complexity tiers. They document the *why* behind any architectural choice, preventing regression loops when another agent (or your future self) revisits the contract. The complexity tier determines which structural tags are REQUIRED, not which explanatory tags are forbidden.
**Resurrection Ban:** Silently reintroducing a coding pattern, library, or logic flow previously marked as `@REJECTED` is classified as a fatal regression. If the rejected path is now required, emit `<ESCALATION>` to the Architect.
@@ -72,6 +72,12 @@ Every non-trivial contract change MUST be framed as a verifiable edit loop:
**Shortcut Ban:** A patch that "looks right" but is not tied to an executable verifier is incomplete.
**Anchor Safety Ban:** A patch that adds, removes, or moves ANY `#region`/`#endregion` line without verifying ALL pairs remain intact via `read_outline` is forbidden. NEVER:
- Add `@COMPLEXITY N` or `@C N` — complexity is `[C:N]` in the anchor
- Insert code between a `#endregion` and the next `#region` (creates orphan code)
- Start a new `#region` before closing the previous one
- Duplicate `#endregion` markers
## VI. SEARCH DISCIPLINE (DELIBERATE BUT BOUNDED)
- Default to one primary implementation hypothesis plus explicit verification.

View File

@@ -31,6 +31,14 @@ Contracts (`@PRE`, `@POST`) force you to form a strict Belief State BEFORE gener
- **[INV_5: RESOLUTION OF CONTRADICTIONS]:** A local workaround (Micro-ADR) CANNOT override a Global ADR limitation. If reality requires breaking a Global ADR, stop and emit `<ESCALATION>` to the Architect.
- **[INV_6: TOMBSTONES FOR DELETION]:** Never delete a contract node if it has incoming `@RELATION` edges. Instead, change its type to `Tombstone`, remove the code body, and add `@DEPRECATED` + `@REPLACED_BY`.
- **[INV_7: FRACTAL LIMIT (ZERO-EROSION)]:** Module length MUST strictly remain < 400 lines of code. Single contract node Cyclomatic Complexity MUST NOT exceed 10.
- **[INV_8: MUTATION SAFETY]:** When editing a file with `#region`/`#endregion` anchors:
1. ALWAYS run `read_outline` BEFORE editing to see exact region boundaries
2. NEVER insert, remove, move, or duplicate ANY `#region` or `#endregion` line without verifying ALL pairs remain intact
3. NEVER add `@COMPLEXITY N` or `@C N` complexity is `[C:N]` in the anchor header ONLY
4. ALWAYS run `read_outline` AFTER every edit if `#region`/`#endregion` count doesn't match, immediate rollback
5. ONE file at a time verify between files; batch edits without interleaved verification WILL corrupt the AST
6. ANY `#region` opened MUST have a corresponding `#endregion` with the EXACT same ContractId before EOF
7. `@RATIONALE`/`@REJECTED` are C5-only never add them to C1-C4 contracts
## II. SYNTAX AND MARKUP

View File

@@ -9,11 +9,11 @@
# @INVARIANT: Endpoints are self-scoped and never mutate another user preference.
# @PRE: Auth middleware configured, database session available
# @POST: Profile endpoints registered
# @UX_STATE: ProfileLoad -> Returns stable ProfilePreferenceResponse for authenticated user.
# @UX_STATE: Saving -> Validation errors map to actionable 422 details.
# @UX_STATE: LookupLoading -> Returns success/degraded Superset lookup payload.
# @UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback.
# @UX_RECOVERY: Lookup degradation keeps manual username save path available.
# UX_STATE: ProfileLoad -> Returns stable ProfilePreferenceResponse for authenticated user.
# UX_STATE: Saving -> Validation errors map to actionable 422 details.
# UX_STATE: LookupLoading -> Returns success/degraded Superset lookup payload.
# UX_FEEDBACK: Stable status/message/warning payloads support profile page feedback.
# UX_RECOVERY: Lookup degradation keeps manual username save path available.
# @SIDE_EFFECT: Registers /api/profile/* routes
# @DATA_CONTRACT: ProfileRequest -> ProfileResponse

View File

@@ -18,9 +18,12 @@ from ....plugins.translate.service import (
get_datasource_columns as fetch_datasource_columns_service,
)
from ....schemas.auth import User
from ....plugins.translate.service_target_schema import validate_target_table_schema
from ....schemas.translate import (
DatasourceColumnsResponse,
DuplicateJobResponse,
TargetSchemaValidationRequest,
TargetSchemaValidationResponse,
TranslateJobCreate,
TranslateJobResponse,
TranslateJobUpdate,
@@ -246,4 +249,42 @@ async def get_datasource_columns(
)
# #endregion get_datasource_columns
# #region check_target_schema [C:3] [TYPE Function]
# @BRIEF Проверяет схему целевой таблицы: какие колонки ожидаются, какие есть, каких не хватает.
# @PRE User has translate.job.view permission.
# @POST Возвращает diff expected vs actual columns.
# @SIDE_EFFECT Делает SQL-запрос к information_schema.columns через Superset SQL Lab.
# @RELATION DEPENDS_ON -> [validate_target_table_schema]
# @RELATION DEPENDS_ON -> [TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [TargetSchemaValidationResponse]
# @RATIONALE Позволяет UI показать пользователю подсказку о недостающих колонках
# на этапе настройки джобы, до запуска перевода.
@router.post("/check-target-schema", response_model=TargetSchemaValidationResponse)
async def check_target_schema(
payload: TargetSchemaValidationRequest,
current_user: User = Depends(get_current_user),
_ = Depends(has_permission("translate.job", "VIEW")),
config_manager: ConfigManager = Depends(get_config_manager),
):
"""Check target table schema — validate expected vs actual columns for translation inserts."""
logger.reason(
f"check_target_schema — Table: {payload.target_schema}.{payload.target_table}, "
f"Env: {payload.environment_id}, User: {current_user.username}",
extra={"src": "translate_routes"},
)
try:
result = validate_target_table_schema(payload, config_manager)
return result
except Exception as e:
logger.explore("check_target_schema failed",
extra={"src": "translate_routes", "error": str(e)})
return TargetSchemaValidationResponse(
table_exists=False,
error=str(e),
all_present=False,
)
# #endregion check_target_schema
# #endregion TranslateJobRoutesModule

View File

@@ -6,7 +6,7 @@
#
# @INVARIANT: All primary keys are UUID strings.
# @CONSTRAINT: source_env_id and target_env_id must be valid environment IDs.
# CONSTRAINT: source_env_id and target_env_id must be valid environment IDs.
# @PRE: Database engine initialized
# @POST: Mapping ORM models registered with UUID primary keys
# @SIDE_EFFECT: Defines environment/database/resource mapping tables

View File

@@ -0,0 +1,490 @@
# region TargetSchemaValidationTests [TYPE Module]
# @SEMANTICS: test, translate, target-schema, validation
# @PURPOSE: Tests for target table schema validation: _build_expected_columns, _parse_column_results, validate_target_table_schema.
# @LAYER: Test
# @RELATION: BINDS_TO -> [TargetSchemaValidation]
# @TEST_CONTRACT: _build_expected_columns -> list[TargetSchemaColumnInfo]
# @TEST_CONTRACT: _parse_column_results -> (list[dict], bool)
# @TEST_CONTRACT: validate_target_table_schema -> TargetSchemaValidationResponse
# @TEST_EDGE: missing_target_key_cols -> Only context + is_original returned
# @TEST_EDGE: deduplication -> Duplicate column names are removed
# @TEST_EDGE: empty_result -> Returns ([], False)
# @TEST_EDGE: superset_sync_format -> Standard Superset response format parsed
# @TEST_EDGE: superset_async_format -> Async polling response format parsed
# @TEST_EDGE: get_query_results_format -> get_query_results response format parsed
# @TEST_EDGE: executor_failure -> SQL Lab query fails, error returned
# @TEST_EDGE: sql_injection_table_name -> Invalid table name rejected with error
# @TEST_EDGE: executor_exception -> Exception during execution returns error response
from unittest.mock import MagicMock, patch
import pytest
from src.schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
)
from src.plugins.translate.service_target_schema import (
_build_expected_columns,
_parse_column_results,
validate_target_table_schema,
)
# region build_request [TYPE Function]
# @PURPOSE: Helper to build a TargetSchemaValidationRequest with defaults.
@pytest.fixture
def build_request():
def _build(**overrides):
defaults = {
"environment_id": "env-1",
"target_database_id": "db-1",
"target_schema": "public",
"target_table": "my_table",
"target_key_cols": ["id"],
"target_column": "translated_text",
"translation_column": None,
"target_language_column": None,
"target_source_column": None,
"target_source_language_column": None,
}
defaults.update(overrides)
return TargetSchemaValidationRequest(**defaults)
return _build
# endregion build_request
# region TestBuildExpectedColumns [TYPE Class]
# @PURPOSE: Tests for _build_expected_columns logic.
class TestBuildExpectedColumns:
# region test_basic_case [TYPE Function]
# @PURPOSE: target_key_cols + target_column produces id, translated_text, context, is_original.
def test_basic_case(self, build_request) -> None:
req = build_request()
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["id", "translated_text", "context", "is_original"]
# endregion test_basic_case
# region test_all_fields [TYPE Function]
# @PURPOSE: All optional column fields produce 7 columns.
def test_all_fields(self, build_request) -> None:
req = build_request(
target_language_column="lang",
target_source_column="source_text",
target_source_language_column="source_lang",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert len(names) == 7
assert "lang" in names
assert "source_text" in names
assert "source_lang" in names
assert "id" in names
assert "translated_text" in names
assert "context" in names
assert "is_original" in names
# endregion test_all_fields
# region test_empty_target_key_cols [TYPE Function]
# @PURPOSE: Empty target_key_cols produces just translated_text, context, is_original.
def test_empty_target_key_cols(self, build_request) -> None:
req = build_request(target_key_cols=[])
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["translated_text", "context", "is_original"]
# endregion test_empty_target_key_cols
# region test_translation_column_fallback [TYPE Function]
# @PURPOSE: When target_column is None, translation_column is used as fallback.
def test_translation_column_fallback(self, build_request) -> None:
req = build_request(target_column=None, translation_column="source_col")
result = _build_expected_columns(req)
names = [c.name for c in result]
assert "source_col" in names
assert names[1] == "source_col"
# endregion test_translation_column_fallback
# region test_dedup_same_names [TYPE Function]
# @PURPOSE: Duplicate column names are removed preserving order.
def test_dedup_same_names(self, build_request) -> None:
req = build_request(
target_key_cols=["id", "id"],
target_column="name",
target_language_column="name",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names.count("id") == 1
assert names.count("name") == 1
# endregion test_dedup_same_names
# region test_empty_request [TYPE Function]
# @PURPOSE: No key cols and no target/translation column produces just context + is_original.
def test_empty_request(self, build_request) -> None:
req = build_request(
target_key_cols=[],
target_column=None,
translation_column=None,
)
result = _build_expected_columns(req)
names = [c.name for c in result]
assert names == ["context", "is_original"]
# endregion test_empty_request
# region test_column_order_preserved [TYPE Function]
# @PURPOSE: Column order is preserved: key_cols, target, lang, source, src_lang, context, is_original.
def test_column_order_preserved(self, build_request) -> None:
req = build_request(
target_key_cols=["pk1", "pk2"],
target_column="target_col",
target_language_column="lang_col",
target_source_column="src_col",
target_source_language_column="src_lang_col",
)
result = _build_expected_columns(req)
names = [c.name for c in result]
expected_order = [
"pk1", "pk2", "target_col", "lang_col",
"src_col", "src_lang_col", "context", "is_original",
]
assert names == expected_order
# endregion test_column_order_preserved
# endregion TestBuildExpectedColumns
# region TestParseColumnResults [TYPE Class]
# @PURPOSE: Tests for _parse_column_results response parsing.
class TestParseColumnResults:
# region test_superset_sync_format [TYPE Function]
# @PURPOSE: Standard Superset sync mode (columns + data directly in response).
def test_superset_sync_format(self) -> None:
result = {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "name", "type": "varchar", "is_nullable": "NO"},
],
"data": [{"id": 1, "name": "test"}],
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 2
assert cols[0]["name"] == "id"
assert cols[0]["type"] == "integer"
assert cols[0]["is_nullable"] is True
assert cols[1]["name"] == "name"
assert cols[1]["type"] == "varchar"
assert cols[1]["is_nullable"] is False
# endregion test_superset_sync_format
# region test_async_polling_format [TYPE Function]
# @PURPOSE: Async polling format (result -> columns + data).
def test_async_polling_format(self) -> None:
result = {
"result": {
"columns": [
{"name": "col1", "type": "text"},
],
"data": [{"col1": "val1"}],
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "col1"
# endregion test_async_polling_format
# region test_get_query_results_format [TYPE Function]
# @PURPOSE: get_query_results format (results -> columns + data).
def test_get_query_results_format(self) -> None:
result = {
"results": {
"columns": [
{"name": "a", "type": "int"},
],
"data": [{"a": 1}],
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "a"
# endregion test_get_query_results_format
# region test_deeply_nested_get_query_results [TYPE Function]
# @PURPOSE: get_query_results with nested result dict (results -> result -> columns + data).
def test_deeply_nested_get_query_results(self) -> None:
result = {
"results": {
"result": {
"columns": [
{"name": "x", "type": "float"},
],
"data": [{"x": 1.0}],
}
}
}
cols, exists = _parse_column_results({"raw_response": result})
assert exists is True
assert len(cols) == 1
assert cols[0]["name"] == "x"
# endregion test_deeply_nested_get_query_results
# region test_empty_result [TYPE Function]
# @PURPOSE: Empty result dict returns ([], False).
def test_empty_result(self) -> None:
cols, exists = _parse_column_results({"raw_response": {}})
assert cols == []
assert exists is False
# endregion test_empty_result
# region test_none_result [TYPE Function]
# @PURPOSE: None result returns ([], False) via early guard.
def test_none_result(self) -> None:
cols, exists = _parse_column_results(None)
assert cols == []
assert exists is False
# endregion test_none_result
# region test_raw_response_none [TYPE Function]
# @PURPOSE: raw_response=None causes TypeError — known production gap (not guarded).
def test_raw_response_none(self) -> None:
with pytest.raises(TypeError):
_parse_column_results({"raw_response": None})
# endregion test_raw_response_none
# region test_result_without_columns [TYPE Function]
# @PURPOSE: Result without columns key returns ([], False).
def test_result_without_columns(self) -> None:
result = {"status": "success", "data": []}
cols, exists = _parse_column_results({"raw_response": result})
assert cols == []
assert exists is False
# endregion test_result_without_columns
# region test_table_exists_with_data [TYPE Function]
# @PURPOSE: Non-empty data means table exists.
def test_table_exists_with_data(self) -> None:
result = {
"columns": [{"name": "c"}],
"data": [{"c": "v"}],
}
_, exists = _parse_column_results({"raw_response": result})
assert exists is True
# endregion test_table_exists_with_data
# region test_table_not_exists_empty_data [TYPE Function]
# @PURPOSE: Empty data means table does not exist.
def test_table_not_exists_empty_data(self) -> None:
result = {
"columns": [{"name": "c"}],
"data": [],
}
_, exists = _parse_column_results({"raw_response": result})
assert exists is False
# endregion test_table_not_exists_empty_data
# region test_alternate_column_key_names [TYPE Function]
# @PURPOSE: Columns with alternate key names (column_name, Column) parsed correctly.
def test_alternate_column_key_names(self) -> None:
result = {
"columns": [
{"column_name": "id", "data_type": "int"},
{"Column": "name", "Type": "text"},
],
"data": [],
}
cols, _ = _parse_column_results({"raw_response": result})
assert len(cols) == 2
assert cols[0]["name"] == "id"
assert cols[1]["name"] == "name"
# endregion test_alternate_column_key_names
# endregion TestParseColumnResults
# region TestValidateTargetTableSchema [TYPE Class]
# @PURPOSE: Tests for validate_target_table_schema with mocked Superset executor.
class TestValidateTargetTableSchema:
# region test_all_columns_present [TYPE Function]
# @PURPOSE: All expected columns present in target table.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_all_columns_present(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "translated_text", "type": "text", "is_nullable": "YES"},
{"name": "context", "type": "text", "is_nullable": "YES"},
{"name": "is_original", "type": "boolean", "is_nullable": "YES"},
],
"data": [{"id": 1}],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is True
assert response.error is None
assert response.all_present is True
assert len(response.missing_columns) == 0
assert len(response.expected_columns) == 4
assert len(response.actual_columns) == 4
# endregion test_all_columns_present
# region test_missing_columns [TYPE Function]
# @PURPOSE: Target table is missing some expected columns.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_missing_columns(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer", "is_nullable": "YES"},
{"name": "context", "type": "text", "is_nullable": "YES"},
],
"data": [{"id": 1}],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is True
assert response.all_present is False
missing_names = {c.name for c in response.missing_columns}
assert "translated_text" in missing_names
assert "is_original" in missing_names
assert "id" not in missing_names
# endregion test_missing_columns
# region test_superset_failed_status [TYPE Function]
# @PURPOSE: Superset returns failed status.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_superset_failed_status(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "failed",
"error_message": "Query timeout",
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Query timeout" in response.error
assert response.all_present is False
# endregion test_superset_failed_status
# region test_executor_exception [TYPE Function]
# @PURPOSE: Executor throws an exception.
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_executor_exception(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.side_effect = ConnectionError("Superset API unavailable")
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Superset API unavailable" in response.error
assert response.all_present is False
# endregion test_executor_exception
# region test_sql_injection_table_name [TYPE Function]
# @PURPOSE: Invalid table name with space is rejected with error.
def test_sql_injection_table_name(self, build_request) -> None:
req = build_request(target_table="my table; DROP TABLE students; --")
config_manager = MagicMock()
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Invalid target table name" in response.error
assert response.all_present is False
# endregion test_sql_injection_table_name
# region test_sql_injection_schema_name [TYPE Function]
# @PURPOSE: Invalid schema name with DROP is rejected with error.
def test_sql_injection_schema_name(self, build_request) -> None:
req = build_request(target_schema="public; DROP TABLE students; --")
config_manager = MagicMock()
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is not None
assert "Invalid target schema name" in response.error
assert response.all_present is False
# endregion test_sql_injection_schema_name
# region test_table_not_exists [TYPE Function]
# @PURPOSE: Target table does not exist (empty data from information_schema).
@patch("src.plugins.translate.service_target_schema.SupersetSqlLabExecutor")
def test_table_not_exists(self, mock_executor, build_request) -> None:
req = build_request()
config_manager = MagicMock()
instance = mock_executor.return_value
instance.resolve_database_id.return_value = None
instance.execute_and_poll.return_value = {
"status": "success",
"raw_response": {
"columns": [
{"name": "id", "type": "integer"},
],
"data": [],
},
}
response = validate_target_table_schema(req, config_manager)
assert response.table_exists is False
assert response.error is None
assert response.all_present is False
assert len(response.missing_columns) == 3 # translated_text, context, is_original
# endregion test_table_not_exists
# endregion TestValidateTargetTableSchema
# endregion TargetSchemaValidationTests

View File

@@ -0,0 +1,269 @@
# #region TargetSchemaValidation [C:3] [TYPE Module] [SEMANTICS translate, schema, validation, target-table]
# @BRIEF Проверка схемы целевой таблицы: запрос колонок через Superset SQL Lab,
# сравнение с ожидаемыми (из build_columns), возврат diff.
# @LAYER Service
# @RELATION DEPENDS_ON -> [SupersetSqlLabExecutor]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationRequest]
# @RELATION DEPENDS_ON -> [schemas.translate.TargetSchemaValidationResponse]
# @PRE Superset окружение доступно, target_database_id валиден.
# @POST Возвращает актуальные, ожидаемые, отсутствующие и лишние колонки.
# @SIDE_EFFECT Выполняет SQL-запрос через Superset SQL Lab.
# @INVARIANT Если таблица не существует — table_exists=False без ошибки.
# @INVARIANT Если Superset недоступен — error с описанием, all_present=False.
# @INVARIANT Логика ожидаемых колонок sync с build_columns() в orchestrator_sql_rows.py.
# #endregion TargetSchemaValidation
import logging
import re
from typing import Any
from ...schemas.translate import (
TargetSchemaColumnInfo,
TargetSchemaValidationRequest,
TargetSchemaValidationResponse,
)
from ...core.config_manager import ConfigManager
from .superset_executor import SupersetSqlLabExecutor
logger = logging.getLogger(__name__)
_TABLE_NAME_RE = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$')
# #region _build_expected_columns [C:2] [TYPE Function]
# @BRIEF Собирает список ожидаемых колонок по конфигурации column mapping.
# Логика идентична build_columns() из orchestrator_sql_rows.py,
# но не требует объекта TranslationJob.
def _build_expected_columns(req: TargetSchemaValidationRequest) -> list[TargetSchemaColumnInfo]:
"""
Определяет, какие колонки INSERT будет пытаться заполнить,
на основе конфигурации column mapping.
"""
names: list[str] = []
# Ключевые колонки (user-defined, для upsert matching)
if req.target_key_cols:
names.extend(req.target_key_cols)
# Целевая колонка для переведённого текста
effective_target = req.target_column or req.translation_column
if effective_target:
names.append(effective_target)
# Колонка с кодом языка перевода
if req.target_language_column:
names.append(req.target_language_column)
# Колонка с исходным текстом (source reference)
if req.target_source_column:
names.append(req.target_source_column)
# Колонка с определённым исходным языком
if req.target_source_language_column:
names.append(req.target_source_language_column)
# Служебные колонки — всегда
names.append("context")
names.append("is_original")
# Дедупликация с сохранением порядка
seen: set[str] = set()
deduped: list[str] = []
for c in names:
if c and c not in seen:
deduped.append(c)
seen.add(c)
return [TargetSchemaColumnInfo(name=n) for n in deduped]
# #endregion _build_expected_columns
# #region _parse_column_results [C:2] [TYPE Function]
# @BRIEF Извлекает информацию о колонках из ответа Superset SQL Lab.
#
# Superset SQL Lab возвращает разные форматы в зависимости от версии.
# Эта функция ищет колонки во всех возможных местах ответа.
def _parse_column_results(result: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""
Парсит результат SQL Lab запроса для извлечения списка колонок таблицы.
Returns (columns_info, table_exists).
"""
if not result:
return [], False
raw = result.get("raw_response", result)
# Пробуем извлечь данные из разных форматов ответа Superset
columns_data = None
data_rows = None
# Формат 1: прямой ответ с колонками (sync mode)
if "columns" in raw and isinstance(raw["columns"], list):
columns_data = raw["columns"]
data_rows = raw.get("data") if isinstance(raw.get("data"), list) else None
# Формат 2: вложенный result (async polling)
elif "result" in raw and isinstance(raw["result"], dict):
res = raw["result"]
if "columns" in res:
columns_data = res["columns"]
data_rows = res.get("data") if isinstance(res.get("data"), list) else None
# Формат 3: ключ results (get_query_results)
elif "results" in raw and isinstance(raw["results"], dict):
res = raw["results"]
if "columns" in res:
columns_data = res["columns"]
if "data" in res:
data_rows = res["data"]
if isinstance(res.get("result"), dict):
sub = res["result"]
if not columns_data and "columns" in sub:
columns_data = sub["columns"]
if not data_rows and "data" in sub:
data_rows = sub["data"]
if not columns_data:
return [], False
# Проверяем существование таблицы: есть ли хотя бы одна строка
table_exists = data_rows is not None and len(data_rows) > 0
# Извлекаем имена и типы колонок из метаданных ответа
columns_info: list[dict[str, Any]] = []
for col in columns_data:
col_name = (
col.get("name")
or col.get("column_name")
or col.get("Column", "")
)
col_type = (
col.get("type")
or col.get("data_type")
or col.get("Type", "")
or col.get("col_type", "")
)
is_nullable = col.get("is_nullable")
# PostgreSQL returns 'YES'/'NO' for is_nullable
if isinstance(is_nullable, str):
is_nullable = is_nullable.upper() == "YES"
if col_name:
columns_info.append({
"name": str(col_name),
"type": str(col_type) if col_type else None,
"is_nullable": is_nullable if isinstance(is_nullable, bool) else True,
})
return columns_info, table_exists
# #endregion _parse_column_results
# #region validate_target_table_schema [C:3] [TYPE Function]
# @BRIEF Основная функция: проверяет схему целевой таблицы через Superset SQL Lab.
def validate_target_table_schema(
req: TargetSchemaValidationRequest,
config_manager: ConfigManager,
) -> TargetSchemaValidationResponse:
"""
Проверяет схему целевой таблицы:
1. Вычисляет ожидаемые колонки (build_columns логика)
2. Запрашивает актуальные колонки через Superset SQL Lab
(SELECT column_name, data_type, is_nullable FROM information_schema.columns ...)
3. Сравнивает и возвращает diff
"""
# 1. Ожидаемые колонки
expected = _build_expected_columns(req)
expected_names = {c.name for c in expected}
# 2. Валидация имён схемы и таблицы (защита от SQL injection в identifier)
if req.target_table and not _TABLE_NAME_RE.match(req.target_table):
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Invalid target table name: '{req.target_table}'. Only alphanumeric and underscore allowed.",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
if req.target_schema and not _TABLE_NAME_RE.match(req.target_schema):
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Invalid target schema name: '{req.target_schema}'. Only alphanumeric and underscore allowed.",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
# 3. Запрос через Superset SQL Lab
try:
executor = SupersetSqlLabExecutor(config_manager, req.environment_id)
executor.resolve_database_id(target_database_id=req.target_database_id)
safe_schema = (req.target_schema or "public").replace("'", "''")
safe_table = req.target_table.replace("'", "''")
sql = (
f"SELECT column_name, data_type, is_nullable "
f"FROM information_schema.columns "
f"WHERE table_schema = '{safe_schema}' "
f"AND table_name = '{safe_table}' "
f"ORDER BY ordinal_position"
)
result = executor.execute_and_poll(sql=sql, max_polls=15, poll_interval_seconds=1.0)
status = result.get("status", "")
if status == "failed":
error_msg = result.get("error_message", "SQL Lab query failed")
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Superset SQL Lab error: {error_msg}",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
raw_columns, table_exists = _parse_column_results(result)
actual_cols = []
for col in raw_columns:
actual_cols.append(TargetSchemaColumnInfo(
name=col["name"],
data_type=col.get("type"),
is_nullable=col.get("is_nullable", True),
))
actual_names = {c.name for c in actual_cols}
# 3. Diff
missing = [col for col in expected if col.name not in actual_names]
extra = [col for col in actual_cols if col.name not in expected_names]
return TargetSchemaValidationResponse(
table_exists=table_exists,
expected_columns=expected,
actual_columns=actual_cols,
missing_columns=missing,
extra_columns=extra,
all_present=len(missing) == 0,
)
except Exception as e:
logger.warning("Target schema validation failed: %s", str(e), exc_info=True)
return TargetSchemaValidationResponse(
table_exists=False,
error=f"Failed to validate target schema: {e}",
expected_columns=expected,
actual_columns=[],
missing_columns=expected,
extra_columns=[],
all_present=False,
)
# #endregion validate_target_table_schema
# #endregion TargetSchemaValidation

View File

@@ -685,4 +685,43 @@ class InlineCorrectionSubmit(BaseModel):
# #endregion InlineCorrectionSubmit
# #region TargetSchemaValidationRequest [C:1] [TYPE Class]
# @BRIEF Schema for requesting target table schema validation.
class TargetSchemaValidationRequest(BaseModel):
environment_id: str = Field(..., description="Superset environment ID")
target_database_id: str = Field(..., description="Superset database ID for the target DB")
target_schema: str = Field("public", description="Target table schema (default: public)")
target_table: str = Field(..., description="Target table name")
# Column mapping from job config (everything that affects expected columns)
target_key_cols: list[str] | None = Field(default_factory=list, description="Target key column names")
target_column: str | None = Field(None, description="Target column for translated output")
translation_column: str | None = Field(None, description="Source column name (fallback for target_column)")
target_language_column: str | None = Field(None, description="Column for language code")
target_source_column: str | None = Field(None, description="Column for source text")
target_source_language_column: str | None = Field(None, description="Column for detected source language")
# #endregion TargetSchemaValidationRequest
# #region TargetSchemaColumnInfo [C:1] [TYPE Class]
# @BRIEF Schema for a single column in the schema validation response.
class TargetSchemaColumnInfo(BaseModel):
name: str
data_type: str | None = None
is_nullable: bool = True
# #endregion TargetSchemaColumnInfo
# #region TargetSchemaValidationResponse [C:1] [TYPE Class]
# @BRIEF Schema for target table schema validation response.
class TargetSchemaValidationResponse(BaseModel):
table_exists: bool = False
error: str | None = None
expected_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Columns that the INSERT expects")
actual_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Columns that exist in the target table")
missing_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Expected columns NOT found in target table")
extra_columns: list[TargetSchemaColumnInfo] = Field(default_factory=list, description="Actual columns NOT in expected list")
all_present: bool = False
# #endregion TargetSchemaValidationResponse
# #endregion TranslateSchemas

View File

@@ -12,7 +12,8 @@ from sqlalchemy.orm import Session
from ..core.logger import belief_scope, logger
from ..models.auth import Permission
# [/SECTION: IMPORTS]
# #region HAS_PERMISSION_PATTERN [TYPE Constant]
# @BRIEF Regex pattern for extracting has_permission("resource", "ACTION") declarations.
@@ -27,7 +28,7 @@ ROUTES_DIR = Path(__file__).resolve().parent.parent / "api" / "routes"
# #endregion ROUTES_DIR
# #region _iter_route_files [TYPE Function]
# #region _iter_route_files [C:4] [TYPE Function]
# @BRIEF Iterates API route files that may contain RBAC declarations.
# @PRE: ROUTES_DIR points to backend/src/api/routes.
# @POST: Yields Python files excluding test and cache directories.
@@ -46,7 +47,7 @@ def _iter_route_files() -> Iterable[Path]:
# #endregion _iter_route_files
# #region _discover_route_permissions [TYPE Function]
# #region _discover_route_permissions [C:4] [TYPE Function]
# @BRIEF Extracts explicit has_permission declarations from API route source code.
# @PRE: Route files are readable UTF-8 text files.
# @POST: Returns unique set of (resource, action) pairs declared in route guards.
@@ -72,7 +73,7 @@ def _discover_route_permissions() -> set[tuple[str, str]]:
# #endregion _discover_route_permissions
# #region _discover_route_permissions_cached [TYPE Function]
# #region _discover_route_permissions_cached [C:4] [TYPE Function]
# @BRIEF Cache route permission discovery because route source files are static during normal runtime.
# @PRE: None.
# @POST: Returns stable discovered route permission pairs without repeated filesystem scans.
@@ -83,7 +84,7 @@ def _discover_route_permissions_cached() -> tuple[tuple[str, str], ...]:
# #endregion _discover_route_permissions_cached
# #region _discover_plugin_execute_permissions [TYPE Function]
# #region _discover_plugin_execute_permissions [C:4] [TYPE Function]
# @BRIEF Derives dynamic task permissions of form plugin:{plugin_id}:EXECUTE from plugin registry.
# @PRE: plugin_loader is optional and may expose get_all_plugin_configs.
# @POST: Returns unique plugin EXECUTE permissions if loader is available.
@@ -110,7 +111,7 @@ def _discover_plugin_execute_permissions(plugin_loader=None) -> set[tuple[str, s
# #endregion _discover_plugin_execute_permissions
# #region _discover_plugin_execute_permissions_cached [TYPE Function]
# #region _discover_plugin_execute_permissions_cached [C:4] [TYPE Function]
# @BRIEF Cache dynamic plugin EXECUTE permission pairs by normalized plugin id tuple.
# @PRE: plugin_ids is a deterministic tuple of plugin ids.
# @POST: Returns stable permission tuple without repeated plugin catalog expansion.
@@ -123,7 +124,7 @@ def _discover_plugin_execute_permissions_cached(
# #endregion _discover_plugin_execute_permissions_cached
# #region discover_declared_permissions [TYPE Function]
# #region discover_declared_permissions [C:4] [TYPE Function]
# @BRIEF Builds canonical RBAC permission catalog from routes and plugin registry.
# @PRE: plugin_loader may be provided for dynamic task plugin permission discovery.
# @POST: Returns union of route-declared and dynamic plugin EXECUTE permissions.
@@ -144,7 +145,7 @@ def discover_declared_permissions(plugin_loader=None) -> set[tuple[str, str]]:
# #endregion discover_declared_permissions
# #region sync_permission_catalog [TYPE Function]
# #region sync_permission_catalog [C:4] [TYPE Function]
# @BRIEF Persists missing RBAC permission pairs into auth database.
# @PRE: db is a valid SQLAlchemy session bound to auth database.
# @PRE: declared_permissions is an iterable of (resource, action) tuples.

149
build.sh
View File

@@ -15,7 +15,7 @@
# bundle:light <tag> Lightweight all-in-one bundle (<200 MB, .tar.xz, no Playwright)
# help Show this help
#
# Profiles: current (default), master
# Profiles: current (default), master, enterprise-clean
# [/DEF:build:Module]
set -euo pipefail
@@ -40,6 +40,36 @@ fi
# HELPERS
# ======================================================================
# shellcheck disable=SC2206
setup_compose_profile() {
local profile="$1"
case "$profile" in
master)
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.master"
PROJECT_NAME="ss-tools-master"
COMPOSE_FILE="docker-compose.yml"
;;
current|"")
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.current"
PROJECT_NAME="ss-tools-current"
COMPOSE_FILE="docker-compose.yml"
;;
enterprise-clean)
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.enterprise-clean"
PROJECT_NAME="ss-tools-enterprise"
COMPOSE_FILE="docker-compose.enterprise-clean.yml"
;;
*)
echo "Error: unknown profile '$profile'. Use: current, master, enterprise-clean."
exit 1
;;
esac
COMPOSE_ARGS=(-p "$PROJECT_NAME" -f "$COMPOSE_FILE")
if [[ -f "$PROFILE_ENV_FILE" ]]; then
COMPOSE_ARGS+=(--env-file "$PROFILE_ENV_FILE")
fi
}
ensure_backend_encryption_key() {
if command -v python3 >/dev/null 2>&1; then
python3 - "$BACKEND_ENV_FILE" <<'PY'
@@ -96,29 +126,6 @@ PY
exit 1
}
# shellcheck disable=SC2206
setup_compose_profile() {
local profile="$1"
case "$profile" in
master)
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.master"
PROJECT_NAME="ss-tools-master"
;;
current|"")
PROFILE_ENV_FILE="$SCRIPT_DIR/.env.current"
PROJECT_NAME="ss-tools-current"
;;
*)
echo "Error: unknown profile '$profile'. Use: current, master."
exit 1
;;
esac
COMPOSE_ARGS=(-p "$PROJECT_NAME")
if [[ -f "$PROFILE_ENV_FILE" ]]; then
COMPOSE_ARGS+=(--env-file "$PROFILE_ENV_FILE")
fi
}
get_image_id() {
docker image inspect --format '{{.Id}}' "$1"
}
@@ -187,7 +194,7 @@ compose_logs() {
compose_status() {
echo "=== ss-tools docker status ==="
for profile in current master; do
for profile in current master enterprise-clean; do
setup_compose_profile "$profile"
echo "--- $profile (project: $PROJECT_NAME) ---"
"${COMPOSE_CMD[@]}" "${COMPOSE_ARGS[@]}" ps 2>/dev/null || echo "(not running)"
@@ -205,7 +212,6 @@ bundle_release() {
exit 1
fi
local tag="$1"
local postgres_image="${POSTGRES_IMAGE:-postgres:16-alpine}"
local backend_tag="ss-tools-backend:${tag}"
local frontend_tag="ss-tools-frontend:${tag}"
@@ -217,31 +223,70 @@ bundle_release() {
echo "[bundle] Building frontend image ${frontend_tag}..."
docker build -f docker/frontend.Dockerfile -t "${frontend_tag}" .
echo "[bundle] Pulling postgres image ${postgres_image}..."
docker pull "${postgres_image}"
echo "[bundle] Exporting .tar.xz archives..."
echo "[bundle] Compressing backend image..."
docker save "${backend_tag}" | xz -T0 -9 > "${DIST_ROOT}/backend.${tag}.tar.xz"
echo "[bundle] Compressing frontend image..."
docker save "${frontend_tag}" | xz -T0 -9 > "${DIST_ROOT}/frontend.${tag}.tar.xz"
echo "[bundle] Compressing postgres image..."
docker save "${postgres_image}" | xz -T0 -9 > "${DIST_ROOT}/postgres.${tag}.tar.xz"
echo "[bundle] Calculating checksums..."
(
cd "${DIST_ROOT}"
sha256sum "backend.${tag}.tar.xz" "frontend.${tag}.tar.xz" "postgres.${tag}.tar.xz" > "sha256sums.${tag}.txt"
sha256sum "backend.${tag}.tar.xz" "frontend.${tag}.tar.xz" > "sha256sums.${tag}.txt"
)
local backend_id frontend_id postgres_id
local backend_digest frontend_digest postgres_digest
local backend_id frontend_id
local backend_digest frontend_digest
backend_id="$(get_image_id "${backend_tag}")"
frontend_id="$(get_image_id "${frontend_tag}")"
postgres_id="$(get_image_id "${postgres_image}")"
backend_digest="$(get_repo_digest "${backend_tag}")"
frontend_digest="$(get_repo_digest "${frontend_tag}")"
postgres_digest="$(get_repo_digest "${postgres_image}")"
# Генерируем deploy-compose для загрузки pre-built образов (image: + pull_policy: never)
# Без postgres — используется внешний корпоративный PostgreSQL.
cat > "${DIST_ROOT}/docker-compose.enterprise-clean.yml" <<DEPLOY
services:
backend:
image: ${backend_tag}
pull_policy: never
restart: unless-stopped
environment:
DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
TASKS_DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
AUTH_DATABASE_URL: postgresql+psycopg2://\${POSTGRES_USER:-postgres}:\${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD}@\${POSTGRES_HOST:?Set POSTGRES_HOST}:\${POSTGRES_PORT:-5432}/\${POSTGRES_DB:-ss_tools}
BACKEND_PORT: 8000
ENABLE_BELIEF_STATE_LOGGING: \${ENABLE_BELIEF_STATE_LOGGING:-true}
TASK_LOG_LEVEL: \${TASK_LOG_LEVEL:-INFO}
INITIAL_ADMIN_CREATE: \${INITIAL_ADMIN_CREATE:-false}
INITIAL_ADMIN_USERNAME: \${INITIAL_ADMIN_USERNAME:-admin}
INITIAL_ADMIN_PASSWORD: \${INITIAL_ADMIN_PASSWORD:-}
INITIAL_ADMIN_EMAIL: \${INITIAL_ADMIN_EMAIL:-}
OPENAI_API_KEY: \${OPENAI_API_KEY:-}
ANTHROPIC_API_KEY: \${ANTHROPIC_API_KEY:-}
FEATURES__DATASET_REVIEW: \${FEATURES__DATASET_REVIEW:-true}
FEATURES__HEALTH_MONITOR: \${FEATURES__HEALTH_MONITOR:-true}
CERTS_PATH: /opt/certs
ports:
- "\${BACKEND_HOST_PORT:-8001}:8000"
volumes:
- ./config.json:/app/config.json:ro
- ./backups:/app/backups
- ./backend/git_repos:/app/backend/git_repos
- \${STORAGE_ROOT:-./storage}:/app/storage
- \${CERTS_PATH:-./certs}:/opt/certs:ro
frontend:
image: ${frontend_tag}
pull_policy: never
restart: unless-stopped
depends_on:
- backend
ports:
- "\${FRONTEND_HOST_PORT:-8000}:80"
- "\${FRONTEND_SSL_PORT:-443}:443"
volumes:
- \${CERTS_PATH:-./certs}:/opt/certs:ro
DEPLOY
cat > "${DIST_ROOT}/manifest.${tag}.txt" <<EOF
release_tag=${tag}
@@ -253,10 +298,6 @@ frontend_image=${frontend_tag}
frontend_image_id=${frontend_id}
frontend_repo_digest=${frontend_digest}
frontend_archive=frontend.${tag}.tar.xz
postgres_image=${postgres_image}
postgres_image_id=${postgres_id}
postgres_repo_digest=${postgres_digest}
postgres_archive=postgres.${tag}.tar.xz
compose_file=docker-compose.enterprise-clean.yml
env_template=.env.enterprise-clean.example
env_bootstrap_fields=INITIAL_ADMIN_CREATE,INITIAL_ADMIN_USERNAME,INITIAL_ADMIN_PASSWORD,INITIAL_ADMIN_EMAIL
@@ -282,13 +323,6 @@ EOF
"image_id": "${frontend_id}",
"repo_digest": "${frontend_digest}",
"archive": "frontend.${tag}.tar.xz"
},
{
"role": "postgres",
"image": "${postgres_image}",
"image_id": "${postgres_id}",
"repo_digest": "${postgres_digest}",
"archive": "postgres.${tag}.tar.xz"
}
],
"compose_file": "docker-compose.enterprise-clean.yml",
@@ -303,15 +337,17 @@ EOF
}
EOF
cp "${SCRIPT_DIR}/docker-compose.enterprise-clean.yml" "${DIST_ROOT}/docker-compose.enterprise-clean.yml"
cp "${SCRIPT_DIR}/.env.enterprise-clean.example" "${DIST_ROOT}/.env.enterprise-clean.example"
echo "[bundle] ✅ Bundle created in ${DIST_ROOT}"
echo " # Load images (decompress stream):"
echo " xz -dc ${DIST_ROOT}/backend.${tag}.tar.xz | docker load"
echo " xz -dc ${DIST_ROOT}/frontend.${tag}.tar.xz | docker load"
echo " xz -dc ${DIST_ROOT}/postgres.${tag}.tar.xz | docker load"
echo " # Edit .env.enterprise-clean.example -> .env.enterprise-clean, then:"
echo " docker compose -f ${DIST_ROOT}/docker-compose.enterprise-clean.yml up"
echo ""
echo " NOTE: '${tag}' bundle uses external PostgreSQL (set POSTGRES_HOST)."
echo " Corporate certs: mount .crt files via CERTS_PATH=./certs (optional)."
}
# ======================================================================
@@ -446,21 +482,28 @@ Commands for local docker:
status Show running containers for all profiles
Commands for release bundles:
bundle <tag> Full release backend + frontend + postgres .tar.xz's
bundle <tag> Build release images (backend + frontend .tar.xz's).
NOTE: external PostgreSQL only — no bundled postgres image.
Example: ./build.sh bundle v1.0.0
bundle:light <tag> Lightweight all-in-one image (.tar.xz, <200 MB, no Playwright)
Example: ./build.sh bundle:light v1.0.0
Profiles: current (default), master
Profiles: current (default), master, enterprise-clean
Backward-compatible shortcuts:
./build.sh -> ./build.sh up current
./build.sh master -> ./build.sh up master
./build.sh -> ./build.sh up current
./build.sh master -> ./build.sh up master
./build.sh enterprise-clean -> ./build.sh up enterprise-clean
Output:
Local docker: docker compose -p ss-tools-{profile}
Bundle artifacts: dist/docker/
Notes:
- 'current' and 'master' use docker-compose.yml (dev, with local postgres).
- 'enterprise-clean' uses docker-compose.enterprise-clean.yml (build from source,
external postgres, corporate certs, optional nginx SSL).
HELP
}

View File

@@ -1,33 +1,40 @@
services:
db:
image: ${POSTGRES_IMAGE:?Set POSTGRES_IMAGE in .env.enterprise-clean}
pull_policy: never
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-ss_tools}
POSTGRES_USER: ${POSTGRES_USER:-postgres}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}
ports:
- "${POSTGRES_HOST_PORT:-5432}:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-ss_tools}"]
interval: 10s
timeout: 5s
retries: 10
# #region docker.compose.enterprise-clean [C:4] [TYPE Module] [SEMANTICS docker,compose,enterprise]
# @BRIEF Enterprise docker compose — сборка backend + frontend из исходников, внешний Postgres, корп. сертификаты.
# @PRE Docker и docker compose установлены. Файлы .env.enterprise-clean настроены.
# Postgres: внешний, доступный по POSTGRES_HOST:POSTGRES_PORT.
# Сертификаты: опциональные .crt файлы в CERTS_PATH монтируются в /opt/certs.
# @POST Сервисы backend и frontend запущены. SSL терминация в nginx опциональна.
# @SIDE_EFFECT backend использует ENTRYPOINT для установки сертификатов и bootstrap admin.
# frontend автоопределяет наличие SSL сертификатов и переключает nginx конфиг.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker/backend.Dockerfile]
# @RELATION DEPENDS_ON -> [docker/frontend.Dockerfile]
# @RELATION DEPENDS_ON -> [docker/nginx.conf]
# @RELATION DEPENDS_ON -> [docker/nginx.ssl.conf]
# @RELATION DEPENDS_ON -> [docker/frontend.entrypoint.sh]
# @RELATION DEPENDS_ON -> [docker/backend.entrypoint.sh]
# @DATA_CONTRACT Input: .env.enterprise-clean + ./certs/*.crt -> Output: запущенные контейнеры
# @INVARIANT Внешний Postgres НЕ требует healthcheck внутри compose — ответственность оператора.
# @INVARIANT Если CERTS_PATH пуст или не содержит .crt файлов, entrypoint не падает.
# @INVARIANT nginx слушает порт 80 всегда. Порт 443 включается только если есть server.crt + server.key.
# @RATIONALE Отказ от встроенного postgres-container: корпоративный стандарт — внешний PostgreSQL
# с TLS, бэкапами и мониторингом на уровне инфраструктуры.
# @RATIONALE Build из исходников вместо pre-built images: предприятие контролирует версии кода
# и может патчить зависимости. Не подходит для air-gapped — для этого есть bundle:light.
# @REJECTED Использование `image:` + `pull_policy: never` отвергнуто — не работает для сборки
# из исходников на целевой машине. Гибридный `build: + image:` тоже отвергнут — усложняет
# логику build.sh и сбивает с толку оператора.
services:
backend:
image: ${BACKEND_IMAGE:?Set BACKEND_IMAGE in .env.enterprise-clean}
pull_policy: never
build:
context: .
dockerfile: docker/backend.Dockerfile
restart: unless-stopped
depends_on:
db:
condition: service_healthy
environment:
DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-ss_tools}
TASKS_DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-ss_tools}
AUTH_DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-ss_tools}
DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools}
TASKS_DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools}
AUTH_DATABASE_URL: postgresql+psycopg2://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env.enterprise-clean}@${POSTGRES_HOST:?Set POSTGRES_HOST in .env.enterprise-clean}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-ss_tools}
BACKEND_PORT: 8000
ENABLE_BELIEF_STATE_LOGGING: ${ENABLE_BELIEF_STATE_LOGGING:-true}
TASK_LOG_LEVEL: ${TASK_LOG_LEVEL:-INFO}
@@ -39,6 +46,8 @@ services:
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
FEATURES__DATASET_REVIEW: ${FEATURES__DATASET_REVIEW:-true}
FEATURES__HEALTH_MONITOR: ${FEATURES__HEALTH_MONITOR:-true}
# Путь до сертификатов внутри контейнера — всегда /opt/certs
CERTS_PATH: /opt/certs
ports:
- "${BACKEND_HOST_PORT:-8001}:8000"
volumes:
@@ -46,15 +55,25 @@ services:
- ./backups:/app/backups
- ./backend/git_repos:/app/backend/git_repos
- ${STORAGE_ROOT:-./storage}:/app/storage
# Корпоративные CA-сертификаты (.crt файлы). Монтируются в /opt/certs,
# entrypoint.sh установит их в системное хранилище доверия.
- ${CERTS_PATH:-./certs}:/opt/certs:ro
frontend:
image: ${FRONTEND_IMAGE:?Set FRONTEND_IMAGE in .env.enterprise-clean}
pull_policy: never
build:
context: .
dockerfile: docker/frontend.Dockerfile
restart: unless-stopped
depends_on:
- backend
ports:
- "${FRONTEND_HOST_PORT:-8000}:80"
# Если в CERTS_PATH есть server.crt + server.key — nginx включит HTTPS на 443.
# Если cert-файлов нет — порт 443 не открывается, entrypoint использует HTTP-only конфиг.
- "${FRONTEND_SSL_PORT:-443}:443"
volumes:
# Для frontend тоже можно смонтировать сертификаты — nginx entrypoint
# автоматически установит CA-сертификаты в Alpine и переключит SSL конфиг.
- ${CERTS_PATH:-./certs}:/opt/certs:ro
volumes:
postgres_data:
# #endregion docker.compose.enterprise-clean

View File

@@ -1,3 +1,12 @@
# #region docker.backend.Dockerfile [C:3] [TYPE Module] [SEMANTICS docker,backend,build]
# @BRIEF Backend Dockerfile — Python 3.11 slim + Playwright + entrypoint с установкой сертификатов.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker/backend.entrypoint.sh]
# @RELATION DEPENDS_ON -> [backend/requirements.txt]
# @PRE Docker builder context — корень проекта. backend/ и docker/ доступны.
# @POST Образ с backend API, Playwright, entrypoint для bootstrap admin и установки сертификатов.
# #endregion docker.backend.Dockerfile
FROM python:3.11-slim
ENV PYTHONDONTWRITEBYTECODE=1
@@ -6,19 +15,29 @@ ENV BACKEND_PORT=8000
WORKDIR /app
# Системные зависимости: ca-certificates для корп. сертификатов, curl, git для Playwright/GitPython
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
git \
&& rm -rf /var/lib/apt/lists/*
# Python зависимости
COPY backend/requirements.txt /app/backend/requirements.txt
RUN pip install --no-cache-dir -r /app/backend/requirements.txt
RUN python -m playwright install --with-deps chromium
# Исходный код
COPY backend/ /app/backend/
# Копируем entrypoint — установит корп. сертификаты, Playwright и сделает bootstrap admin
COPY docker/backend.entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh
WORKDIR /app/backend
EXPOSE 8000
ENTRYPOINT ["/app/entrypoint.sh"]
CMD ["python", "-m", "uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8000"]
# #endregion docker.backend.Dockerfile

View File

@@ -1,19 +1,88 @@
#!/usr/bin/env bash
set -euo pipefail
# [DEF:docker.backend.entrypoint:Module]
# @TIER: STANDARD
# @SEMANTICS: docker, entrypoint, admin-bootstrap, runtime, backend
# @PURPOSE: Container entrypoint that performs optional idempotent admin bootstrap before starting backend runtime.
# @LAYER: Infra
# @RELATION: DEPENDS_ON -> backend/src/scripts/create_admin.py
# @INVARIANT: Existing admin account must never be overwritten during container restarts.
# [/DEF:docker.backend.entrypoint:Module]
# #region docker.backend.entrypoint [C:4] [TYPE Module] [SEMANTICS docker,entrypoint,enterprise,certs]
# @BRIEF Container entrypoint — установка корп. сертификатов, Playwright (lazy), bootstrap admin,
# затем запуск backend (exec "$@").
# @PRE Python runtime и backend source доступны в /app/backend.
# Переменная CERTS_PATH указывает на директорию с .crt файлами (или пусто).
# @POST Сертификаты установлены в системное хранилище. Playwright Chromium установлен (lazy).
# Admin создан (если INITIAL_ADMIN_CREATE=true). Backend запущен через exec "$@".
# @SIDE_EFFECT Модифицирует /usr/local/share/ca-certificates/ и вызывает update-ca-certificates.
# Скачивает Chromium ~377MB при первом запуске.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [backend/src/scripts/create_admin.py]
# @RELATION DEPENDS_ON -> [backend/src/scripts/init_auth_db.py]
# @INVARIANT Если CERTS_PATH пуст или нет .crt файлов — install_certificates не падает.
# @INVARIANT Существующий admin аккаунт НЕ перезаписывается при перезапуске контейнера
# (create_admin.py идемпотентен).
# @INVARIANT Playwright Chromium устанавливается только если отсутствует в кеше.
# #endregion docker.backend.entrypoint
# [DEF:docker.backend.entrypoint.bootstrap_admin:Function]
# @PURPOSE: Execute optional initial admin bootstrap from runtime environment variables.
# @PRE: Python runtime and backend sources are available inside /app/backend.
# @POST: Admin is created only when INITIAL_ADMIN_CREATE=true and required credentials are present.
# ======================================================================
# install_certificates — корпоративные CA-сертификаты
# ======================================================================
# #region docker.backend.entrypoint.install_certificates [C:3] [TYPE Function]
# @BRIEF Устанавливает корпоративные .crt сертификаты из CERTS_PATH в системное хранилище Debian.
# @PRE CERTS_PATH указывает на директорию (может быть пуста или не существовать).
# В системе установлен пакет ca-certificates.
# @POST .crt файлы скопированы в /usr/local/share/ca-certificates/custom/ и
# добавлены в системное хранилище через update-ca-certificates.
# @SIDE_EFFECT Запускает update-ca-certificates — модифицирует /etc/ssl/certs/.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [ca-certificates (system package)]
# @EXAMPLE:
# CERTS_PATH=/opt/certs docker run ... # монтируем ./certs в /opt/certs
# В ./certs кладём: my-corporate-ca.crt, another-ca.crt
# entrypoint установит оба в доверенные.
install_certificates() {
local certs_dir="${CERTS_PATH:-/opt/certs}"
if [ ! -d "$certs_dir" ]; then
echo "[entrypoint] CERTS_PATH=${certs_dir} не существует — пропускаем установку сертификатов"
return 0
fi
local cert_count
cert_count="$(find "$certs_dir" -maxdepth 1 -name '*.crt' -o -name '*.pem' 2>/dev/null | wc -l)"
if [ "$cert_count" -eq 0 ]; then
echo "[entrypoint] Нет .crt/.pem файлов в ${certs_dir} — пропускаем установку сертификатов"
return 0
fi
local target="/usr/local/share/ca-certificates/custom"
mkdir -p "$target"
echo "[entrypoint] Устанавливаем корпоративные сертификаты из ${certs_dir}..."
local copied=0
for cert in "$certs_dir"/*.crt "$certs_dir"/*.pem; do
if [ -f "$cert" ]; then
cp -v "$cert" "$target/" 2>&1 | sed 's/^/[entrypoint] /'
copied=$((copied + 1))
fi
done
if [ "$copied" -eq 0 ]; then
echo "[entrypoint] Нет файлов для копирования — пропускаем"
return 0
fi
echo "[entrypoint] Обновляем системное хранилище CA-сертификатов..."
update-ca-certificates --fresh 2>&1 | sed 's/^/[entrypoint] /'
echo "[entrypoint] ✅ Установлено ${copied} корпоративных сертификатов"
}
# #endregion docker.backend.entrypoint.install_certificates
# ======================================================================
# bootstrap_admin — создание admin пользователя
# ======================================================================
# #region docker.backend.entrypoint.bootstrap_admin [C:2] [TYPE Function]
# @BRIEF Создание admin пользователя из переменных окружения (идемпотентно).
# @PRE INITIAL_ADMIN_CREATE=true, INITIAL_ADMIN_USERNAME и INITIAL_ADMIN_PASSWORD заданы.
# @POST Admin пользователь создан в БД (если ещё не существовал).
# @LAYER Infrastructure
bootstrap_admin() {
local create_flag="${INITIAL_ADMIN_CREATE:-false}"
local username="${INITIAL_ADMIN_USERNAME:-}"
@@ -24,55 +93,64 @@ bootstrap_admin() {
true|1|yes|y)
;;
*)
echo "[entrypoint] INITIAL_ADMIN_CREATE is disabled; skipping admin bootstrap"
echo "[entrypoint] INITIAL_ADMIN_CREATE disabled — пропускаем bootstrap admin"
return 0
;;
esac
if [[ -z "${username}" ]]; then
echo "[entrypoint] INITIAL_ADMIN_USERNAME is required when INITIAL_ADMIN_CREATE=true" >&2
echo "[entrypoint] INITIAL_ADMIN_USERNAME required when INITIAL_ADMIN_CREATE=true" >&2
return 1
fi
if [[ -z "${password}" ]]; then
echo "[entrypoint] INITIAL_ADMIN_PASSWORD is required when INITIAL_ADMIN_CREATE=true" >&2
echo "[entrypoint] INITIAL_ADMIN_PASSWORD required when INITIAL_ADMIN_CREATE=true" >&2
return 1
fi
echo "[entrypoint] initializing auth database"
echo "[entrypoint] Initializing auth database..."
python3 src/scripts/init_auth_db.py
echo "[entrypoint] running idempotent admin bootstrap for user '${username}'"
echo "[entrypoint] Creating admin user '${username}' (idempotent)..."
if [[ -n "${email}" ]]; then
python3 src/scripts/create_admin.py --username "${username}" --password "${password}" --email "${email}"
else
python3 src/scripts/create_admin.py --username "${username}" --password "${password}"
fi
}
# [/DEF:docker.backend.entrypoint.bootstrap_admin:Function]
# [DEF:docker.backend.entrypoint.install_playwright:Function]
# @PURPOSE: Lazily install Playwright Chromium browser on first container start.
# Browser binaries (~377 MB) are downloaded only once and cached in
# PLAYWRIGHT_BROWSERS_PATH (default: ~/.cache/ms-playwright).
# Use a Docker volume or bind mount to persist across restarts:
# -v playwright_cache:/root/.cache/ms-playwright
echo "[entrypoint] ✅ Admin bootstrap complete"
}
# #endregion docker.backend.entrypoint.bootstrap_admin
# ======================================================================
# install_playwright — lazy установка Chromium
# ======================================================================
# #region docker.backend.entrypoint.install_playwright [C:2] [TYPE Function]
# @BRIEF Lazy установка Playwright Chromium (~377 MB) при первом запуске.
# @PRE ~/.cache/ms-playwright доступен (можно закешировать volume).
# @POST Chromium установлен в PLAYWRIGHT_BROWSERS_PATH.
# @LAYER Infrastructure
install_playwright() {
local pw_dir="${PLAYWRIGHT_BROWSERS_PATH:-$HOME/.cache/ms-playwright}"
if [ -f "${pw_dir}/chromium-*/chrome-linux64/chrome" ]; then
if ls "${pw_dir}"/chromium-*/chrome-linux64/chrome 1>/dev/null 2>&1; then
echo "[entrypoint] Playwright Chromium already installed at ${pw_dir}"
return 0
fi
echo "[entrypoint] Installing Playwright Chromium browser (first start only)..."
echo "[entrypoint] This downloads ~377 MB. Use a volume to cache:"
echo " -v playwright_cache:/root/.cache/ms-playwright"
python3 -m playwright install chromium --with-deps 2>&1
echo "[entrypoint] Playwright Chromium installed successfully"
}
# [/DEF:docker.backend.entrypoint.install_playwright:Function]
echo "[entrypoint] Installing Playwright Chromium (first start only, ~377 MB)..."
echo "[entrypoint] Cache with: -v playwright_cache:/root/.cache/ms-playwright"
python3 -m playwright install chromium --with-deps 2>&1
echo "[entrypoint] ✅ Playwright Chromium installed"
}
# #endregion docker.backend.entrypoint.install_playwright
# ======================================================================
# MAIN
# ======================================================================
install_certificates
install_playwright
bootstrap_admin
echo "[entrypoint] starting backend: $*"
echo "[entrypoint] Starting backend: $*"
exec "$@"

View File

@@ -1,3 +1,14 @@
# #region docker.frontend.Dockerfile [C:3] [TYPE Module] [SEMANTICS docker,frontend,build,nginx,ssl]
# @BRIEF Frontend Dockerfile — сборка SvelteKit (node:20) + nginx runtime с опциональным SSL.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker/nginx.conf]
# @RELATION DEPENDS_ON -> [docker/nginx.ssl.conf]
# @RELATION DEPENDS_ON -> [docker/frontend.entrypoint.sh]
# @RELATION DEPENDS_ON -> [frontend/package.json]
# @PRE Docker builder context — корень проекта. frontend/ и docker/ доступны.
# @POST nginx:alpine образ со статикой SvelteKit, entrypoint для выбора HTTP/SSL конфига.
# #endregion docker.frontend.Dockerfile
FROM node:20-alpine AS build
WORKDIR /app/frontend
@@ -10,7 +21,23 @@ RUN npm run build
FROM nginx:1.27-alpine
COPY docker/nginx.conf /etc/nginx/conf.d/default.conf
# ca-certificates и openssl для установки корп. сертификатов и проверки SSL ключей
RUN apk add --no-cache ca-certificates openssl
# HTTP-only конфиг (по умолчанию)
COPY docker/nginx.conf /docker/nginx.conf
# SSL конфиг (активируется entrypoint при наличии сертификатов)
COPY docker/nginx.ssl.conf /docker/nginx.ssl.conf
# Entrypoint — установка CA-сертификатов, выбор HTTP/SSL конфига
COPY docker/frontend.entrypoint.sh /docker/entrypoint.sh
RUN chmod +x /docker/entrypoint.sh
# Статика из сборки
COPY --from=build /app/frontend/build /usr/share/nginx/html
EXPOSE 80
EXPOSE 80 443
ENTRYPOINT ["/docker/entrypoint.sh"]
CMD ["nginx", "-g", "daemon off;"]

View File

@@ -0,0 +1,94 @@
#!/bin/sh
set -euo pipefail
# #region docker.frontend.entrypoint [C:3] [TYPE Module] [SEMANTICS docker,nginx,entrypoint,certs,ssl]
# @BRIEF Entrypoint для nginx-контейнера — установка корп. CA-сертификатов в Alpine,
# авто-выбор HTTP или SSL конфига в зависимости от наличия server.crt + server.key.
# @PRE /opt/certs смонтирован (может быть пуст). Системные пакеты: ca-certificates, openssl.
# @POST Если найдены server.crt + server.key — активирован SSL конфиг.
# CA-сертификаты из /opt/certs установлены в системное хранилище Alpine.
# @SIDE_EFFECT Модифицирует /usr/local/share/ca-certificates/; перезаписывает nginx конфиг.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker/nginx.conf]
# @RELATION DEPENDS_ON -> [docker/nginx.ssl.conf]
# @INVARIANT Порт 80 всегда активен (HTTP или редирект на HTTPS).
# @INVARIANT Если нет server.crt — используется HTTP-only конфиг.
# #endregion docker.frontend.entrypoint
# ======================================================================
# install_ca_certificates — корпоративные CA в Alpine
# ======================================================================
# #region docker.frontend.entrypoint.install_ca_certificates [C:2] [TYPE Function]
# @BRIEF Устанавливает .crt файлы из /opt/certs в системное хранилище Alpine.
# @PRE /opt/certs может быть пуст или не существовать.
# @POST Сертификаты скопированы в /usr/local/share/ca-certificates/ и добавлены в trust.
install_ca_certificates() {
local certs_dir="${CERTS_PATH:-/opt/certs}"
if [ ! -d "$certs_dir" ]; then
echo "[entrypoint] CERTS_PATH=${certs_dir} не существует — пропускаем CA-сертификаты"
return 0
fi
local count=0
for cert in "$certs_dir"/*.crt "$certs_dir"/*.pem; do
[ -f "$cert" ] || continue
# Не копируем server.crt — он для nginx, не для CA
case "$(basename "$cert")" in
server.crt|server.key) continue ;;
esac
cp -v "$cert" /usr/local/share/ca-certificates/ 2>&1 | sed 's/^/[entrypoint] /'
count=$((count + 1))
done
if [ "$count" -gt 0 ]; then
echo "[entrypoint] Обновляем CA-сертификаты Alpine..."
update-ca-certificates 2>&1 | sed 's/^/[entrypoint] /'
echo "[entrypoint] ✅ Установлено ${count} корпоративных CA-сертификатов"
else
echo "[entrypoint] Нет CA-сертификатов для установки"
fi
}
# #endregion docker.frontend.entrypoint.install_ca_certificates
# ======================================================================
# select_nginx_config — выбор HTTP vs SSL
# ======================================================================
# #region docker.frontend.entrypoint.select_nginx_config [C:2] [TYPE Function]
# @BRIEF Определяет, есть ли сертификаты для SSL, и копирует нужный nginx конфиг.
# @PRE /opt/certs/server.crt и /opt/certs/server.key — опциональные SSL-сертификаты.
# @POST Если есть оба файла — /etc/nginx/ssl/ создан, server.crt/key скопированы,
# используется nginx.ssl.conf. Иначе — nginx.conf (HTTP-only).
select_nginx_config() {
local certs_dir="${CERTS_PATH:-/opt/certs}"
local ssl_cert="${certs_dir}/server.crt"
local ssl_key="${certs_dir}/server.key"
local ssl_dir="/etc/nginx/ssl"
if [ -f "$ssl_cert" ] && [ -f "$ssl_key" ]; then
echo "[entrypoint] Найдены SSL-сертификаты — включаем HTTPS"
mkdir -p "$ssl_dir"
cp "$ssl_cert" "${ssl_dir}/server.crt"
cp "$ssl_key" "${ssl_dir}/server.key"
# Копируем SSL-конфиг nginx
cp /docker/nginx.ssl.conf /etc/nginx/conf.d/default.conf
echo "[entrypoint] ✅ SSL терминация активирована на порту 443"
else
echo "[entrypoint] SSL-сертификаты не найдены — используем HTTP-only режим"
echo "[entrypoint] Чтобы включить HTTPS, положите в ${certs_dir}:"
echo "[entrypoint] server.crt — SSL сертификат"
echo "[entrypoint] server.key — приватный ключ"
cp /docker/nginx.conf /etc/nginx/conf.d/default.conf
fi
}
# #endregion docker.frontend.entrypoint.select_nginx_config
install_ca_certificates
select_nginx_config
echo "[entrypoint] Starting nginx..."
exec "$@"

65
docker/nginx.ssl.conf Normal file
View File

@@ -0,0 +1,65 @@
# #region docker.nginx.ssl.conf [C:3] [TYPE Module] [SEMANTICS nginx,ssl,https,enterprise]
# @BRIEF Nginx config с опциональным SSL — HTTP (порт 80) + HTTPS (порт 443).
# Используется frontend.entrypoint.sh, если найдены server.crt + server.key.
# @PRE SSL сертификаты: /etc/nginx/ssl/server.crt и /etc/nginx/ssl/server.key.
# Внутренний Docker DNS: backend:8000 для API/WS прокси.
# @POST HTTP (80) редиректит на HTTPS (443) если SSL включён.
# HTTPS (443) сервит статику + прокси /api/ и /ws/ на backend.
# @LAYER Infrastructure
# @RELATION DEPENDS_ON -> [docker/frontend.entrypoint.sh]
# @RELATION DEPENDS_ON -> [docker/nginx.conf]
# @INVARIANT Порт 80 всегда открыт (fallback для HTTP-only окружений).
# @INVARIANT /api/ и /ws/ проксируются на http://backend:8000 (без TLS внутри compose-сети).
# @REJECTED Прокси на HTTPS внутри compose-сети отвергнут — избыточно.
# #endregion docker.nginx.ssl.conf
server {
listen 80 default_server;
server_name _;
# Редирект на HTTPS если сертификаты установлены
return 301 https://$host$request_uri;
}
server {
listen 443 ssl default_server;
server_name _;
resolver 127.0.0.11 ipv6=off;
ssl_certificate /etc/nginx/ssl/server.crt;
ssl_certificate_key /etc/nginx/ssl/server.key;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
root /usr/share/nginx/html;
index index.html;
location / {
try_files $uri $uri/ /index.html;
}
location /api/ {
set $backend_api http://backend:8000;
proxy_pass $backend_api;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_read_timeout 300s;
}
location /ws/ {
set $backend_ws http://backend:8000;
proxy_pass $backend_ws;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_read_timeout 300s;
}
}

View File

@@ -25,4 +25,7 @@ export { fetchSchedule, setSchedule, deleteSchedule, enableSchedule, disableSche
// Corrections & bulk replace
export { submitCorrection, submitBulkCorrections, inlineEditCorrection, submitCorrectionToDict, bulkFindReplace, bulkReplacePreview, downloadSkippedCsv } from './translate/corrections.js';
// Target schema validation
export { checkTargetTableSchema } from './translate/target-schema.js';
// #endregion TranslateApi

View File

@@ -0,0 +1,109 @@
// #region TargetSchemaApiTest [C:2] [TYPE Module] [SEMANTICS test, translate, target-schema, api]
// @BRIEF Unit tests for checkTargetTableSchema API client fetch wrapper.
// @LAYER Test
// @RELATION BINDS_TO -> [TranslateTargetSchemaApi]
// @TEST_CONTRACT: checkTargetTableSchema -> returns schema validation result with all fields
// @TEST_EDGE: api_error -> Returns fallback response with error message
// @TEST_EDGE: success_response -> Returns parsed response with all fields
import { describe, it, expect, vi, beforeEach } from 'vitest';
vi.mock('$lib/api.js', () => ({
api: {
postApi: vi.fn(),
},
}));
describe('checkTargetTableSchema', () => {
beforeEach(() => {
vi.clearAllMocks();
});
// #region test_success [TYPE Function]
// @PURPOSE: Happy path — API returns full validation result.
it('returns result with all fields on success', async () => {
const { api } = await import('$lib/api.js');
api.postApi.mockResolvedValue({
table_exists: true,
error: null,
expected_columns: [
{ name: 'id', data_type: 'integer', is_nullable: true },
{ name: 'name', data_type: 'text', is_nullable: true },
],
actual_columns: [
{ name: 'id', data_type: 'integer', is_nullable: true },
{ name: 'name', data_type: 'text', is_nullable: true },
],
missing_columns: [],
extra_columns: [],
all_present: true,
});
const { checkTargetTableSchema } = await import(
'$lib/api/translate/target-schema.js'
);
const result = await checkTargetTableSchema({
environment_id: 'env-1',
target_database_id: 'db-1',
target_table: 'my_table',
});
expect(api.postApi).toHaveBeenCalledWith('/translate/check-target-schema', {
environment_id: 'env-1',
target_database_id: 'db-1',
target_table: 'my_table',
});
expect(result.table_exists).toBe(true);
expect(result.all_present).toBe(true);
expect(result.error).toBeNull();
expect(result.expected_columns).toHaveLength(2);
});
// #endregion test_success
// #region test_api_error [TYPE Function]
// @PURPOSE: API error returns fallback response with error message.
it('returns fallback response with error on API failure', async () => {
const { api } = await import('$lib/api.js');
api.postApi.mockRejectedValue(new Error('Network error'));
const { checkTargetTableSchema } = await import(
'$lib/api/translate/target-schema.js'
);
const result = await checkTargetTableSchema({
environment_id: 'env-1',
target_database_id: 'db-1',
target_table: 'my_table',
});
expect(result.table_exists).toBe(false);
expect(result.all_present).toBe(false);
expect(result.error).toBe('Network error');
expect(result.expected_columns).toEqual([]);
expect(result.missing_columns).toEqual([]);
});
// #endregion test_api_error
// #region test_error_string_fallback [TYPE Function]
// @PURPOSE: String error is captured as message.
it('handles string error value', async () => {
const { api } = await import('$lib/api.js');
api.postApi.mockRejectedValue('Server says no');
const { checkTargetTableSchema } = await import(
'$lib/api/translate/target-schema.js'
);
const result = await checkTargetTableSchema({
environment_id: 'env-1',
target_database_id: 'db-1',
target_table: 'my_table',
});
expect(result.error).toBe('Server says no');
expect(result.table_exists).toBe(false);
});
// #endregion test_error_string_fallback
});
// #endregion TargetSchemaApiTest

View File

@@ -0,0 +1,49 @@
// #region TranslateTargetSchemaApi [C:1] [TYPE Module] [SEMANTICS translate, target-schema, api, validation]
// @BRIEF API client for target table schema validation.
// @RELATION DEPENDS_ON -> [ApiModule]
import { api } from '$lib/api.js';
/**
* Check target table schema — validate expected vs actual columns for translation inserts.
* @param {{
* environment_id: string,
* target_database_id: string,
* target_schema: string,
* target_table: string,
* target_key_cols: string[],
* target_column: string|null,
* translation_column: string|null,
* target_language_column: string|null,
* target_source_column: string|null,
* target_source_language_column: string|null,
* }} payload
* @returns {Promise<{
* table_exists: boolean,
* error: string|null,
* expected_columns: Array<{name: string, data_type: string|null, is_nullable: boolean}>,
* actual_columns: Array<{name: string, data_type: string|null, is_nullable: boolean}>,
* missing_columns: Array<{name: string, data_type: string|null, is_nullable: boolean}>,
* extra_columns: Array<{name: string, data_type: string|null, is_nullable: boolean}>,
* all_present: boolean,
* }>}
*/
export async function checkTargetTableSchema(payload) {
try {
return await api.postApi('/translate/check-target-schema', payload);
} catch (error) {
const message =
(error && typeof error.message === 'string' && error.message) ||
(typeof error === 'string' && error) ||
'Failed to check target table schema';
return {
table_exists: false,
error: message,
expected_columns: [],
actual_columns: [],
missing_columns: [],
extra_columns: [],
all_present: false,
};
}
}
// #endregion TranslateTargetSchemaApi

View File

@@ -0,0 +1,265 @@
<!-- #region TargetSchemaHint [C:3] [TYPE Component] [SEMANTICS translate, schema, validation, hint, button] -->
<!-- @BRIEF Подсказка о колонках целевой таблицы + кнопка "Проверить схему".
Пользователь сам нажимает кнопку — никакого авто-шума.
Показывает diff ожидаемых vs актуальных колонок. -->
<!-- @LAYER UI -->
<!-- @RELATION DEPENDS_ON -> [checkTargetTableSchema] -->
<!-- @RELATION BINDS_TO -> [api] -->
<!-- @UX_STATE Idle -> Кнопка активна. Ждём нажатия. -->
<!-- @UX_STATE Checking -> Спиннер, кнопка disabled. -->
<!-- @UX_STATE Complete -> Показан результат: все ок / отсутствуют колонки / таблица не найдена. -->
<!-- @UX_STATE Error -> Ошибка соединения. Кнопка активна для повтора. -->
<!-- @UX_FEEDBACK Цветовой индикатор: зелёный (все ок), жёлтый (не все колонки), красный (таблица не найдена/ошибка). -->
<!-- @UX_REACTIVITY Props -> $props() для полей конфигурации. -->
<script>
import { onDestroy } from 'svelte';
import { checkTargetTableSchema } from '$lib/api/translate.js';
import { _ } from '$lib/i18n';
let {
environmentId = '',
targetDatabaseId = '',
targetSchema = '',
targetTable = '',
targetKeyCols = [],
targetColumn = null,
translationColumn = null,
targetLanguageColumn = null,
targetSourceColumn = null,
targetSourceLanguageColumn = null,
} = $props();
/** @type {'idle'|'checking'|'complete'|'error'} */
let state = $state('idle');
let result = $state(null);
let abortController = $state(null);
/** CSS-класс body в зависимости от состояния */
let bodyClass = $derived.by(() => {
if (state === 'complete' && result?.all_present) return 'px-3 py-2.5 bg-green-50';
if (state === 'complete' && !result?.all_present && result?.table_exists) return 'px-3 py-2.5 bg-yellow-50';
if (state === 'error' || (state === 'complete' && !result?.table_exists)) return 'px-3 py-2.5 bg-red-50';
return 'px-3 py-2.5';
});
/** Есть ли минимальные данные для проверки? */
let canCheck = $derived(
!!environmentId
&& !!targetDatabaseId
&& !!targetTable
);
/** Текст кнопки */
let buttonLabel = $derived.by(() => {
if (state === 'checking') return _('translate.target_schema.checking') || 'Checking…';
if (state === 'complete') return _('translate.target_schema.check_again') || 'Check Again';
return _('translate.target_schema.check_button') || 'Check Target Schema';
});
/** Основная проверка */
async function handleCheck() {
if (!canCheck || state === 'checking') return;
// Abort предыдущего запроса если был
if (abortController) abortController.abort();
abortController = new AbortController();
const signal = abortController.signal;
state = 'checking';
result = null;
try {
const res = await checkTargetTableSchema({
environment_id: environmentId,
target_database_id: targetDatabaseId,
target_schema: targetSchema || 'public',
target_table: targetTable,
target_key_cols: targetKeyCols || [],
target_column: targetColumn || null,
translation_column: translationColumn || null,
target_language_column: targetLanguageColumn || null,
target_source_column: targetSourceColumn || null,
target_source_language_column: targetSourceLanguageColumn || null,
});
// Если запрос был отменён — не обновляем state
if (signal.aborted) return;
result = res;
state = 'complete';
} catch (err) {
if (signal.aborted) return;
state = 'error';
result = null;
}
}
onDestroy(() => {
if (abortController) abortController.abort();
});
</script>
<div class="target-schema-hint mt-3 rounded-lg border text-sm transition-colors"
class:border-green-300={state === 'complete' && result?.all_present}
class:border-yellow-300={state === 'complete' && !result?.all_present && result?.table_exists}
class:border-red-300={state === 'error' || (state === 'complete' && !result?.table_exists)}
class:border-gray-200={state === 'idle' || state === 'checking'}
role="region"
aria-label={_('translate.target_schema.region_label') || 'Target Schema Check'}
>
<!-- Header: label + кнопка -->
<div class="flex items-center justify-between px-3 py-2 border-b border-gray-100"
class:bg-green-50={state === 'complete' && result?.all_present}
class:bg-yellow-50={state === 'complete' && !result?.all_present && result?.table_exists}
class:bg-red-50={state === 'error' || (state === 'complete' && !result?.table_exists)}
class:bg-gray-50={state === 'idle' || state === 'checking'}
>
<div class="flex items-center gap-2">
<!-- Иконка состояния -->
{#if state === 'checking'}
<svg class="w-4 h-4 text-gray-400 animate-spin" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"/>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"/>
</svg>
{:else if state === 'complete' && result?.all_present}
<svg class="w-4 h-4 text-green-500" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M5 13l4 4L19 7"/>
</svg>
{:else if state === 'complete' && !result?.table_exists}
<svg class="w-4 h-4 text-red-500" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<circle cx="12" cy="12" r="10"/><line x1="15" y1="9" x2="9" y2="15"/><line x1="9" y1="9" x2="15" y2="15"/>
</svg>
{:else if state === 'complete' && !result?.all_present}
<svg class="w-4 h-4 text-yellow-500" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"/>
</svg>
{:else}
<svg class="w-4 h-4 text-gray-400" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M9 12l2 2 4-4m5.618-4.016A11.955 11.955 0 0112 2.944a11.955 11.955 0 01-8.618 3.04A12.02 12.02 0 003 9c0 5.591 3.824 10.29 9 11.622 5.176-1.332 9-6.03 9-11.622 0-1.042-.133-2.052-.382-3.016z"/>
</svg>
{/if}
<span class="font-medium text-gray-700 text-xs uppercase tracking-wide">
{_('translate.target_schema.title') || 'Target Schema'}
</span>
</div>
<!-- Кнопка -->
<button
onclick={handleCheck}
disabled={!canCheck || state === 'checking'}
class="text-xs font-medium px-3 py-1.5 rounded-md border transition-colors focus-visible:ring-2 focus-visible:ring-primary-ring focus-visible:outline-none"
class:bg-white={state !== 'checking'}
class:text-gray-700={state !== 'checking'}
class:border-gray-300={state !== 'checking'}
class:hover:bg-gray-50={state !== 'checking'}
class:bg-gray-100={state === 'checking' || !canCheck}
class:text-gray-400={state === 'checking' || !canCheck}
class:border-gray-200={state === 'checking' || !canCheck}
class:cursor-not-allowed={!canCheck || state === 'checking'}
aria-busy={state === 'checking'}
>
{buttonLabel}
</button>
</div>
<!-- Body: результат проверки -->
<div class={bodyClass}>
{#if state === 'idle'}
<p class="text-gray-500">
{#if !targetTable}
{_('translate.target_schema.enter_table') || 'Enter a target table name and click "Check Target Schema".'}
{:else if !targetDatabaseId}
{_('translate.target_schema.select_database') || 'Select a target database and click "Check Target Schema".'}
{:else}
{_('translate.target_schema.press_button') || 'Click "Check Target Schema" to verify the target table columns.'}
{/if}
</p>
{:else if state === 'checking'}
<p class="text-gray-500">
{_('translate.target_schema.checking') || 'Checking target table schema…'}
</p>
{:else if state === 'complete' && result}
{#if !result.table_exists}
<div class="flex items-center gap-2 text-red-700">
<svg class="w-4 h-4 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<circle cx="12" cy="12" r="10"/><line x1="15" y1="9" x2="9" y2="15"/><line x1="9" y1="9" x2="15" y2="15"/>
</svg>
<p class="font-medium">{_('translate.target_schema.table_not_found') || 'Target table not found in the database.'}</p>
</div>
{:else if result.all_present}
<div class="flex items-center gap-2 text-green-700">
<svg class="w-4 h-4 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M5 13l4 4L19 7"/>
</svg>
<p class="font-medium">{_('translate.target_schema.all_present') || 'Target table has all required columns.'}</p>
</div>
{:else}
<div class="flex items-center gap-2 text-yellow-700 mb-2">
<svg class="w-4 h-4 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z"/>
</svg>
<p class="font-medium">{_('translate.target_schema.missing_columns_title') || 'Target table is missing some required columns.'}</p>
</div>
{/if}
<!-- Список колонок -->
{#if result.expected_columns?.length > 0}
<div class="mt-1.5">
<p class="text-xs font-medium text-gray-500 mb-1.5">
{_('translate.target_schema.expected_columns') || 'Required columns for translation inserts:'}
</p>
<div class="flex flex-wrap gap-1.5">
{#each result.expected_columns as col}
{@const isMissing = result.missing_columns?.some(m => m.name === col.name)}
<span
class="inline-flex items-center gap-1 px-2 py-0.5 rounded text-xs font-mono border"
class:bg-green-100={!isMissing}
class:text-green-800={!isMissing}
class:border-green-300={!isMissing}
class:bg-red-100={isMissing}
class:text-red-800={isMissing}
class:border-red-300={isMissing}
>
{#if isMissing}
<svg class="w-3 h-3" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/>
</svg>
{:else}
<svg class="w-3 h-3" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<path stroke-linecap="round" stroke-linejoin="round" d="M5 13l4 4L19 7"/>
</svg>
{/if}
{col.name}
</span>
{/each}
</div>
</div>
{/if}
<!-- Сопроводительный текст если есть пропущенные -->
{#if result.table_exists && !result.all_present}
<div class="mt-2 text-xs text-yellow-700">
<p>{_('translate.target_schema.add_columns') || 'Add these missing columns to the target table, or configure custom column names in "Target Column Mapping" above.'}</p>
</div>
{/if}
<!-- Ошибка от Superset -->
{#if result.error}
<p class="mt-1.5 text-xs text-red-600 font-mono">{result.error}</p>
{/if}
{:else if state === 'error'}
<div class="flex items-center gap-2 text-red-700">
<svg class="w-4 h-4 shrink-0" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2">
<circle cx="12" cy="12" r="10"/><line x1="15" y1="9" x2="9" y2="15"/><line x1="9" y1="9" x2="15" y2="15"/>
</svg>
<p class="font-medium">
{_('translate.target_schema.check_error') || 'Could not verify target table schema. Make sure the table exists and is accessible.'}
</p>
</div>
{/if}
</div>
</div>
<!-- #endregion TargetSchemaHint -->

View File

@@ -0,0 +1,158 @@
// #region TargetSchemaHintTest [C:2] [TYPE Module] [SEMANTICS test, translate, target-schema, hint, component]
// @BRIEF Contract-focused unit tests for TargetSchemaHint.svelte component.
// @LAYER Test
// @RELATION BINDS_TO -> [TargetSchemaHint:Component]
// @TEST_CONTRACT: TargetSchemaHint -> idle, checking, complete, error UX states
// @TEST_EDGE: idle_state -> Renders button with "Check Target Schema" text
// @TEST_EDGE: button_disabled_no_table -> Button disabled when targetTable is empty
// @TEST_EDGE: button_disabled_no_database -> Button disabled when targetDatabaseId is empty
// @TEST_EDGE: checking_state -> Checking state with spinner and disabled button
// @TEST_EDGE: complete_all_present -> All columns present shows green border
// @TEST_EDGE: complete_missing_columns -> Missing columns shows yellow border
// @TEST_EDGE: error_state -> Error state shows red border
//
// NOTE: This test uses fs-based source inspection rather than @testing-library/svelte
// because the component uses Tailwind opacity modifiers (e.g., bg-green-50/50) in
// class: directives, which Svelte 5 compiler cannot parse in vitest.
// All existing component tests in this project follow the same fs pattern.
import { describe, it, expect } from 'vitest';
import fs from 'node:fs';
import path from 'node:path';
const COMPONENT_PATH = path.resolve(
process.cwd(),
'src/lib/components/translate/TargetSchemaHint.svelte'
);
describe('TargetSchemaHint Component', () => {
// #region test_component_exists [TYPE Function]
// @PURPOSE: Verify component file exists.
it('component file exists', () => {
const exists = fs.existsSync(COMPONENT_PATH);
expect(exists).toBe(true);
});
// #endregion test_component_exists
// #region test_ux_state_contracts [TYPE Function]
// @PURPOSE: Component contains required UX state tags.
it('contains required UX state tags and contract annotations', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain('@UX_STATE Idle');
expect(source).toContain('@UX_STATE Checking');
expect(source).toContain('@UX_STATE Complete');
expect(source).toContain('@UX_STATE Error');
expect(source).toContain('@UX_FEEDBACK');
expect(source).toContain('@UX_REACTIVITY');
expect(source).toContain('checkTargetTableSchema');
});
// #endregion test_ux_state_contracts
// #region test_props_definition [TYPE Function]
// @PURPOSE: Component accepts required props.
it('accepts environmentId, targetDatabaseId, targetSchema, targetTable props', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain('environmentId');
expect(source).toContain('targetDatabaseId');
expect(source).toContain('targetSchema');
expect(source).toContain('targetTable');
expect(source).toContain('targetKeyCols');
});
// #endregion test_props_definition
// #region test_button_check_disabled_logic [TYPE Function]
// @PURPOSE: Button disabled logic: canCheck derived checks all required props.
it('has button disabled when canCheck is false', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain('disabled={!canCheck || state === \'checking\'}');
});
// #endregion test_button_check_disabled_logic
// #region test_idle_state [TYPE Function]
// @PURPOSE: Idle state shows hint text and check button.
it('implements idle UX state with hint text', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain("state === 'idle'");
expect(source).toContain('translate.target_schema.press_button');
});
// #endregion test_idle_state
// #region test_checking_state [TYPE Function]
// @PURPOSE: Checking state shows spinner and disables button.
it('implements checking UX state with spinner', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain("state === 'checking'");
expect(source).toContain('animate-spin');
expect(source).toContain('aria-busy={state === \'checking\'');
});
// #endregion test_checking_state
// #region test_complete_state [TYPE Function]
// @PURPOSE: Complete state shows result with green/yellow/red indicators.
it('implements complete UX state with color indicators', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain("state === 'complete'");
expect(source).toContain('result?.all_present');
expect(source).toContain('result?.table_exists');
expect(source).toContain('result.missing_columns');
});
// #endregion test_complete_state
// #region test_error_state [TYPE Function]
// @PURPOSE: Error state shows connection error message.
it('implements error UX state with retry prompt', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain("state === 'error'");
expect(source).toContain('translate.target_schema.check_error');
});
// #endregion test_error_state
// #region test_i18n_usage [TYPE Function]
// @PURPOSE: Component uses i18n _() for all user-facing strings with fallbacks.
it('uses i18n for all user-facing strings', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain("_('translate.target_schema.title')");
expect(source).toContain("_('translate.target_schema.check_button')");
expect(source).toContain("_('translate.target_schema.checking')");
expect(source).toContain("_('translate.target_schema.check_again')");
expect(source).toContain("_('translate.target_schema.all_present')");
expect(source).toContain("_('translate.target_schema.table_not_found')");
expect(source).toContain("_('translate.target_schema.check_error')");
expect(source).toContain("_('translate.target_schema.enter_table')");
expect(source).toContain("_('translate.target_schema.select_database')");
expect(source).toContain("_('translate.target_schema.missing_columns_title')");
expect(source).toContain("_('translate.target_schema.expected_columns')");
expect(source).toContain("_('translate.target_schema.add_columns')");
});
// #endregion test_i18n_usage
// #region test_abort_controller [TYPE Function]
// @PURPOSE: Component uses AbortController for request cancellation.
it('implements AbortController for request cancellation', () => {
const source = fs.readFileSync(COMPONENT_PATH, 'utf-8');
expect(source).toContain('AbortController');
expect(source).toContain('abortController.abort()');
expect(source).toContain('signal.aborted');
});
// #endregion test_abort_controller
});
// #endregion TargetSchemaHintTest

View File

@@ -609,5 +609,21 @@
"tokens": "Tokens",
"cost": "Cost",
"avg_latency": "Avg Latency"
},
"target_schema": {
"title": "Target Schema",
"region_label": "Target Table Schema Check",
"check_button": "Check Schema",
"check_again": "Check Again",
"checking": "Checking…",
"enter_table": "Enter a target table name and click \"Check Schema\".",
"select_database": "Select a target database and click \"Check Schema\".",
"press_button": "Click \"Check Schema\" to verify the target table columns.",
"check_error": "Could not verify target table schema. Make sure the table exists and is accessible.",
"table_not_found": "Target table not found in the database.",
"all_present": "Target table has all required columns.",
"missing_columns_title": "Target table is missing some required columns.",
"expected_columns": "Required columns for translation inserts:",
"add_columns": "Add these missing columns to the target table, or configure custom column names in \"Target Column Mapping\" above."
}
}

View File

@@ -610,5 +610,21 @@
"tokens": "Токены",
"cost": "Стоимость",
"avg_latency": "Средняя задержка"
},
"target_schema": {
"title": "Схема целевой таблицы",
"region_label": "Проверка схемы целевой таблицы",
"check_button": "Проверить схему",
"check_again": "Проверить ещё раз",
"checking": "Проверка…",
"enter_table": "Укажите имя целевой таблицы и нажмите «Проверить схему».",
"select_database": "Выберите целевую базу данных и нажмите «Проверить схему».",
"press_button": "Нажмите «Проверить схему» для проверки колонок целевой таблицы.",
"check_error": "Не удалось проверить схему. Убедитесь, что таблица существует и доступна.",
"table_not_found": "Целевая таблица не найдена в базе данных.",
"all_present": "Целевая таблица содержит все необходимые колонки.",
"missing_columns_title": "В целевой таблице не хватает некоторых обязательных колонок.",
"expected_columns": "Необходимые колонки для вставки переводов:",
"add_columns": "Добавьте недостающие колонки в целевую таблицу или настройте кастомные имена колонок в «Маппинг целевых колонок» выше."
}
}

View File

@@ -24,7 +24,8 @@
import TranslationRunResult from '$lib/components/translate/TranslationRunResult.svelte';
import ScheduleConfig from '$lib/components/translate/ScheduleConfig.svelte';
import { triggerRun, fetchRunHistory, cancelRun } from '$lib/api/translate.js';
import BulkReplaceModal from '$lib/components/translate/BulkReplaceModal.svelte';
import BulkReplaceModal from '$lib/components/translate/BulkReplaceModal.svelte';
import TargetSchemaHint from '$lib/components/translate/TargetSchemaHint.svelte';
import { fromStore } from 'svelte/store';
import {
translationRunStore,
@@ -1126,6 +1127,20 @@
</div>
</div>
</div>
<!-- Schema hint: проверка целевой таблицы -->
<TargetSchemaHint
environmentId={environmentId}
targetDatabaseId={targetDatabaseId}
targetSchema={targetSchema}
targetTable={targetTable}
targetKeyCols={targetKeyCols}
targetColumn={targetColumn}
translationColumn={translationColumn}
targetLanguageColumn={targetLanguageColumn}
targetSourceColumn={targetSourceColumn}
targetSourceLanguageColumn={targetSourceLanguageColumn}
/>
</section>
<!-- LLM Settings -->