fix: resolve production log errors — profile, datasets, migration recovery, mapper validation, reencrypt
- AuthRepository: add missing save_user_dashboard_preference (PATCH /api/profile/preferences returned 500) - DatasetItem.schema_name nullable — GET /api/datasets no longer 503s on Superset datasets with null schema - migration: TaskLogger calls used extra= kwarg which raised TypeError and killed the composite-key recovery branch, leaving the target server unrepaired after failed dashboard imports (e.g. PR-0160); switched to metadata= - maintenance settings: lazy-create singleton with IntegrityError race guard instead of permanent 404 - mapper upload-xlsx: structured 400 detail (expected vs actual columns); MapperTool shows inline error panel instead of toast; i18n ru/en - connection_service: eager re-encryption of legacy plaintext passwords in get_connection; returns a copy so live config never holds plaintext - check_target_schema: log exception type/repr/traceback (was empty error) - scheduler/maintenance auto-end: skip empty-tick INFO logs; suppress HTTP framing for GET polling endpoints - reencrypt script: repair broken merge artifact, JSON-serialize payload on save, fix failed-count arithmetic
This commit is contained in:
@@ -45,7 +45,7 @@ class LastTask(BaseModel):
|
||||
class DatasetItem(BaseModel):
|
||||
id: int
|
||||
table_name: str
|
||||
schema_name: str = Field(..., alias="schema")
|
||||
schema_name: str | None = Field(None, alias="schema")
|
||||
database: str
|
||||
mapped_fields: MappedFields | None = None
|
||||
last_task: LastTask | None = None
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import Any, cast
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ....core.logger import belief_scope, logger as app_logger
|
||||
@@ -852,19 +853,35 @@ async def get_maintenance_settings(
|
||||
).first()
|
||||
|
||||
if not settings:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Maintenance settings not configured",
|
||||
# Settings are optional until an administrator customizes them. Create
|
||||
# the singleton lazily so read-only clients receive a usable response.
|
||||
settings = MaintenanceSettings(
|
||||
id="default",
|
||||
target_environment_id="",
|
||||
)
|
||||
db.add(settings)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# Concurrent request created the singleton first — reuse its row.
|
||||
db.rollback()
|
||||
settings = db.query(MaintenanceSettings).filter(
|
||||
MaintenanceSettings.id == "default"
|
||||
).first()
|
||||
db.refresh(settings)
|
||||
|
||||
return MaintenanceSettingsResponse(
|
||||
target_environment_id=settings.target_environment_id,
|
||||
display_timezone=settings.display_timezone,
|
||||
display_timezone=settings.display_timezone or "Europe/Moscow",
|
||||
date_format=settings.date_format or "YYYY.MM.DD hh:mm:ss",
|
||||
banner_template=settings.banner_template,
|
||||
default_message=settings.default_message,
|
||||
banner_template=settings.banner_template or "",
|
||||
default_message=settings.default_message or "",
|
||||
banner_height=settings.banner_height,
|
||||
dashboard_scope=settings.dashboard_scope.value,
|
||||
dashboard_scope=(
|
||||
settings.dashboard_scope.value
|
||||
if settings.dashboard_scope
|
||||
else "published_only"
|
||||
),
|
||||
excluded_dashboard_ids=list(settings.excluded_dashboard_ids or []),
|
||||
forced_dashboard_ids=list(settings.forced_dashboard_ids or []),
|
||||
updated_at=settings.updated_at.isoformat() if settings.updated_at else None,
|
||||
|
||||
@@ -99,7 +99,16 @@ async def upload_xlsx_mapping(
|
||||
os.unlink(temp_path)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"XLSX file must contain columns: 'column_name' and 'verbose_name'. Missing: {', '.join(missing)}",
|
||||
detail={
|
||||
"message": (
|
||||
"Invalid XLSX header. The first row must contain exactly the "
|
||||
"mapping columns 'column_name' and 'verbose_name'. "
|
||||
f"Missing: {', '.join(sorted(missing))}."
|
||||
),
|
||||
"error_code": "xlsx_invalid_header",
|
||||
"expected_columns": ["column_name", "verbose_name"],
|
||||
"actual_columns": sorted(str(column) for column in actual_columns),
|
||||
},
|
||||
)
|
||||
|
||||
if df.empty:
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
# @LAYER API
|
||||
|
||||
|
||||
import traceback
|
||||
|
||||
from fastapi import Depends, HTTPException, Query, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -289,7 +291,12 @@ async def check_target_schema(
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.explore("check_target_schema failed",
|
||||
extra={"src": "translate_routes", "error": str(e)})
|
||||
extra={
|
||||
"src": "translate_routes",
|
||||
"exception_type": type(e).__name__,
|
||||
"exception_repr": repr(e),
|
||||
"traceback": traceback.format_exc(),
|
||||
})
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_502_BAD_GATEWAY,
|
||||
detail=f"Failed to validate target schema: {e}",
|
||||
|
||||
@@ -166,6 +166,20 @@ class AuthRepository:
|
||||
.first()
|
||||
)
|
||||
# #endregion Core.Repository.GetUserDashboardPreference
|
||||
# #region Core.Repository.SaveUserDashboardPreference [TYPE Function]
|
||||
# @ingroup Auth
|
||||
# @BRIEF: Persist a user's dashboard preference row and return the refreshed entity.
|
||||
# @PRE preference is a UserDashboardPreference bound to this repository session.
|
||||
# @POST The preference is committed and has current database state.
|
||||
def save_user_dashboard_preference(
|
||||
self, preference: UserDashboardPreference
|
||||
) -> UserDashboardPreference:
|
||||
with belief_scope("AuthRepository.save_user_dashboard_preference"):
|
||||
self.db.add(preference)
|
||||
self.db.commit()
|
||||
self.db.refresh(preference)
|
||||
return preference
|
||||
# #endregion Core.Repository.SaveUserDashboardPreference
|
||||
# #region Core.Repository.GetRolesByAdGroups [TYPE Function]
|
||||
# @ingroup Auth
|
||||
# @BRIEF: Retrieve roles that match a list of AD group names.
|
||||
|
||||
@@ -149,11 +149,13 @@ class ConnectionService:
|
||||
if password == MASKED_PASSWORD:
|
||||
return password
|
||||
|
||||
# If it doesn't look like Fernet, it's legacy plaintext — warn and return
|
||||
# If it doesn't look like Fernet, it's legacy plaintext — warn and return.
|
||||
# Eager re-encryption happens in get_connection(); this branch remains as a
|
||||
# safety net for direct callers.
|
||||
if not is_fernet_token(password):
|
||||
logger.warning(
|
||||
"Plaintext password detected in DatabaseConnection. "
|
||||
"Will be auto-encrypted on next save.",
|
||||
"It will be encrypted on the next save.",
|
||||
)
|
||||
return password
|
||||
|
||||
@@ -197,9 +199,20 @@ class ConnectionService:
|
||||
_, conn = _lookup_connection(self.config_manager, connection_id)
|
||||
if conn is None:
|
||||
return None
|
||||
decrypted_pwd = self._decrypt_password(conn.password)
|
||||
conn.password = decrypted_pwd
|
||||
return conn
|
||||
stored_password = conn.password
|
||||
if (
|
||||
stored_password
|
||||
and not is_fernet_token(stored_password)
|
||||
and stored_password != MASKED_PASSWORD
|
||||
):
|
||||
conn.password = self._encrypt_password(stored_password)
|
||||
self.config_manager.save()
|
||||
decrypted_pwd = stored_password
|
||||
else:
|
||||
decrypted_pwd = self._decrypt_password(stored_password)
|
||||
result = conn.model_copy(deep=True)
|
||||
result.password = decrypted_pwd
|
||||
return result
|
||||
|
||||
# #endregion Core.ConnectionService.GetConnection
|
||||
|
||||
|
||||
@@ -57,10 +57,11 @@ def execute_scheduled_scenario_cancel_finalizer() -> None:
|
||||
|
||||
terminalized = finalize_expired_cancellations(db)
|
||||
db.commit()
|
||||
logger.reason(
|
||||
"Scenario cancellation drain finalizer completed",
|
||||
payload={"terminalized": len(terminalized)},
|
||||
)
|
||||
if terminalized:
|
||||
logger.reason(
|
||||
"Scenario cancellation drain finalizer completed",
|
||||
payload={"terminalized": len(terminalized)},
|
||||
)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.explore("Scenario cancellation drain finalizer failed", error=str(exc))
|
||||
@@ -82,7 +83,8 @@ def execute_scheduled_queued_scenario_dispatch() -> None:
|
||||
|
||||
outcomes = dispatch_queued_runs(db, worker_id="scenario-queue-scheduler")
|
||||
db.commit()
|
||||
logger.reason("Queued ScenarioRun dispatch completed", payload={"dispatched": len(outcomes)})
|
||||
if outcomes:
|
||||
logger.reason("Queued ScenarioRun dispatch completed", payload={"dispatched": len(outcomes)})
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.explore("Queued ScenarioRun dispatch failed", error=str(exc))
|
||||
|
||||
@@ -252,10 +252,6 @@ async def _attempt_composite_key_fallback(
|
||||
# @BRIEF Implementation of the migration plugin workflow and transformation orchestration.
|
||||
# @PRE SupersetClient authenticated, database session active
|
||||
# @POST Returns MigrationResult with success/failure status and artifact list
|
||||
# @TEST_FIXTURE superset_export_zip -> file:backend/tests/fixtures/migration/dashboard_export.zip
|
||||
# @TEST_FIXTURE db_mapping_payload -> INLINE_JSON: {"db_mappings": {"source_uuid_1": "target_uuid_2"}}
|
||||
# @TEST_FIXTURE password_inject_payload -> INLINE_JSON: {"passwords": {"PostgreSQL": "secret123"}}
|
||||
# @TEST_INVARIANT strict_db_isolation -> VERIFIED_BY: [successful_dashboard_transfer, missing_mapping_resolution]
|
||||
# @SIDE_EFFECT Writes migration artifacts to database, triggers dashboard imports
|
||||
# @DATA_CONTRACT MigrationPlan AST, DryRunResult, RiskAssessment
|
||||
class MigrationPlugin(PluginBase):
|
||||
@@ -392,13 +388,6 @@ class MigrationPlugin(PluginBase):
|
||||
# @PRE Source and target environments must resolve. Matching dashboards must exist.
|
||||
# @POST Dashboard ZIP bundles are transformed and imported. ID mappings are synchronized.
|
||||
# @SIDE_EFFECT Creates temp files, mutates target Superset state, blocks on user input (passwords/mappings).
|
||||
# @TEST_CONTRACT Dict[str, Any] -> Dict[str, Any]
|
||||
# @TEST_SCENARIO successful_dashboard_transfer -> ZIP is downloaded, DB mappings applied via AST, target import succeeds.
|
||||
# @TEST_SCENARIO missing_password_injection -> Target import fails on auth, TaskManager pauses for user input, retries with password successfully.
|
||||
# @TEST_SCENARIO empty_selection -> Returns NO_MATCHES gracefully when regex finds zero dashboards.
|
||||
# @TEST_EDGE missing_env_field -> [ValueError: Could not resolve source or target environment]
|
||||
# @TEST_EDGE invalid_regex_pattern -> [Regex compilation exception is thrown or caught gracefully]
|
||||
# @TEST_EDGE target_api_timeout -> [Dashboard added to failed_dashboards, task concludes with PARTIAL_SUCCESS]
|
||||
# @INVARIANT Per-dashboard export/transform/import failure never aborts the batch; result is PARTIAL_SUCCESS.
|
||||
# @INVARIANT Password keys sent to Superset are normalized to databases/<name>.yaml paths.
|
||||
async def execute(self, params: dict[str, Any], context: TaskContext | None = None):
|
||||
@@ -638,7 +627,7 @@ class MigrationPlugin(PluginBase):
|
||||
if sync_dataset_composite_keys:
|
||||
migration_log.warning(
|
||||
"Initial import failed; starting automatic dataset-key recovery",
|
||||
extra={
|
||||
metadata={
|
||||
"dashboard_id": dash_id,
|
||||
"dashboard_title": title,
|
||||
"attempt": "initial",
|
||||
@@ -648,7 +637,7 @@ class MigrationPlugin(PluginBase):
|
||||
)
|
||||
migration_log.info(
|
||||
"Synchronizing dataset composite keys before retry",
|
||||
extra={
|
||||
metadata={
|
||||
"dashboard_id": dash_id,
|
||||
"dashboard_title": title,
|
||||
"attempt": "recovery",
|
||||
@@ -671,7 +660,7 @@ class MigrationPlugin(PluginBase):
|
||||
)
|
||||
migration_log.info(
|
||||
"Dataset composite-key synchronization finished; attempting dashboard import retry",
|
||||
extra={
|
||||
metadata={
|
||||
"dashboard_id": dash_id,
|
||||
"dashboard_title": title,
|
||||
"attempt": "recovery",
|
||||
@@ -691,7 +680,7 @@ class MigrationPlugin(PluginBase):
|
||||
)
|
||||
migration_log.info(
|
||||
"Recovery retry import succeeded; dashboard migrated",
|
||||
extra={
|
||||
metadata={
|
||||
"dashboard_id": dash_id,
|
||||
"dashboard_title": title,
|
||||
"attempt": "recovery",
|
||||
@@ -708,7 +697,7 @@ class MigrationPlugin(PluginBase):
|
||||
)
|
||||
migration_log.error(
|
||||
"Recovery retry import failed; dashboard could not be migrated",
|
||||
extra={
|
||||
metadata={
|
||||
"dashboard_id": dash_id,
|
||||
"dashboard_title": title,
|
||||
"attempt": "recovery",
|
||||
|
||||
@@ -21,48 +21,24 @@
|
||||
# #endregion Scripts.Reencrypt
|
||||
|
||||
import argparse
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
from cryptography.fernet import Fernet
|
||||
except ImportError:
|
||||
sys.exit("ERROR: cryptography is not installed. Run: pip install cryptography")
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# ── Fernet helpers (standalone — no app dependency) ────────────────────
|
||||
|
||||
|
||||
def _make_fernet(key_b64: str) -> Fernet:
|
||||
try:
|
||||
return Fernet(key_b64.encode())
|
||||
except Exception as e:
|
||||
sys.exit(f"ERROR: Invalid Fernet key: {e}")
|
||||
|
||||
|
||||
def _is_fernet_token(value: str) -> bool:
|
||||
if not value or len(value) < 60:
|
||||
return False
|
||||
try:
|
||||
import base64
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
from cryptography.fernet import Fernet
|
||||
except ImportError:
|
||||
sys.exit("ERROR: cryptography is not installed. Run: pip install cryptography")
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import Column, String, Text, create_engine
|
||||
from sqlalchemy.orm import Session, declarative_base
|
||||
|
||||
from src.core.encryption import is_fernet_token
|
||||
|
||||
# ── Fernet helpers (standalone — no app dependency) ────────────────────
|
||||
Base = declarative_base()
|
||||
|
||||
# ── Fernet helpers ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_fernet(key_b64: str) -> Fernet:
|
||||
@@ -127,7 +103,7 @@ def main() -> None:
|
||||
_r("🔍 DRY RUN — no changes will be made")
|
||||
else:
|
||||
_r("🔐 Re-encrypting all secrets with new ENCRYPTION_KEY...")
|
||||
_r(f" Started at: {datetime.now(timezone.utc).isoformat()}")
|
||||
_r(f" Started at: {datetime.now(UTC).isoformat()}")
|
||||
_r("")
|
||||
|
||||
# ── Load database URL ──────────────────────────────────────────
|
||||
@@ -137,10 +113,6 @@ def main() -> None:
|
||||
|
||||
# ── Step 1: Environment passwords (AppConfigRecord.payload.environments) ──
|
||||
_r("── Environment passwords (ConfigManager) ──")
|
||||
from sqlalchemy import Column, String, Integer, Text
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
class AppConfigRecord(Base):
|
||||
__tablename__ = "app_configurations"
|
||||
@@ -154,8 +126,8 @@ def main() -> None:
|
||||
with Session(engine) as session:
|
||||
record = session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first()
|
||||
if record and record.payload:
|
||||
import json
|
||||
payload = record.payload if isinstance(record.payload, dict) else json.loads(record.payload)
|
||||
payload_is_text = not isinstance(record.payload, dict)
|
||||
payload = json.loads(record.payload) if payload_is_text else dict(record.payload)
|
||||
environments = payload.get("environments", [])
|
||||
for env in environments:
|
||||
pwd = env.get("password", "")
|
||||
@@ -172,23 +144,22 @@ def main() -> None:
|
||||
total_env_passwords += 1
|
||||
|
||||
if not args.dry_run and reencrypted_env > 0:
|
||||
record.payload = payload
|
||||
record.payload = json.dumps(payload) if payload_is_text else payload
|
||||
session.commit()
|
||||
_r(f" ✓ Committed {reencrypted_env} re-encrypted environment passwords")
|
||||
else:
|
||||
_r(" - No AppConfigRecord found, skipping")
|
||||
|
||||
_r(f" Environment passwords: {reencrypted_env} re-encrypted, {total_env_passwords - reencrypted_env - skipped_env} failed, {skipped_env} skipped")
|
||||
_r(f" Environment passwords: {reencrypted_env} re-encrypted, {total_env_passwords - reencrypted_env} failed, {skipped_env} skipped")
|
||||
_r("")
|
||||
|
||||
# ── Step 2: LLM Provider API keys ──────────────────────────────
|
||||
_r("── LLM Provider API keys ──")
|
||||
from sqlalchemy import Column, String as SAString, Boolean, Integer as SAInteger
|
||||
|
||||
class LLMProvider(Base):
|
||||
__tablename__ = "llm_providers"
|
||||
id = Column(SAString, primary_key=True)
|
||||
api_key = Column(SAString)
|
||||
id = Column(String, primary_key=True)
|
||||
api_key = Column(String)
|
||||
|
||||
total_providers = 0
|
||||
reencrypted_keys = 0
|
||||
@@ -214,13 +185,13 @@ def main() -> None:
|
||||
session.commit()
|
||||
_r(f" ✓ Committed {reencrypted_keys} re-encrypted API keys")
|
||||
|
||||
_r(f" Provider API keys: {reencrypted_keys} re-encrypted, {total_providers - reencrypted_keys - skipped_providers} failed, {skipped_providers} skipped")
|
||||
_r(f" Provider API keys: {reencrypted_keys} re-encrypted, {total_providers - reencrypted_keys} failed, {skipped_providers} skipped")
|
||||
_r("")
|
||||
|
||||
# ── Summary ────────────────────────────────────────────────────
|
||||
_r("── Summary ──")
|
||||
_r(f" Total re-encrypted: {reencrypted_env + reencrypted_keys}")
|
||||
_r(f" Total failed: {(total_env_passwords - reencrypted_env - skipped_env) + (total_providers - reencrypted_keys - skipped_providers)}")
|
||||
_r(f" Total failed: {(total_env_passwords - reencrypted_env) + (total_providers - reencrypted_keys)}")
|
||||
_r(f" Total skipped: {skipped_env + skipped_providers}")
|
||||
if args.dry_run:
|
||||
_r(" (dry run — no changes written)")
|
||||
|
||||
@@ -98,10 +98,11 @@ async def _auto_end_expired_events(task_manager, db_session=None) -> dict:
|
||||
extra={"event_id": event.id},
|
||||
error=str(e),
|
||||
)
|
||||
app_logger.reflect(
|
||||
"Maintenance auto-end scan complete",
|
||||
payload={"ended": len(ended), "candidates": len(candidates)},
|
||||
)
|
||||
if ended or candidates:
|
||||
app_logger.reflect(
|
||||
"Maintenance auto-end scan complete",
|
||||
payload={"ended": len(ended), "candidates": len(candidates)},
|
||||
)
|
||||
return {"ended": len(ended), "event_ids": ended}
|
||||
finally:
|
||||
if db_session is None:
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
# @TEST_EDGE: start_maintenance_idempotent -> 200 with already_active
|
||||
# @TEST_EDGE: end_maintenance_not_found -> 404
|
||||
# @TEST_EDGE: end_maintenance_already_completed -> 200 with already_completed
|
||||
# @TEST_EDGE: get_settings_not_configured -> 404
|
||||
# @TEST_EDGE: get_settings_not_configured -> 200 with defaults
|
||||
# @TEST_EDGE: create_settings_on_first_update -> 200 with created settings
|
||||
# @TEST_EDGE: end_all_maintenance_with_body -> environment_id from body
|
||||
|
||||
@@ -444,17 +444,21 @@ class TestMaintenanceSettings:
|
||||
"""get_maintenance_settings and update_maintenance_settings."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settings_not_found_returns_404(self):
|
||||
"""Missing settings row raises 404."""
|
||||
async def test_settings_not_found_returns_defaults(self):
|
||||
"""Missing settings row is created lazily and returns defaults."""
|
||||
from src.api.routes.maintenance._routes import get_maintenance_settings
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = None
|
||||
mock_db.refresh.side_effect = lambda settings: setattr(
|
||||
settings, "target_environment_id", ""
|
||||
)
|
||||
|
||||
with patch("src.api.routes.maintenance._routes.belief_scope"):
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await get_maintenance_settings(db=mock_db, _=None)
|
||||
assert exc.value.status_code == 404
|
||||
result = await get_maintenance_settings(db=mock_db, _=None)
|
||||
assert result.target_environment_id == ""
|
||||
assert result.display_timezone == "Europe/Moscow"
|
||||
mock_db.commit.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_on_first_update(self):
|
||||
|
||||
@@ -121,7 +121,11 @@ class TestUploadXlsxMapping:
|
||||
files={"file": ("test.xlsx", xlsx_data, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "verbose_name" in resp.json()["detail"]
|
||||
detail = resp.json()["detail"]
|
||||
assert detail["error_code"] == "xlsx_invalid_header"
|
||||
assert detail["expected_columns"] == ["column_name", "verbose_name"]
|
||||
assert detail["actual_columns"] == ["column_name", "other_col"]
|
||||
assert "verbose_name" in detail["message"]
|
||||
# #endregion Test.ToolsMapper.TestMissingRequiredColumns
|
||||
|
||||
# #region Test.ToolsMapper.TestEmptyXlsx [C:2] [TYPE Function]
|
||||
|
||||
@@ -551,7 +551,12 @@ interface ValidationRunQueryParams {
|
||||
// must auto-set Content-Type with the boundary). This is the ONLY exception to the
|
||||
// "never native fetch" invariant.
|
||||
// @RELATION CALLED_BY -> [Tools.MapperTool]
|
||||
async function uploadFile<T = unknown>(endpoint: string, file: File, fields: Record<string, string> = {}): Promise<T> {
|
||||
async function uploadFile<T = unknown>(
|
||||
endpoint: string,
|
||||
file: File,
|
||||
fields: Record<string, string> = {},
|
||||
options: { notifyOnError?: boolean } = {},
|
||||
): Promise<T> {
|
||||
const _start = performance.now();
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
@@ -583,7 +588,7 @@ async function uploadFile<T = unknown>(endpoint: string, file: File, fields: Rec
|
||||
} catch (error) {
|
||||
const apiError = error as ApiError;
|
||||
log('ApiClient', 'EXPLORE', 'Upload failed', { endpoint, filename: file.name }, apiError?.message || 'unknown');
|
||||
notifyApiError(apiError);
|
||||
if (options.notifyOnError !== false) notifyApiError(apiError);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -608,7 +613,12 @@ export const api = {
|
||||
deleteApi: deleteApi as <T = unknown>(endpoint: string, options?: FetchOptions) => Promise<T>,
|
||||
requestApi: requestApi as <T = unknown>(endpoint: string, method?: string, body?: unknown, requestOptions?: FetchOptions) => Promise<T>,
|
||||
fetchApiBlob: fetchApiBlob as (endpoint: string, options?: FetchOptions) => Promise<Blob>,
|
||||
uploadFile: uploadFile as <T = unknown>(endpoint: string, file: File) => Promise<T>,
|
||||
uploadFile: uploadFile as <T = unknown>(
|
||||
endpoint: string,
|
||||
file: File,
|
||||
fields?: Record<string, string>,
|
||||
options?: { notifyOnError?: boolean },
|
||||
) => Promise<T>,
|
||||
|
||||
// ═══ Tasks ════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
<!-- @UX_STATE Uploading -> File upload in progress, upload button disabled. -->
|
||||
<!-- @UX_STATE DragOver -> File being dragged over drop zone, visual highlight on border + icon. -->
|
||||
<!-- @UX_STATE Uploaded -> File uploaded, filename shown, remove button visible. -->
|
||||
<!-- @UX_STATE Error -> Validation errors shown inline, toast notifications displayed. -->
|
||||
<!-- @UX_FEEDBACK Toast notifications on upload success/error, task start/error, doc apply success/error. -->
|
||||
<!-- @UX_STATE Error -> Validation errors shown inline; operational failures use toast notifications. -->
|
||||
<!-- @UX_FEEDBACK Toast notifications on upload success, task start/error, doc apply success/error. -->
|
||||
<!-- @UX_RECOVERY Fix validation errors, retry file upload, retry mapper operation. -->
|
||||
<!-- @UX_REACTIVITY Props -> $props(), LocalState -> let vars (Svelte 5 legacy bind: compat). -->
|
||||
<!-- @SEMANTICS: mapper, tool, dataset, sqllab, excel, upload, file
|
||||
@@ -41,6 +41,7 @@
|
||||
|
||||
// File upload state
|
||||
let uploadedFile: { name: string; uploadId: string } | null = null;
|
||||
let uploadError: { message: string; actualColumns?: string[] } | null = null;
|
||||
let isUploading = false;
|
||||
let isDragOver = false;
|
||||
let isRunning = false;
|
||||
@@ -75,9 +76,10 @@
|
||||
// @ingroup Components
|
||||
// @BRIEF: Upload a single XLSX file to backend and update uploadedFile state.
|
||||
// @PRE: file is a non-null File.
|
||||
// @POST: On success, uploadedFile set with name + temp path. On failure, error toast shown.
|
||||
// @POST: On success, uploadedFile set with name + upload ID. Header validation failures are inline.
|
||||
// @RELATION DEPENDS_ON -> [Api.ApiModule.UploadFile]
|
||||
async function uploadSingleFile(file: File) {
|
||||
uploadError = null;
|
||||
// Client-side extension check
|
||||
if (!file.name.toLowerCase().endsWith('.xlsx')) {
|
||||
notifications.error($t.mapper?.errors?.upload_failed || 'Only .xlsx files are accepted');
|
||||
@@ -86,13 +88,32 @@
|
||||
|
||||
isUploading = true;
|
||||
try {
|
||||
const result = await uploadFile<{ upload_id: string }>('/tools/mapper/upload-xlsx', file);
|
||||
const result = await uploadFile<{ upload_id: string }>(
|
||||
'/tools/mapper/upload-xlsx',
|
||||
file,
|
||||
{},
|
||||
{ notifyOnError: false },
|
||||
);
|
||||
uploadedFile = { name: file.name, uploadId: result.upload_id };
|
||||
notifications.success($t.mapper?.success?.file_uploaded || 'Excel file uploaded');
|
||||
if (fileInputEl) fileInputEl.value = '';
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Upload failed';
|
||||
notifications.error($t.mapper?.errors?.upload_failed || message);
|
||||
const apiError = err as { status?: number; detail?: unknown };
|
||||
const detail = apiError.detail as {
|
||||
error_code?: string;
|
||||
actual_columns?: unknown;
|
||||
} | undefined;
|
||||
if (apiError.status === 400 && detail?.error_code === 'xlsx_invalid_header') {
|
||||
uploadError = {
|
||||
message,
|
||||
actualColumns: Array.isArray(detail.actual_columns)
|
||||
? detail.actual_columns.map(String)
|
||||
: undefined,
|
||||
};
|
||||
} else {
|
||||
notifications.error($t.mapper?.errors?.upload_failed || message);
|
||||
}
|
||||
if (fileInputEl) fileInputEl.value = '';
|
||||
} finally {
|
||||
isUploading = false;
|
||||
@@ -144,6 +165,7 @@
|
||||
// @POST: uploadedFile set to null, file input reset.
|
||||
function handleRemoveFile() {
|
||||
uploadedFile = null;
|
||||
uploadError = null;
|
||||
if (fileInputEl) fileInputEl.value = '';
|
||||
}
|
||||
// #endregion Tools.MapperTool.HandleRemoveFileFunction
|
||||
@@ -482,6 +504,22 @@
|
||||
</span>
|
||||
</button>
|
||||
{/if}
|
||||
{#if uploadError}
|
||||
<div class="mt-3 rounded-lg border border-destructive/40 bg-destructive/5 p-3 text-sm text-text" role="alert">
|
||||
<div class="font-semibold text-destructive">{$t.mapper?.errors?.invalid_header_title || 'Invalid Excel file format'}</div>
|
||||
<p class="mt-1">{uploadError.message}</p>
|
||||
<p class="mt-2">
|
||||
<span class="font-medium">{$t.mapper?.errors?.expected_columns || 'Required columns'}:</span>
|
||||
<code>column_name, verbose_name</code>
|
||||
</p>
|
||||
{#if uploadError.actualColumns?.length}
|
||||
<p class="mt-1 break-words">
|
||||
<span class="font-medium">{$t.mapper?.errors?.actual_columns || 'Columns found in your file'}:</span>
|
||||
<code>{uploadError.actualColumns.join(', ')}</code>
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
"excel_required": "Excel path is required for excel source",
|
||||
"env_not_found": "Environment not found",
|
||||
"upload_failed": "Failed to upload file",
|
||||
"invalid_header_title": "Invalid Excel file format",
|
||||
"expected_columns": "Required columns",
|
||||
"actual_columns": "Columns found in your file",
|
||||
"no_active_llm_provider": "No active LLM provider found",
|
||||
"docs_start_failed": "Failed to start documentation generation",
|
||||
"docs_apply_failed": "Failed to apply documentation",
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
"excel_required": "Путь к Excel обязателен для источника Excel",
|
||||
"env_not_found": "Окружение не найдено",
|
||||
"upload_failed": "Ошибка загрузки файла",
|
||||
"invalid_header_title": "Неверный формат Excel-файла",
|
||||
"expected_columns": "Ожидаемые колонки",
|
||||
"actual_columns": "Колонки в загруженном файле",
|
||||
"no_active_llm_provider": "Не найден активный LLM-провайдер",
|
||||
"docs_start_failed": "Не удалось запустить генерацию документации",
|
||||
"docs_apply_failed": "Не удалось применить документацию",
|
||||
|
||||
Reference in New Issue
Block a user