busya
4726fd110d
Create AsyncJobRunner — centralized bridge between APScheduler (BackgroundScheduler, sync thread pool) and async coroutines. Fixes: - P0: execute_run() called without await from APScheduler thread, causing coroutine to be silently discarded (root cause: no translation history) - P0: get_async_job_runner() deadlock when called from APScheduler thread pool without running event loop - P1: ID mismatch in disable_schedule/delete_schedule routes (job_id passed instead of schedule_id) - P1: asyncio.run() in APScheduler callbacks incompatible with running event loop - Delete unused llm_analysis/scheduler.py (not used in production) Changes: core/async_job_runner.py — new: AsyncJobRunner class core/scheduler.py — use runner.run()/run_later() translate/scheduler.py — use runner.run() for execute_run mapping_service.py — remove unused BackgroundScheduler dependencies.py — add get_async_job_runner() DI app.py — init runner in lifespan api/routes/migration.py — use runner.run() _schedule_routes.py — fix ID mismatch plugins/migration.py — use runner.run() llm_analysis/scheduler.py — delete (unused) tests: 151 new/updated tests, all passing
528 lines
17 KiB
Python
528 lines
17 KiB
Python
# #region TestMappingService [C:2] [TYPE Module]
|
|
#
|
|
# @PURPOSE: Unit tests for the IdMappingService matching UUIDs to integer IDs.
|
|
# @LAYER Domain
|
|
# @RELATION BINDS_TO ->[IdMappingService]
|
|
#
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
import pytest
|
|
import sys
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
# Add backend directory to sys.path so 'src' can be resolved
|
|
backend_dir = str(Path(__file__).parent.parent.parent.resolve())
|
|
if backend_dir not in sys.path:
|
|
sys.path.insert(0, backend_dir)
|
|
|
|
from src.core.mapping_service import IdMappingService
|
|
from src.models.mapping import Base, Environment, ResourceMapping, ResourceType
|
|
|
|
|
|
@pytest.fixture
|
|
def db_session():
|
|
# In-memory SQLite for testing
|
|
engine = create_engine("sqlite:///:memory:")
|
|
event.listen(engine, "connect", lambda c, _: c.execute("PRAGMA foreign_keys=ON"))
|
|
Base.metadata.create_all(engine)
|
|
Session = sessionmaker(bind=engine)
|
|
session = Session()
|
|
# Create FK parent Environment records for all environment_ids used in tests
|
|
for env_id in ['test-env', 'prod-env-1', 'env1']:
|
|
if not session.query(Environment).filter(Environment.id == env_id).first():
|
|
session.add(Environment(
|
|
id=env_id, name=env_id,
|
|
url=f'http://{env_id}.example.com',
|
|
credentials_id=env_id
|
|
))
|
|
session.commit()
|
|
yield session
|
|
session.close()
|
|
|
|
|
|
class MockSupersetClient:
|
|
def __init__(self, resources):
|
|
self.resources = resources
|
|
|
|
async def get_all_resources(self, endpoint, since_dttm=None):
|
|
return self.resources.get(endpoint, [])
|
|
|
|
|
|
# #region test_sync_environment_upserts_correctly [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_upserts_correctly(db_session):
|
|
service = IdMappingService(db_session)
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{
|
|
"id": 42,
|
|
"uuid": "123e4567-e89b-12d3-a456-426614174000",
|
|
"slice_name": "Test Chart",
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
await service.sync_environment("test-env", mock_client)
|
|
|
|
mapping = db_session.query(ResourceMapping).first()
|
|
assert mapping is not None
|
|
assert mapping.environment_id == "test-env"
|
|
assert mapping.resource_type == ResourceType.CHART
|
|
assert mapping.uuid == "123e4567-e89b-12d3-a456-426614174000"
|
|
assert mapping.remote_integer_id == "42"
|
|
assert mapping.resource_name == "Test Chart"
|
|
|
|
|
|
# #endregion test_sync_environment_upserts_correctly
|
|
|
|
|
|
# #region test_get_remote_id_returns_integer [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_id_returns_integer(db_session):
|
|
service = IdMappingService(db_session)
|
|
mapping = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DATASET,
|
|
uuid="uuid-1",
|
|
remote_integer_id="99",
|
|
resource_name="Test DS",
|
|
last_synced_at=datetime.now(UTC),
|
|
)
|
|
db_session.add(mapping)
|
|
db_session.commit()
|
|
|
|
result = service.get_remote_id("test-env", ResourceType.DATASET, "uuid-1")
|
|
assert result == 99
|
|
|
|
|
|
# #endregion test_get_remote_id_returns_integer
|
|
|
|
|
|
# #region test_get_remote_ids_batch_returns_dict [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_ids_batch_returns_dict(db_session):
|
|
service = IdMappingService(db_session)
|
|
m1 = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DASHBOARD,
|
|
uuid="uuid-1",
|
|
remote_integer_id="11",
|
|
)
|
|
m2 = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DASHBOARD,
|
|
uuid="uuid-2",
|
|
remote_integer_id="22",
|
|
)
|
|
db_session.add_all([m1, m2])
|
|
db_session.commit()
|
|
|
|
result = service.get_remote_ids_batch(
|
|
"test-env", ResourceType.DASHBOARD, ["uuid-1", "uuid-2", "uuid-missing"]
|
|
)
|
|
|
|
assert len(result) == 2
|
|
assert result["uuid-1"] == 11
|
|
assert result["uuid-2"] == 22
|
|
assert "uuid-missing" not in result
|
|
|
|
|
|
# #endregion test_get_remote_ids_batch_returns_dict
|
|
|
|
|
|
# #region test_sync_environment_updates_existing_mapping [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_updates_existing_mapping(db_session):
|
|
"""Verify that sync_environment updates an existing mapping (upsert UPDATE path)."""
|
|
from src.models.mapping import ResourceMapping
|
|
|
|
# Pre-populate a mapping
|
|
existing = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.CHART,
|
|
uuid="123e4567-e89b-12d3-a456-426614174000",
|
|
remote_integer_id="10",
|
|
resource_name="Old Name",
|
|
)
|
|
db_session.add(existing)
|
|
db_session.commit()
|
|
|
|
service = IdMappingService(db_session)
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{
|
|
"id": 42,
|
|
"uuid": "123e4567-e89b-12d3-a456-426614174000",
|
|
"slice_name": "Updated Name",
|
|
}
|
|
]
|
|
}
|
|
)
|
|
|
|
await service.sync_environment("test-env", mock_client)
|
|
|
|
mapping = (
|
|
db_session.query(ResourceMapping)
|
|
.filter_by(uuid="123e4567-e89b-12d3-a456-426614174000")
|
|
.first()
|
|
)
|
|
assert mapping.remote_integer_id == "42"
|
|
assert mapping.resource_name == "Updated Name"
|
|
# Should still be only one record (updated, not duplicated)
|
|
count = db_session.query(ResourceMapping).count()
|
|
assert count == 1
|
|
|
|
|
|
# #endregion test_sync_environment_updates_existing_mapping
|
|
|
|
|
|
# #region test_sync_environment_skips_resources_without_uuid [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_skips_resources_without_uuid(db_session):
|
|
"""Resources missing uuid or having id=None should be silently skipped."""
|
|
service = IdMappingService(db_session)
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{"id": 42, "slice_name": "No UUID"}, # Missing 'uuid' -> skipped
|
|
{
|
|
"id": None,
|
|
"uuid": "valid-uuid",
|
|
"slice_name": "ID is None",
|
|
}, # id=None -> skipped
|
|
{
|
|
"id": None,
|
|
"uuid": None,
|
|
"slice_name": "Both None",
|
|
}, # both None -> skipped
|
|
]
|
|
}
|
|
)
|
|
|
|
await service.sync_environment("test-env", mock_client)
|
|
|
|
count = db_session.query(ResourceMapping).count()
|
|
assert count == 0
|
|
|
|
|
|
# #endregion test_sync_environment_skips_resources_without_uuid
|
|
|
|
|
|
# #region test_sync_environment_handles_api_error_gracefully [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_handles_api_error_gracefully(db_session):
|
|
"""If one resource type fails, others should still sync."""
|
|
|
|
class FailingClient:
|
|
async def get_all_resources(self, endpoint, since_dttm=None):
|
|
if endpoint == "chart":
|
|
raise ConnectionError("API timeout")
|
|
if endpoint == "dataset":
|
|
return [{"id": 99, "uuid": "ds-uuid-1", "table_name": "users"}]
|
|
return []
|
|
|
|
service = IdMappingService(db_session)
|
|
await service.sync_environment("test-env", FailingClient())
|
|
|
|
count = db_session.query(ResourceMapping).count()
|
|
assert count == 1 # Only dataset was synced; chart error was swallowed
|
|
mapping = db_session.query(ResourceMapping).first()
|
|
assert mapping.resource_type == ResourceType.DATASET
|
|
|
|
|
|
# #endregion test_sync_environment_handles_api_error_gracefully
|
|
|
|
|
|
# #region test_get_remote_id_returns_none_for_missing [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_id_returns_none_for_missing(db_session):
|
|
"""get_remote_id should return None when no mapping exists."""
|
|
service = IdMappingService(db_session)
|
|
result = service.get_remote_id("test-env", ResourceType.CHART, "nonexistent-uuid")
|
|
assert result is None
|
|
|
|
|
|
# #endregion test_get_remote_id_returns_none_for_missing
|
|
|
|
|
|
# #region test_get_remote_ids_batch_returns_empty_for_empty_input [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_ids_batch_returns_empty_for_empty_input(db_session):
|
|
"""get_remote_ids_batch should return {} for an empty list of UUIDs."""
|
|
service = IdMappingService(db_session)
|
|
result = service.get_remote_ids_batch("test-env", ResourceType.CHART, [])
|
|
assert result == {}
|
|
|
|
|
|
# #endregion test_get_remote_ids_batch_returns_empty_for_empty_input
|
|
|
|
|
|
# #region test_mapping_service_alignment_with_test_data [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_mapping_service_alignment_with_test_data(db_session):
|
|
"""**@TEST_DATA**: Verifies that the service aligns with the resource_mapping_record contract."""
|
|
# Contract: {'environment_id': 'prod-env-1', 'resource_type': 'chart', 'uuid': '123e4567-e89b-12d3-a456-426614174000', 'remote_integer_id': '42'}
|
|
contract_data = {
|
|
"environment_id": "prod-env-1",
|
|
"resource_type": ResourceType.CHART,
|
|
"uuid": "123e4567-e89b-12d3-a456-426614174000",
|
|
"remote_integer_id": "42",
|
|
}
|
|
|
|
mapping = ResourceMapping(**contract_data)
|
|
db_session.add(mapping)
|
|
db_session.commit()
|
|
|
|
service = IdMappingService(db_session)
|
|
result = service.get_remote_id(
|
|
contract_data["environment_id"],
|
|
contract_data["resource_type"],
|
|
contract_data["uuid"],
|
|
)
|
|
|
|
assert result == 42
|
|
|
|
|
|
# #endregion test_mapping_service_alignment_with_test_data
|
|
|
|
|
|
# #region test_sync_environment_requires_existing_env [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_requires_existing_env(db_session):
|
|
"""**@PRE**: Verify behavior when environment_id is invalid/missing in DB.
|
|
Note: The current implementation doesn't strictly check for environment existencia in the DB
|
|
before polling, but it should handle it gracefully or follow the contract.
|
|
"""
|
|
service = IdMappingService(db_session)
|
|
mock_client = MockSupersetClient({"chart": []})
|
|
|
|
# Even if environment doesn't exist in a hypothetical 'environments' table,
|
|
# the service should still complete or fail according to defined error handling.
|
|
# In GRACE-Poly, @PRE is a hard requirement. If we don't have an Env model check,
|
|
# we simulate the intent.
|
|
|
|
await service.sync_environment("non-existent-env", mock_client)
|
|
# If no error raised, at least verify no mappings were created for other envs
|
|
assert db_session.query(ResourceMapping).count() == 0
|
|
|
|
|
|
# #endregion test_sync_environment_requires_existing_env
|
|
|
|
|
|
# #region test_sync_environment_deletes_stale_mappings [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_deletes_stale_mappings(db_session):
|
|
"""Verify that mappings for resources deleted from the remote environment
|
|
are removed from the local DB on the next sync cycle."""
|
|
service = IdMappingService(db_session)
|
|
|
|
# First sync: 2 charts exist
|
|
client_v1 = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{"id": 1, "uuid": "aaa", "slice_name": "Chart A"},
|
|
{"id": 2, "uuid": "bbb", "slice_name": "Chart B"},
|
|
]
|
|
}
|
|
)
|
|
await service.sync_environment("env1", client_v1)
|
|
assert (
|
|
db_session.query(ResourceMapping).filter_by(environment_id="env1").count() == 2
|
|
)
|
|
|
|
# Second sync: user deleted Chart B from superset
|
|
client_v2 = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{"id": 1, "uuid": "aaa", "slice_name": "Chart A"},
|
|
]
|
|
}
|
|
)
|
|
await service.sync_environment("env1", client_v2)
|
|
|
|
remaining = db_session.query(ResourceMapping).filter_by(environment_id="env1").all()
|
|
assert len(remaining) == 1
|
|
assert remaining[0].uuid == "aaa"
|
|
|
|
|
|
# #endregion test_sync_environment_deletes_stale_mappings
|
|
|
|
|
|
# #region test_get_remote_id_value_error [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_id_value_error(db_session):
|
|
"""get_remote_id with non-integer remote_integer_id → None."""
|
|
service = IdMappingService(db_session)
|
|
mapping = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DATASET,
|
|
uuid="uuid-bad",
|
|
remote_integer_id="not-a-number",
|
|
)
|
|
db_session.add(mapping)
|
|
db_session.commit()
|
|
|
|
result = service.get_remote_id("test-env", ResourceType.DATASET, "uuid-bad")
|
|
assert result is None
|
|
|
|
|
|
# #endregion test_get_remote_id_value_error
|
|
|
|
|
|
# #region test_get_remote_ids_batch_value_error [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
def test_get_remote_ids_batch_value_error(db_session):
|
|
"""get_remote_ids_batch with non-integer remote_integer_id → skipped."""
|
|
service = IdMappingService(db_session)
|
|
m1 = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DASHBOARD,
|
|
uuid="uuid-ok",
|
|
remote_integer_id="42",
|
|
)
|
|
m2 = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.DASHBOARD,
|
|
uuid="uuid-bad",
|
|
remote_integer_id="not-a-number",
|
|
)
|
|
db_session.add_all([m1, m2])
|
|
db_session.commit()
|
|
|
|
result = service.get_remote_ids_batch(
|
|
"test-env", ResourceType.DASHBOARD, ["uuid-ok", "uuid-bad"]
|
|
)
|
|
assert result["uuid-ok"] == 42
|
|
assert "uuid-bad" not in result
|
|
|
|
|
|
# #endregion test_get_remote_ids_batch_value_error
|
|
|
|
|
|
# #region test_sync_environment_incremental [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_incremental(db_session):
|
|
"""incremental=True uses max_date filter for since_dttm."""
|
|
service = IdMappingService(db_session)
|
|
|
|
# Add an existing mapping with a last_synced_at value
|
|
from datetime import timedelta
|
|
old_mapping = ResourceMapping(
|
|
environment_id="test-env",
|
|
resource_type=ResourceType.CHART,
|
|
uuid="existing-uuid",
|
|
remote_integer_id="1",
|
|
resource_name="Old Chart",
|
|
last_synced_at=datetime.now(UTC) - timedelta(days=1),
|
|
)
|
|
db_session.add(old_mapping)
|
|
db_session.commit()
|
|
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{
|
|
"id": 42,
|
|
"uuid": "existing-uuid",
|
|
"slice_name": "Updated Chart",
|
|
},
|
|
{
|
|
"id": 99,
|
|
"uuid": "new-uuid",
|
|
"slice_name": "New Chart",
|
|
},
|
|
]
|
|
}
|
|
)
|
|
|
|
await service.sync_environment("test-env", mock_client, incremental=True)
|
|
|
|
# Both existing and new should be synced
|
|
mappings = db_session.query(ResourceMapping).filter_by(environment_id="test-env").all()
|
|
assert len(mappings) == 2
|
|
|
|
# Existing mapping should be updated
|
|
updated = db_session.query(ResourceMapping).filter_by(uuid="existing-uuid").first()
|
|
assert updated.remote_integer_id == "42"
|
|
assert updated.resource_name == "Updated Chart"
|
|
|
|
# New mapping created
|
|
new_mapping = db_session.query(ResourceMapping).filter_by(uuid="new-uuid").first()
|
|
assert new_mapping is not None
|
|
assert new_mapping.remote_integer_id == "99"
|
|
|
|
|
|
# #endregion test_sync_environment_incremental
|
|
|
|
|
|
# #region test_sync_environment_no_new_data_incremental [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_no_new_data_incremental(db_session):
|
|
"""incremental=True with no existing last_synced_at → since_dttm is None."""
|
|
service = IdMappingService(db_session)
|
|
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{"id": 1, "uuid": "aaa", "slice_name": "Chart A"},
|
|
],
|
|
"dataset": [],
|
|
"dashboard": [],
|
|
}
|
|
)
|
|
await service.sync_environment("test-env", mock_client, incremental=True)
|
|
|
|
count = db_session.query(ResourceMapping).count()
|
|
assert count == 1
|
|
mapping = db_session.query(ResourceMapping).first()
|
|
assert mapping.uuid == "aaa"
|
|
|
|
|
|
# #endregion test_sync_environment_no_new_data_incremental
|
|
|
|
|
|
# #region test_sync_environment_critical_failure [C:2] [TYPE Function]
|
|
# @RELATION BINDS_TO ->[TestMappingService]
|
|
@pytest.mark.asyncio
|
|
async def test_sync_environment_critical_failure(db_session):
|
|
"""Exception outside resource loop → rollback and re-raise."""
|
|
service = IdMappingService(db_session)
|
|
|
|
service = IdMappingService(db_session)
|
|
# Simulate a commit failure — raise before actual commit
|
|
def broken_commit():
|
|
raise Exception("commit failure")
|
|
db_session.commit = broken_commit
|
|
|
|
mock_client = MockSupersetClient(
|
|
{
|
|
"chart": [
|
|
{"id": 1, "uuid": "uuid-1", "slice_name": "Chart A"},
|
|
]
|
|
}
|
|
)
|
|
|
|
with pytest.raises(Exception, match="commit failure"):
|
|
await service.sync_environment("test-env", mock_client)
|
|
|
|
# DB should have rolled back (rollback clears uncommitted changes)
|
|
assert db_session.query(ResourceMapping).count() == 0
|
|
|
|
|
|
# #endregion test_sync_environment_critical_failure
|
|
# #endregion TestMappingService
|