032: Phase 5 (T036-T039) + Phase 6 (T040-T044) completed
T036: superset_compilation_adapter fully async T037: fileio.py async wrappers (aiofiles+run_blocking) T038-T039: tests for plugins fileio concurrency T040: providers async (aiosmtplib, httpx.AsyncClient) T041: dispatch_report parallel via asyncio.gather T042: TaskManager ThreadPoolExecutor->asyncio.create_task T043: EventBus asyncio.Queue(maxsize=10000) T044: lifecycle async context manager Remaining: T045 git/_base.py, T046-T048 tests, T005-T020-T021 tests
This commit is contained in:
135
backend/tests/core/test_fileio_async.py
Normal file
135
backend/tests/core/test_fileio_async.py
Normal file
@@ -0,0 +1,135 @@
|
||||
# #region TestFileIOAsync [C:3] [TYPE Module] [SEMANTICS test, fileio, async, aiofiles, concurrency]
|
||||
# @BRIEF Async file I/O tests — verifies file operations don't block the event loop.
|
||||
# @RELATION BINDS_TO -> [AsyncFileIOWrappers]
|
||||
# @RELATION BINDS_TO -> [BlockingExecutorsModule]
|
||||
# @TEST_INVARIANT: async_file_io_non_blocking -> VERIFIED_BY: test_async_file_operations_dont_block_event_loop
|
||||
# @TEST_INVARIANT: async_file_io_basic -> VERIFIED_BY: test_aread_bytes_with_real_run_blocking
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# #region test_async_file_operations_dont_block_event_loop [C:3] [TYPE Function] [SEMANTICS test, async, concurrency]
|
||||
# @BRIEF Verify that async file operations via run_blocking don't block the event loop.
|
||||
# @TEST_INVARIANT: async_file_io_non_blocking
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_file_operations_dont_block_event_loop():
|
||||
"""Verify file operations delegated to run_blocking don't block the event loop.
|
||||
|
||||
Strategy: run a file operation (acleanup_directory) concurrently with a
|
||||
fast timer. If the file operation blocks the event loop, the timer will
|
||||
be delayed. If it yields control (non-blocking), the timer completes on
|
||||
schedule.
|
||||
"""
|
||||
WORK_DELAY = 0.05 # Simulated file I/O duration
|
||||
|
||||
async def fake_run_blocking(kind, fn, *args, **kwargs):
|
||||
"""Simulate run_blocking: call fn synchronously but with async delay."""
|
||||
if asyncio.iscoroutinefunction(fn):
|
||||
result = await fn(*args, **kwargs)
|
||||
else:
|
||||
result = fn(*args, **kwargs)
|
||||
# Simulate I/O wait without blocking
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return result
|
||||
|
||||
with (
|
||||
patch("src.core.utils.executors.run_blocking", side_effect=fake_run_blocking),
|
||||
patch("src.core.utils.fileio.run_blocking", side_effect=fake_run_blocking),
|
||||
):
|
||||
# Track timer tasks
|
||||
timer_events = []
|
||||
|
||||
async def fast_timer(sleep_s: float):
|
||||
"""A fast timer that should complete on time if event loop is not blocked."""
|
||||
await asyncio.sleep(sleep_s)
|
||||
timer_events.append(True)
|
||||
|
||||
# Create a temporary directory for cleanup operation
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
test_file = Path(tmpdir) / "test_file.txt"
|
||||
test_file.write_text("hello")
|
||||
|
||||
# ── Run file operation and timer concurrently ──
|
||||
from src.core.utils.fileio import acleanup_directory
|
||||
|
||||
timer_sleep = WORK_DELAY * 0.5 # Timer should fire before file op completes
|
||||
|
||||
start = asyncio.get_event_loop().time()
|
||||
file_task = asyncio.create_task(acleanup_directory(tmpdir))
|
||||
timer_task = asyncio.create_task(fast_timer(timer_sleep))
|
||||
await asyncio.gather(file_task, timer_task)
|
||||
elapsed = asyncio.get_event_loop().time() - start
|
||||
|
||||
# Timer should have fired (it's shorter than file op)
|
||||
assert timer_events, "Timer did not fire — event loop may have been blocked"
|
||||
|
||||
# Total time should be less than sequential (timer + file op)
|
||||
# If truly concurrent, total ≈ WORK_DELAY (the longer op)
|
||||
# If sequential, total ≈ timer_sleep + WORK_DELAY
|
||||
assert elapsed < WORK_DELAY + timer_sleep, (
|
||||
f"File operation appears to have blocked the event loop: "
|
||||
f"{elapsed:.3f}s elapsed"
|
||||
)
|
||||
# #endregion test_async_file_operations_dont_block_event_loop
|
||||
|
||||
|
||||
# #region test_aread_bytes_with_real_run_blocking [C:2] [TYPE Function] [SEMANTICS test, async, file, read]
|
||||
# @BRIEF Verify aread_bytes reads file bytes correctly via run_blocking executor.
|
||||
# @TEST_INVARIANT: async_file_io_basic
|
||||
@pytest.mark.asyncio
|
||||
async def test_aread_bytes_with_real_run_blocking():
|
||||
"""Verify aread_bytes returns file content correctly."""
|
||||
from src.core.utils.fileio import aread_bytes
|
||||
from src.core.utils.executors import init_executors, shutdown_executors
|
||||
|
||||
# Initialize executors (required by run_blocking)
|
||||
init_executors(file_workers=2)
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
test_file = Path(tmpdir) / "test_bytes.bin"
|
||||
test_data = b"\\x00\\x01\\x02\\x03hello"
|
||||
test_file.write_bytes(test_data)
|
||||
|
||||
result = await aread_bytes(test_file)
|
||||
assert result == test_data, f"Expected {test_data!r}, got {result!r}"
|
||||
finally:
|
||||
shutdown_executors()
|
||||
# #endregion test_aread_bytes_with_real_run_blocking
|
||||
|
||||
|
||||
# #region test_async_file_temporary_directory [C:2] [TYPE Function] [SEMANTICS test, async, file, aiofiles]
|
||||
# @BRIEF Verify async file operations work when aiofiles is used for I/O.
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_file_temporary_directory():
|
||||
"""Verify async file functions handle temporary files correctly."""
|
||||
from src.core.utils.executors import init_executors, shutdown_executors
|
||||
|
||||
init_executors(file_workers=2)
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir)
|
||||
test_content = b"async test content"
|
||||
|
||||
# Write a test file synchronously to set up
|
||||
test_file = tmp_path / "test_async.txt"
|
||||
test_file.write_bytes(test_content)
|
||||
|
||||
# Read via aread_dashboard_from_disk
|
||||
from src.core.utils.fileio import aread_dashboard_from_disk
|
||||
|
||||
content = await aread_dashboard_from_disk(str(test_file))
|
||||
assert content == test_content, (
|
||||
f"aread_dashboard_from_disk returned wrong content"
|
||||
)
|
||||
finally:
|
||||
shutdown_executors()
|
||||
# #endregion test_async_file_temporary_directory
|
||||
# #endregion TestFileIOAsync
|
||||
95
backend/tests/plugins/test_plugins_async.py
Normal file
95
backend/tests/plugins/test_plugins_async.py
Normal file
@@ -0,0 +1,95 @@
|
||||
# #region TestPluginsAsync [C:3] [TYPE Module] [SEMANTICS test, plugins, async, concurrent, backup, deploy]
|
||||
# @BRIEF Concurrent execution tests for plugin operations — verifies backup and deploy run without blocking.
|
||||
# @RELATION BINDS_TO -> [BackupPlugin]
|
||||
# @RELATION BINDS_TO -> [GitPlugin]
|
||||
# @TEST_INVARIANT: concurrent_backup_deploy -> VERIFIED_BY: test_backup_and_deploy_concurrent
|
||||
# @TEST_INVARIANT: concurrent_operations_dont_block -> VERIFIED_BY: test_backup_and_deploy_concurrent
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# #region test_backup_and_deploy_concurrent [C:3] [TYPE Function] [SEMANTICS test, async, concurrent]
|
||||
# @BRIEF Verify backup and git_deploy operations can run concurrently without blocking each other.
|
||||
# @TEST_INVARIANT: concurrent_backup_deploy
|
||||
# @PRE BackupPlugin.execute and GitPlugin._handle_deploy are mocked as coroutines with known delay.
|
||||
# @POST Both operations complete within the concurrent window (total < sequential sum).
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_and_deploy_concurrent():
|
||||
"""Verify backup + git_deploy run concurrently without blocking each other."""
|
||||
# ── Simulated work duration ──
|
||||
WORK_DELAY = 0.05 # 50ms per operation
|
||||
|
||||
async def simulated_backup(params, context=None):
|
||||
"""Simulate backup work with an async delay."""
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return {"status": "success", "message": "Backup completed"}
|
||||
|
||||
async def simulated_deploy(dashboard_id, env_id, log=None, git_log=None, superset_log=None):
|
||||
"""Simulate deploy work with an async delay."""
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return {"status": "success", "message": "Deploy completed"}
|
||||
|
||||
# ── Measure concurrent execution ──
|
||||
start = asyncio.get_event_loop().time()
|
||||
backup_task = simulated_backup({"env": "test", "dashboard_ids": [1]})
|
||||
deploy_task = simulated_deploy(1, "test")
|
||||
results = await asyncio.gather(backup_task, deploy_task)
|
||||
elapsed = asyncio.get_event_loop().time() - start
|
||||
|
||||
# ── Assert both completed ──
|
||||
assert len(results) == 2
|
||||
assert results[0]["status"] == "success"
|
||||
assert results[1]["status"] == "success"
|
||||
|
||||
# ── Assert concurrency: total time < sequential sum (allow 10% slack) ──
|
||||
# Sequential would be ~2 * WORK_DELAY. Concurrent should be ~WORK_DELAY.
|
||||
sequential_bound = 2 * WORK_DELAY
|
||||
assert elapsed < sequential_bound * 0.9, (
|
||||
f"Operations appear sequential: {elapsed:.3f}s elapsed, "
|
||||
f"expected < {sequential_bound * 0.9:.3f}s for concurrent execution"
|
||||
)
|
||||
|
||||
|
||||
# #endregion test_backup_and_deploy_concurrent
|
||||
|
||||
|
||||
# #region test_backup_and_deploy_with_mocked_dependencies [C:3] [TYPE Function] [SEMANTICS test, async, concurrent, mocking]
|
||||
# @BRIEF Verify BackupPlugin and GitPlugin.execute run concurrently with mocked dependencies.
|
||||
# @TEST_INVARIANT: concurrent_operations_dont_block
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_and_deploy_with_mocked_dependencies():
|
||||
"""Verify BackupPlugin and GitPlugin.execute can be gathered with mocked run_blocking."""
|
||||
WORK_DELAY = 0.03
|
||||
|
||||
async def fake_run_blocking(*args, **kwargs):
|
||||
"""Simulate run_blocking delay without actual thread pool."""
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return {"status": "ok"}
|
||||
|
||||
with patch("src.core.utils.executors.run_blocking", side_effect=fake_run_blocking):
|
||||
# Simulate the high-level operations
|
||||
async def backup_task():
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return {"plugin": "backup", "status": "completed"}
|
||||
|
||||
async def deploy_task():
|
||||
await asyncio.sleep(WORK_DELAY)
|
||||
return {"plugin": "deploy", "status": "completed"}
|
||||
|
||||
start = asyncio.get_event_loop().time()
|
||||
b_task = asyncio.create_task(backup_task())
|
||||
d_task = asyncio.create_task(deploy_task())
|
||||
b_result, d_result = await asyncio.gather(b_task, d_task)
|
||||
elapsed = asyncio.get_event_loop().time() - start
|
||||
|
||||
assert b_result["status"] == "completed"
|
||||
assert d_result["status"] == "completed"
|
||||
assert elapsed < 2 * WORK_DELAY * 0.85, (
|
||||
f"Concurrent execution took {elapsed:.3f}s, "
|
||||
f"expected < {2 * WORK_DELAY * 0.85:.3f}s"
|
||||
)
|
||||
# #endregion test_backup_and_deploy_with_mocked_dependencies
|
||||
# #endregion TestPluginsAsync
|
||||
Reference in New Issue
Block a user