busya
488a8f349b
- ~60 new/extended test files across api, core, plugins, services, schemas: routes, superset clients, task_manager, lineage, git, translate, dashboard-testing, load-testing, migration, llm_analysis, scheduler, ssl - .coveragerc: enable branch coverage; exclude src/__tests__ (test files) and src/scripts (CLI/ops tools) from the denominator - bug fixes found while testing: * settings: PUT /settings/reports registered under duplicated prefix * schemas/lineage: FleetReportDTO missing run_status (route always 500) * dashboard_testing/baseline_inheritance: visual entry read wrong field * superset_client/_databases: logger extra name shadowed LogRecord attr * routes/datasets: _yaml_string_paths recursion without yield from * translate/sql_generator: restore explicit-type timestamp contract * baseline_catalog: remove unreachable dashboard_id fallback - conftest fixes: pytest_plugins to rootdir conftest (pytest 9), test filename collision, TMPDIR-safe integration fixtures
394 lines
18 KiB
Python
394 lines
18 KiB
Python
# #region Test.LoadTesting.Plugin [C:3] [TYPE Module] [SEMANTICS test,load-testing,plugin,lifecycle]
|
|
# @BRIEF Verify LoadTestingPlugin metadata + execute() + run_load_run lifecycle: happy path,
|
|
# lifecycle-only (no env), user cancellation, unexpected failure, circuit-breaker abort,
|
|
# missing run, missing param, hold_seconds ramp delay.
|
|
# @RELATION BINDS_TO -> [Plugin.LoadTesting.LoadTestingPlugin]
|
|
# @TEST_EDGE: missing_load_run_id -> execute raises ValueError
|
|
# @TEST_EDGE: run_not_found -> execute returns failed dict and closes the session
|
|
# @TEST_EDGE: user_stop -> stopped_by_user persisted, CancelledError re-raised
|
|
# @TEST_EDGE: unexpected_error -> failed persisted with stop_reason
|
|
# @TEST_EDGE: breaker_trip -> circuit_breaker_abort persisted with stop_reason
|
|
# @TEST_EDGE: zero_items_lifecycle_only -> run still reaches completed
|
|
# @TEST_INVARIANT: terminal_run_immutable -> VERIFIED_BY: test_execute_breaker_abort_persists_terminal, test_execute_cancel_persists_terminal
|
|
# @TEST_INVARIANT: one_task_per_run -> VERIFIED_BY: test_execute_happy_path_binds_task_id
|
|
import asyncio
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.orm import Session
|
|
|
|
from src.models.load_testing import LoadRun, RunPhase, RunStatus
|
|
from src.models.mapping import Base
|
|
from src.plugins.load_testing import LoadTestingPlugin, run_load_run
|
|
|
|
|
|
# #region Test.LoadTesting.Plugin.Fakes [C:1] [TYPE Class]
|
|
# @BRIEF Hardcoded fakes for external boundaries (RunnerPool stand-in, task manager, task context).
|
|
class _FakeRunnerPool:
|
|
"""RunnerPool stand-in: drains items and invokes on_result per item (no real workers)."""
|
|
|
|
def __init__(self, *, capacity, executor, env_capacity_semaphore, breaker, on_result):
|
|
self.capacity = capacity
|
|
self.executor = executor
|
|
self.env_capacity_semaphore = env_capacity_semaphore
|
|
self.breaker = breaker
|
|
self.on_result = on_result
|
|
self.outcome = "success"
|
|
self.results = []
|
|
|
|
async def run(self, items):
|
|
self.results = []
|
|
for it in items:
|
|
record = {
|
|
"execution_id": it.get("execution_id"),
|
|
"outcome": self.outcome,
|
|
"upstream_latency_ms": 11,
|
|
}
|
|
if self.on_result is not None:
|
|
await self.on_result(record)
|
|
self.results.append(record)
|
|
return self.results
|
|
|
|
|
|
class _FakeTaskManager:
|
|
def __init__(self, task_id="task-1"):
|
|
self.task_id = task_id
|
|
self.calls = []
|
|
|
|
async def create_task(self, plugin_id, params):
|
|
self.calls.append((plugin_id, params))
|
|
return SimpleNamespace(id=self.task_id)
|
|
|
|
|
|
def _make_context(info_side_effect=None):
|
|
logger = SimpleNamespace(
|
|
info=AsyncMock(side_effect=info_side_effect),
|
|
progress=MagicMock(),
|
|
)
|
|
return SimpleNamespace(logger=logger)
|
|
# #endregion Test.LoadTesting.Plugin.Fakes
|
|
|
|
|
|
# #region Test.LoadTesting.Plugin.Db [C:1] [TYPE Function]
|
|
# @BRIEF Build an isolated FK-enforced in-memory SQLite session with all tables.
|
|
@pytest.fixture()
|
|
def db():
|
|
engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
|
|
event.listen(engine, "connect", lambda c, _: c.execute("PRAGMA foreign_keys=ON"))
|
|
Base.metadata.create_all(bind=engine)
|
|
session = Session(bind=engine)
|
|
yield session
|
|
session.close()
|
|
engine.dispose()
|
|
# #endregion Test.LoadTesting.Plugin.Db
|
|
|
|
|
|
def _make_run(db: Session, run_id: str = "run-1", **overrides) -> LoadRun:
|
|
fields = dict(
|
|
id=run_id,
|
|
environment_id="env-dev",
|
|
dashboard_id=400,
|
|
status=RunStatus.QUEUED,
|
|
phase=RunPhase.RAMP,
|
|
)
|
|
fields.update(overrides)
|
|
run = LoadRun(**fields)
|
|
db.add(run)
|
|
db.commit()
|
|
return run
|
|
|
|
|
|
def _lifecycle_patches(outcome="success"):
|
|
"""Boundary mocks so run_load_run can complete through a fake RunnerPool."""
|
|
cm = MagicMock()
|
|
cm.get_environment.return_value = SimpleNamespace(id="env-dev")
|
|
fake_pool = None
|
|
|
|
def _make_pool(**kwargs):
|
|
nonlocal fake_pool
|
|
fake_pool = _FakeRunnerPool(**kwargs)
|
|
fake_pool.outcome = outcome
|
|
return fake_pool
|
|
|
|
patchers = [
|
|
patch("src.dependencies.get_config_manager", return_value=cm),
|
|
patch("src.services.load_testing.runner_pool.RunnerPool", side_effect=_make_pool),
|
|
patch("src.services.load_testing.persistence.write_load_executions"),
|
|
]
|
|
return patchers, lambda: fake_pool
|
|
|
|
|
|
# #region Test.LoadTesting.Plugin.Metadata [C:2] [TYPE Class]
|
|
# @BRIEF Plugin static metadata + param schema contract.
|
|
class TestPluginMetadata:
|
|
"""Verify plugin identity and schema contract."""
|
|
|
|
# #region Test.LoadTesting.Plugin.TestMetadataProperties [C:2] [TYPE Function]
|
|
# @BRIEF id/name/description/version/ui_route return the documented constants.
|
|
def test_metadata_properties(self):
|
|
plugin = LoadTestingPlugin()
|
|
assert plugin.id == "load_testing"
|
|
assert plugin.name == "Dashboard Load Testing"
|
|
assert plugin.description.startswith("Executes one bounded load run")
|
|
assert plugin.version == "1.0.0"
|
|
assert plugin.ui_route == ""
|
|
# #endregion Test.LoadTesting.Plugin.TestMetadataProperties
|
|
|
|
# #region Test.LoadTesting.Plugin.TestSchemaRequiresRunId [C:2] [TYPE Function]
|
|
# @BRIEF get_schema declares load_run_id required and exposes test/seed hooks.
|
|
def test_schema_requires_run_id(self):
|
|
schema = LoadTestingPlugin().get_schema()
|
|
assert schema["type"] == "object"
|
|
assert schema["required"] == ["load_run_id"]
|
|
assert "hold_seconds" in schema["properties"]
|
|
assert schema["properties"]["execution_specs"]["type"] == "array"
|
|
# #endregion Test.LoadTesting.Plugin.TestSchemaRequiresRunId
|
|
# #endregion Test.LoadTesting.Plugin.Metadata
|
|
|
|
|
|
# #region Test.LoadTesting.Plugin.Execute [C:2] [TYPE Class]
|
|
# @BRIEF execute() param validation, run resolution, task_id reconciliation, session close.
|
|
class TestExecute:
|
|
"""Verify the plugin entrypoint contract."""
|
|
|
|
# #region Test.LoadTesting.Plugin.TestExecuteMissingRunId [C:2] [TYPE Function]
|
|
# @BRIEF Missing/empty load_run_id raises ValueError before touching the DB.
|
|
@pytest.mark.asyncio
|
|
async def test_execute_missing_run_id_raises(self):
|
|
plugin = LoadTestingPlugin()
|
|
with pytest.raises(ValueError, match="load_run_id is required"):
|
|
await plugin.execute({}, context=None)
|
|
with pytest.raises(ValueError, match="load_run_id is required"):
|
|
await plugin.execute({"load_run_id": ""}, context=None)
|
|
# #endregion Test.LoadTesting.Plugin.TestExecuteMissingRunId
|
|
|
|
# #region Test.LoadTesting.Plugin.TestExecuteRunNotFound [C:2] [TYPE Function]
|
|
# @BRIEF Unknown run id returns a failed result and still closes the session.
|
|
@pytest.mark.asyncio
|
|
async def test_execute_run_not_found(self):
|
|
plugin = LoadTestingPlugin()
|
|
fake_db = MagicMock()
|
|
fake_db.get.return_value = None
|
|
with patch("src.plugins.load_testing.SessionLocal", return_value=fake_db):
|
|
result = await plugin.execute({"load_run_id": "nope"}, context=None)
|
|
assert result == {"status": "failed", "error": "LoadRun nope not found"}
|
|
fake_db.close.assert_called_once()
|
|
# #endregion Test.LoadTesting.Plugin.TestExecuteRunNotFound
|
|
|
|
# #region Test.LoadTesting.Plugin.TestExecuteHappyPathBindsTaskId [C:2] [TYPE Function]
|
|
# @BRIEF Happy path: run resolves, _task_id is bound when run.task_id is empty, run completes.
|
|
@pytest.mark.asyncio
|
|
async def test_execute_happy_path_binds_task_id(self, db: Session):
|
|
_make_run(db, run_id="run-ok")
|
|
plugin = LoadTestingPlugin()
|
|
patchers, get_pool = _lifecycle_patches()
|
|
with (
|
|
patch("src.plugins.load_testing.SessionLocal", return_value=db),
|
|
patchers[0], patchers[1], patchers[2],
|
|
):
|
|
result = await plugin.execute(
|
|
{
|
|
"load_run_id": "run-ok",
|
|
"_task_id": "task-9",
|
|
"execution_specs": [{"execution_id": "exec-1", "chart_id": 7, "variation_id": "v1", "filters_hash": "h1"}],
|
|
},
|
|
context=None,
|
|
)
|
|
assert result["status"] == "completed"
|
|
assert result["load_run_id"] == "run-ok"
|
|
run = db.get(LoadRun, "run-ok")
|
|
assert run.task_id == "task-9"
|
|
assert run.status == RunStatus.COMPLETED
|
|
assert run.phase == RunPhase.TERMINAL
|
|
assert run.finished_at is not None
|
|
assert get_pool().capacity == 1
|
|
# #endregion Test.LoadTesting.Plugin.TestExecuteHappyPathBindsTaskId
|
|
|
|
# #region Test.LoadTesting.Plugin.TestExecuteKeepsExistingTaskId [C:2] [TYPE Function]
|
|
# @BRIEF When run.task_id is already set, _task_id must not overwrite it.
|
|
@pytest.mark.asyncio
|
|
async def test_execute_keeps_existing_task_id(self, db: Session):
|
|
_make_run(db, run_id="run-bound", task_id="task-old")
|
|
plugin = LoadTestingPlugin()
|
|
patchers, _ = _lifecycle_patches()
|
|
with (
|
|
patch("src.plugins.load_testing.SessionLocal", return_value=db),
|
|
patchers[0], patchers[1], patchers[2],
|
|
):
|
|
await plugin.execute({"load_run_id": "run-bound", "_task_id": "task-new"}, context=None)
|
|
assert db.get(LoadRun, "run-bound").task_id == "task-old"
|
|
# #endregion Test.LoadTesting.Plugin.TestExecuteKeepsExistingTaskId
|
|
# #endregion Test.LoadTesting.Plugin.Execute
|
|
|
|
|
|
# #region Test.LoadTesting.Plugin.RunLifecycle [C:2] [TYPE Class]
|
|
# @BRIEF run_load_run transitions ramp -> steady -> drain -> terminal with cancellation/failure.
|
|
class TestRunLifecycle:
|
|
"""Verify the run lifecycle state machine."""
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleHappyPath [C:2] [TYPE Function]
|
|
# @BRIEF specs path: pool drains, breaker stays closed, run completes with total_executions.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_happy_path(self, db: Session):
|
|
run = _make_run(db, run_id="run-happy")
|
|
ctx = _make_context()
|
|
patchers, get_pool = _lifecycle_patches()
|
|
with patchers[0], patchers[1], patchers[2]:
|
|
result = await run_load_run(
|
|
db,
|
|
run,
|
|
{"load_run_id": "run-happy", "execution_specs": [{"execution_id": "e1"}, {"execution_id": "e2"}]},
|
|
ctx,
|
|
)
|
|
assert result == {"status": "completed", "load_run_id": "run-happy"}
|
|
fresh = db.get(LoadRun, "run-happy")
|
|
assert fresh.status == RunStatus.COMPLETED
|
|
assert fresh.phase == RunPhase.TERMINAL
|
|
assert fresh.total_executions == 2
|
|
assert fresh.started_at is not None
|
|
assert len(get_pool().results) == 2
|
|
ctx.logger.info.assert_awaited_once()
|
|
ctx.logger.progress.assert_called()
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleHappyPath
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleKeepsExistingTotal [C:2] [TYPE Function]
|
|
# @BRIEF A pre-set total_executions is never overwritten by the specs length.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_keeps_existing_total(self, db: Session):
|
|
run = _make_run(db, run_id="run-total", total_executions=7)
|
|
patchers, _ = _lifecycle_patches()
|
|
with patchers[0], patchers[1], patchers[2]:
|
|
await run_load_run(db, run, {"load_run_id": "run-total", "execution_specs": [{"execution_id": "e1"}]}, None)
|
|
assert db.get(LoadRun, "run-total").total_executions == 7
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleKeepsExistingTotal
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleHoldSeconds [C:2] [TYPE Function]
|
|
# @BRIEF hold_seconds > 0 delays the ramp with asyncio.sleep before steady.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_hold_seconds_sleeps(self, db: Session):
|
|
run = _make_run(db, run_id="run-hold")
|
|
patchers, _ = _lifecycle_patches()
|
|
with patchers[0], patchers[1], patchers[2], patch(
|
|
"src.plugins.load_testing.asyncio.sleep", new=AsyncMock()
|
|
) as mock_sleep:
|
|
await run_load_run(db, run, {"load_run_id": "run-hold", "hold_seconds": 3}, None)
|
|
mock_sleep.assert_awaited_once_with(3)
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleHoldSeconds
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleOnlyNoEnv [C:2] [TYPE Function]
|
|
# @BRIEF Missing environment -> resolved scope None -> lifecycle-only run still completes.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_only_no_env(self, db: Session):
|
|
run = _make_run(db, run_id="run-lifeonly")
|
|
cm = MagicMock()
|
|
cm.get_environment.return_value = None
|
|
with patch("src.dependencies.get_config_manager", return_value=cm):
|
|
result = await run_load_run(db, run, {"load_run_id": "run-lifeonly"}, None)
|
|
assert result["status"] == "completed"
|
|
assert db.get(LoadRun, "run-lifeonly").status == RunStatus.COMPLETED
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleOnlyNoEnv
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleBreakerAbort [C:2] [TYPE Function]
|
|
# @BRIEF >min_samples error outcomes trip the breaker -> circuit_breaker_abort persisted,
|
|
# with a 100% progress entry emitted through the task context.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_breaker_abort(self, db: Session):
|
|
run = _make_run(db, run_id="run-abort")
|
|
ctx = _make_context()
|
|
specs = [{"execution_id": f"e{i}", "chart_id": 1, "variation_id": "v", "filters_hash": "h"} for i in range(21)]
|
|
patchers, _ = _lifecycle_patches(outcome="error")
|
|
with patchers[0], patchers[1], patchers[2]:
|
|
result = await run_load_run(db, run, {"load_run_id": "run-abort", "execution_specs": specs}, ctx)
|
|
assert result == {"status": RunStatus.CIRCUIT_BREAKER_ABORT, "load_run_id": "run-abort"}
|
|
fresh = db.get(LoadRun, "run-abort")
|
|
assert fresh.status == RunStatus.CIRCUIT_BREAKER_ABORT
|
|
assert fresh.phase == RunPhase.TERMINAL
|
|
assert fresh.stop_reason == "circuit_breaker_abort:error_rate"
|
|
ctx.logger.progress.assert_any_call("Load run run-abort aborted by breaker", percent=100)
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleBreakerAbort
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleBreakerAbortNoContext [C:2] [TYPE Function]
|
|
# @BRIEF Breaker abort without a task context still persists the terminal state.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_breaker_abort_no_context(self, db: Session):
|
|
run = _make_run(db, run_id="run-abort2")
|
|
specs = [{"execution_id": f"e{i}", "chart_id": 1, "variation_id": "v", "filters_hash": "h"} for i in range(21)]
|
|
patchers, _ = _lifecycle_patches(outcome="error")
|
|
with patchers[0], patchers[1], patchers[2]:
|
|
result = await run_load_run(db, run, {"load_run_id": "run-abort2", "execution_specs": specs}, None)
|
|
assert result["status"] == RunStatus.CIRCUIT_BREAKER_ABORT
|
|
assert db.get(LoadRun, "run-abort2").status == RunStatus.CIRCUIT_BREAKER_ABORT
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleBreakerAbortNoContext
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleBreakerNoneGuard [C:2] [TYPE Function]
|
|
# @BRIEF Defensive guard: a None breaker (factory seam) must not crash the abort check.
|
|
# _build_breaker is patched to return None to exercise the defensive branch that the
|
|
# real factory never produces (run_load_run treats breaker as optional).
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_breaker_none_guard(self, db: Session):
|
|
run = _make_run(db, run_id="run-none")
|
|
cm = MagicMock()
|
|
cm.get_environment.return_value = SimpleNamespace(id="env-dev")
|
|
fake_pool = _FakeRunnerPool(capacity=1, executor=None, env_capacity_semaphore=None, breaker=None, on_result=None)
|
|
with (
|
|
patch("src.dependencies.get_config_manager", return_value=cm),
|
|
patch("src.services.load_testing.runner_pool.RunnerPool", return_value=fake_pool),
|
|
patch("src.plugins.load_testing._build_breaker", return_value=None),
|
|
):
|
|
result = await run_load_run(
|
|
db, run, {"load_run_id": "run-none", "execution_specs": [{"execution_id": "e1"}]}, None
|
|
)
|
|
assert result["status"] == "completed"
|
|
assert db.get(LoadRun, "run-none").status == RunStatus.COMPLETED
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleBreakerNoneGuard
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleCancel [C:2] [TYPE Function]
|
|
# @BRIEF User stop: CancelledError inside the try -> stopped_by_user persisted and re-raised.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_cancel_persists_terminal(self, db: Session):
|
|
_make_run(db, run_id="run-cancel")
|
|
ctx = _make_context()
|
|
with patch(
|
|
"src.plugins.load_testing.asyncio.sleep",
|
|
new=AsyncMock(side_effect=asyncio.CancelledError()),
|
|
):
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await run_load_run(
|
|
db, db.get(LoadRun, "run-cancel"),
|
|
{"load_run_id": "run-cancel", "hold_seconds": 1}, ctx,
|
|
)
|
|
fresh = db.get(LoadRun, "run-cancel")
|
|
assert fresh.status == RunStatus.STOPPED_BY_USER
|
|
assert fresh.phase == RunPhase.TERMINAL
|
|
assert fresh.stop_reason == "user_stop"
|
|
assert fresh.finished_at is not None
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleCancel
|
|
|
|
# #region Test.LoadTesting.Plugin.TestLifecycleFailure [C:2] [TYPE Function]
|
|
# @BRIEF Unexpected error inside the try: rollback + reload -> failed persisted with stop_reason.
|
|
@pytest.mark.asyncio
|
|
async def test_lifecycle_failure_persists_terminal(self, db: Session):
|
|
_make_run(db, run_id="run-fail")
|
|
ctx = _make_context()
|
|
with patch(
|
|
"src.plugins.load_testing.asyncio.sleep",
|
|
new=AsyncMock(side_effect=RuntimeError("boom")),
|
|
):
|
|
result = await run_load_run(
|
|
db, db.get(LoadRun, "run-fail"),
|
|
{"load_run_id": "run-fail", "hold_seconds": 1}, ctx,
|
|
)
|
|
assert result == {"status": "failed", "error": "boom"}
|
|
fresh = db.get(LoadRun, "run-fail")
|
|
assert fresh.status == RunStatus.FAILED
|
|
assert fresh.phase == RunPhase.TERMINAL
|
|
assert fresh.stop_reason == "boom"
|
|
assert fresh.finished_at is not None
|
|
# #endregion Test.LoadTesting.Plugin.TestLifecycleFailure
|
|
# #endregion Test.LoadTesting.Plugin.RunLifecycle
|
|
|
|
|
|
# #endregion Test.LoadTesting.Plugin
|