busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: live app log console, cross-filter, JSONL export

Browse Source 
Backend:
- /ws/app-logs — real-time app/cot log stream (raw JSONL)
- /api/logs/recent — REST snapshot of ring buffer
- GET /tasks/{id}/logs/export — streaming JSONL export with CoT parse + redaction
- Thread-safe ring buffer (seq-based polling) replaces unsafe asyncio.Queue
- GIL-friendly multi-row Core insert for log persistence
- task_id ContextVar propagates into CotJsonFormatter for CoT correlation
- Hot-apply logging level on settings update (FR-005)
- Buffer trim under DEBUG floods; drop DEBUG first, preserve ERROR/WARNING
- List projection (include_result=False) keeps reports list slim
- Security event consolidated to single REASON atom

Frontend:
- ReportsLogModel + ReportsLogPanel — full live JSONL console
- Cross-filter pinning: Tasks → Logs tab with task chip badges
- LogEntryRow — CoT-aware rendering (marker icons, expandable payload)
- TaskFilterChip, taskChipMeta — scannable type/id/env chips
- Global drawer push via CSS variable (lg+ padding, not overlay)
- i18n en/ru for all log console strings
- Ctrl+A in log panel selects only log lines (window-level handler)

QA fixes:
- Svelte 5 reactivity: SvelteDate/SvelteSet/SvelteURLSearchParams
- Fix seed_trace_id shadowing (F823) in lifecycle.py
- Remove dead code (selectedEnvironment, goToReportsPage)
- Add @BRIEF to C2 test functions, missing {#each} keys
- Remove unused import json as _json from app.py

All 3200+ frontend tests pass; backend lint clean.
This commit is contained in:
busya
2026-07-20 21:41:34 +03:00
parent 49a566359a
commit 7bfc5553cf
44 changed files with 2808 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/tests/api/test_settings.py
View File

@@ -498,10 +498,22 @@ class TestUpdateLoggingConfig:
"""PATCH /settings/logging"""
def test_success(self):
from src.core.config_models import GlobalSettings, LoggingConfig
mock_config = MagicMock()
# Real settings object so merge preserves file_path and returns patched levels
real_settings = GlobalSettings(
logging=LoggingConfig(
level="INFO",
task_log_level="DEBUG",
enable_belief_state=True,
file_path="logs/app.log",
)
)
mock_cfg = MagicMock()
mock_cfg.settings = MagicMock()
mock_cfg.settings = real_settings
mock_config.get_config.return_value = mock_cfg
mock_config.update_global_settings.side_effect = lambda s: MagicMock(settings=s)
from src.dependencies import get_config_manager
client = _make_client({get_config_manager: lambda: mock_config})
@@ -512,6 +524,9 @@ class TestUpdateLoggingConfig:
data = resp.json()
assert data["level"] == "WARNING"
assert data["task_log_level"] == "INFO"
assert data["enable_belief_state"] is False
# file_path preserved across partial admin-style body
assert real_settings.logging.file_path == "logs/app.log"
# ── get_validation_policies ──

106
backend/tests/test_core/test_log_export_and_bulk.py Normal file
View File

@@ -0,0 +1,106 @@
# #region Test.Core.LogExportBulk [C:2] [TYPE Module] [SEMANTICS test,logs,export,bulk,gil]
# @BRIEF Burst buffer policy + JSONL export record shaping.
from datetime import UTC, datetime
from unittest.mock import MagicMock
import pytest
from src.core.task_manager.event_bus import EventBus
from src.core.task_manager.log_export import row_to_export_record, redact_text
from src.core.task_manager.models import LogEntry
# #region test_trim_buffer_drops_debug_first [C:2]
# @BRIEF Verify buffer trimming drops DEBUG entries first while preserving ERROR/INFO.
def test_trim_buffer_drops_debug_first():
bus = EventBus(MagicMock())
bus.MAX_BUFFER_PER_TASK = 5
entries = [
LogEntry(level="DEBUG", message=f"d{i}") for i in range(10)
] + [
LogEntry(level="ERROR", message="boom"),
LogEntry(level="INFO", message="ok"),
]
trimmed = bus._trim_buffer(entries)
assert len(trimmed) <= 5
assert any(e.level == "ERROR" for e in trimmed)
assert bus._dropped_debug >= 1
# #endregion test_trim_buffer_drops_debug_first
# #region test_export_record_cot_and_redaction [C:2]
# @BRIEF Verify CoT JSON parse, metadata redaction, and credential stripping.
def test_export_record_cot_and_redaction():
import json
row = {
"task_id": "abc",
"timestamp": datetime.now(UTC),
"level": "INFO",
"source": "plugin",
"message": json.dumps(
{
"marker": "REASON",
"intent": "Apply policy",
"src": "x.y",
"trace_id": "t1",
}
),
"metadata": {"password": "secret", "n": 1},
}
rec = row_to_export_record(row, plugin_id="migration")
assert rec["marker"] == "REASON"
assert rec["intent"] == "Apply policy"
assert rec["metadata"]["password"] == "***"
assert "redacted" in redact_text("API Key decrypted (first 8 chars): sk-abcdefgh")
# #endregion test_export_record_cot_and_redaction
# #region test_add_log_respects_task_log_level [C:2]
# @BRIEF Verify DEBUG log entries are filtered when task_log_level is INFO.
@pytest.mark.asyncio
async def test_add_log_respects_task_log_level(monkeypatch):
from src.core import logger as logger_mod
monkeypatch.setattr(logger_mod, "_task_log_level", "INFO")
bus = EventBus(MagicMock())
await bus.add_log("t1", "DEBUG", "noise")
assert bus._log_buffer.get("t1") in (None, [])
await bus.add_log("t1", "INFO", "ok")
assert len(bus._log_buffer["t1"]) == 1
# #endregion test_add_log_respects_task_log_level
# #region test_task_id_contextvar [C:2]
# @BRIEF Verify task_id ContextVar propagates into CotJsonFormatter output.
def test_task_id_contextvar_stamps_formatter():
import json
import logging
from ss_tools.shared.cot_logger import clear_task_id, set_task_id
from ss_tools.shared.logger import CotJsonFormatter
set_task_id("task-xyz-123")
try:
fmt = CotJsonFormatter()
record = logging.LogRecord(
name="test",
level=logging.INFO,
pathname=__file__,
lineno=1,
msg="hello",
args=(),
exc_info=None,
)
record.marker = "REASON"
record.intent = "Do work"
record.src = "unit.test"
line = fmt.format(record)
obj = json.loads(line)
assert obj.get("task_id") == "task-xyz-123"
assert obj.get("payload", {}).get("task_id") == "task-xyz-123"
finally:
clear_task_id()
# #endregion test_task_id_contextvar
# #endregion Test.Core.LogExportBulk

212
backend/tests/test_core/test_logging_hot_apply.py Normal file
View File

@@ -0,0 +1,212 @@
# #region Test.Core.LoggingHotApply [C:3] [TYPE Module] [SEMANTICS test,logging,hot-apply,protocol]
# @BRIEF Runtime reconfigure of logging level on settings update + protocol noise bounds.
# @RELATION BINDS_TO -> [ConfigManager, configure_logger]
# @TEST_EDGE level_stuck_without_reconfigure -> previously failed; now live level tracks settings
from pathlib import Path
import logging
import sys
from unittest.mock import MagicMock, patch
import pytest
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))
from src.core.config_models import GlobalSettings, LoggingConfig
from src.core.logger import configure_logger, logger
_MODULE = "src.core.config_manager"
def _build_manager():
from src.core.config_manager import ConfigManager
mgr = object.__new__(ConfigManager)
mgr.config_path = Path("config.json")
mgr.raw_payload = {}
from src.core.config_models import AppConfig
mgr.config = AppConfig(environments=[], settings=GlobalSettings())
return mgr
# #region test_update_global_settings_hot_applies_log_level [C:2]
def test_update_global_settings_hot_applies_log_level():
"""FR-005: changing settings.logging.level reconfigures live loggers without restart."""
from ss_tools.shared.cot_logger import cot_logger
configure_logger(
LoggingConfig(level="DEBUG", task_log_level="DEBUG", enable_belief_state=True)
)
assert logger.level == logging.DEBUG
assert cot_logger.level == logging.DEBUG
mgr = _build_manager()
new_settings = GlobalSettings(
logging=LoggingConfig(
level="INFO",
task_log_level="WARNING",
enable_belief_state=True,
file_path=None,
)
)
with patch(f"{_MODULE}.belief_scope"), patch.object(mgr, "save"):
mgr.update_global_settings(new_settings)
assert logger.level == logging.INFO
assert cot_logger.level == logging.INFO
# DEBUG records must be filtered after INFO apply
class _Capture(logging.Handler):
def __init__(self):
super().__init__()
self.records = []
def emit(self, record):
self.records.append(record)
h = _Capture()
h.setLevel(logging.DEBUG)
logger.addHandler(h)
try:
logger.debug("should-not-surface")
logger.info("should-surface")
names = [r.levelname for r in h.records]
# Handler is DEBUG but logger is INFO → only INFO passes logger filter
assert "DEBUG" not in names
assert "INFO" in names
finally:
logger.removeHandler(h)
# #endregion test_update_global_settings_hot_applies_log_level
# #region test_dashboard_projection_samples_are_debug [C:2]
@pytest.mark.asyncio
async def test_dashboard_projection_samples_are_debug(caplog):
"""Per-dashboard actor samples must not appear at INFO (high-frequency → DEBUG)."""
from src.core.superset_client._dashboards_list import SupersetDashboardsListMixin
class _Client(SupersetDashboardsListMixin):
env = MagicMock(id="env-1")
def _extract_owner_labels(self, raw):
return []
def _extract_user_display(self, name, raw):
return name or None
def _sanitize_user_text(self, text):
return text
async def get_dashboards(self, query=None):
return 2, [
{
"id": 1,
"slug": "a",
"dashboard_title": "A",
"url": "/a",
"changed_on_utc": None,
"published": True,
"created_by": None,
"changed_by": None,
"changed_by_name": None,
"owners": [],
},
{
"id": 2,
"slug": "b",
"dashboard_title": "B",
"url": "/b",
"changed_on_utc": None,
"published": False,
"created_by": None,
"changed_by": None,
"changed_by_name": None,
"owners": [],
},
]
configure_logger(
LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
)
logger.propagate = True
caplog.set_level(logging.INFO)
client = _Client()
result = await client.get_dashboards_summary()
assert len(result) == 2
messages = [r.getMessage() for r in caplog.records]
assert not any("Dashboard actor projection sample" in m for m in messages)
assert any("Dashboard actor projection complete" in m for m in messages)
# #endregion test_dashboard_projection_samples_are_debug
# #region test_security_event_single_atom [C:2]
def test_security_event_single_atom(caplog):
"""Security audit emits one REASON atom (not recorded + details pair)."""
from src.core.auth.logger import log_security_event
configure_logger(
LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
)
logger.propagate = True
caplog.set_level(logging.INFO)
log_security_event("LOGIN_SUCCESS", "admin", {"source": "LOCAL", "password": "secret"})
security_msgs = [
r.getMessage()
for r in caplog.records
if "Security event" in r.getMessage()
]
assert len(security_msgs) == 1
assert "LOGIN_SUCCESS" in security_msgs[0]
assert "Security event details" not in " ".join(security_msgs)
# #endregion test_security_event_single_atom
# #region test_merge_logging_patch_preserves_file_path [C:2]
def test_merge_logging_patch_preserves_file_path():
"""Partial admin-style logging update must not clear file_path."""
from src.api.routes.settings import update_logging_config
import asyncio
existing = LoggingConfig(
level="DEBUG",
task_log_level="DEBUG",
enable_belief_state=True,
file_path="logs/app.log",
max_bytes=12345,
backup_count=3,
)
settings = GlobalSettings(logging=existing)
config = MagicMock()
config.get_config.return_value = MagicMock(settings=settings)
saved = {}
def _update(s):
saved["logging"] = s.logging
return MagicMock(settings=s)
config.update_global_settings.side_effect = _update
# Incoming body only sets the three admin fields (defaults wipe file_path)
incoming = LoggingConfig(
level="INFO",
task_log_level="WARNING",
enable_belief_state=False,
file_path=None,
)
result = asyncio.run(
update_logging_config(incoming, config_manager=config, _=None)
)
assert result.level == "INFO"
assert result.task_log_level == "WARNING"
assert result.enable_belief_state is False
assert saved["logging"].file_path == "logs/app.log"
assert saved["logging"].max_bytes == 12345
# #endregion test_merge_logging_patch_preserves_file_path
# #endregion Test.Core.LoggingHotApply

91
backend/tests/test_core/test_ws_log_handler.py
View File

@@ -1,7 +1,5 @@
# #region Test.WsLogHandler [C:3] [TYPE Module] [SEMANTICS test,logging,handler,websocket]
# @BRIEF Tests for core/ws_log_handler.py — LogEntry, WebSocketLogHandler.
# @RELATION BINDS_TO -> [WsLogHandlerModule]
# @BRIEF Tests for core/ws_log_handler.py — AppLogLine, WebSocketLogHandler.
from pathlib import Path
import sys
@@ -12,37 +10,19 @@ from unittest.mock import MagicMock, patch
import pytest
# #region test_log_entry [C:2] [TYPE Function]
# @BRIEF Test LogEntry Pydantic model.
class TestLogEntry:
def test_creates_with_minimal_fields(self):
from src.core.ws_log_handler import LogEntry
entry = LogEntry(level="INFO", message="test")
assert entry.level == "INFO"
assert entry.message == "test"
assert entry.context is None
assert entry.timestamp is not None
def test_creates_with_context(self):
from src.core.ws_log_handler import LogEntry
entry = LogEntry(level="ERROR", message="fail", context={"key": "val"})
assert entry.context == {"key": "val"}
# #endregion test_log_entry
# #region test_websocket_log_handler [C:2] [TYPE Function]
# @BRIEF Test WebSocketLogHandler emit and get_recent_logs.
# #region test_websocket_log_handler [C:2]
class TestWebSocketLogHandler:
@pytest.fixture
def handler(self):
from src.core.ws_log_handler import WebSocketLogHandler
return WebSocketLogHandler(capacity=100)
def test_init(self, handler):
assert handler.log_buffer.maxlen == 100
assert len(handler.log_buffer) == 0
def test_emit_stores_log_entry(self, handler):
def test_emit_stores_app_log_line(self, handler):
record = logging.LogRecord(
name="test",
level=logging.INFO,
@@ -56,59 +36,31 @@ class TestWebSocketLogHandler:
assert len(handler.log_buffer) == 1
entry = handler.log_buffer[0]
assert entry.level == "INFO"
assert entry.message == "Test message"
assert entry.context["name"] == "test"
assert entry.context["lineno"] == 10
assert "Test message" in entry.raw
assert entry.seq == 1
assert entry.logger_name == "test"
def test_emit_error_level(self, handler):
record = logging.LogRecord(
name="test",
level=logging.ERROR,
pathname="/test.py",
lineno=20,
msg="Error occurred",
args=(),
exc_info=None,
)
handler.emit(record)
assert handler.log_buffer[0].level == "ERROR"
def test_get_recent_logs_returns_list(self, handler):
record = logging.LogRecord("t", logging.INFO, "/t.py", 1, "msg", (), None)
handler.emit(record)
logs = handler.get_recent_logs()
assert len(logs) == 1
assert isinstance(logs, list)
def test_get_recent_logs_returns_copy(self, handler):
record = logging.LogRecord("t", logging.INFO, "/t.py", 1, "msg", (), None)
handler.emit(record)
logs = handler.get_recent_logs()
assert len(logs) == 1
# Verify it's a snapshot (list copy)
handler.emit(record)
assert len(logs) == 1 # Original list unchanged
def test_get_since(self, handler):
for i in range(3):
record = logging.LogRecord("t", logging.INFO, "/t.py", i, f"msg {i}", (), None)
handler.emit(record)
newer, cur = handler.get_since(1)
assert cur == 3
assert len(newer) == 2
assert newer[0].seq == 2
def test_capacity_limit(self):
from src.core.ws_log_handler import WebSocketLogHandler
small_handler = WebSocketLogHandler(capacity=3)
small = WebSocketLogHandler(capacity=3)
for i in range(5):
record = logging.LogRecord("t", logging.INFO, "/t.py", i, f"msg {i}", (), None)
small_handler.emit(record)
assert len(small_handler.log_buffer) == 3
small.emit(record)
assert len(small.log_buffer) == 3
assert small.current_seq() == 5
def test_emit_handles_exception_gracefully(self, handler):
"""If format raises, handleError is called. We mock format to raise."""
bad_record = MagicMock()
bad_record.levelname = "INFO"
bad_record.name = "test"
bad_record.pathname = "/test.py"
bad_record.lineno = 1
bad_record.funcName = "test"
bad_record.process = 123
bad_record.thread = 456
# Make handler.format raise
with patch.object(handler, "format", side_effect=Exception("Format failed")):
with patch.object(handler, "handleError") as mock_handle:
handler.emit(bad_record)
@@ -116,7 +68,8 @@ class TestWebSocketLogHandler:
def test_default_capacity(self):
from src.core.ws_log_handler import WebSocketLogHandler
h = WebSocketLogHandler()
assert h.log_buffer.maxlen == 1000
assert h.log_buffer.maxlen == 3000
# #endregion test_websocket_log_handler
# #endregion Test.WsLogHandler

4
backend/tests/test_log_persistence.py
View File

@@ -353,7 +353,9 @@ class TestLogPersistence:
s.commit = lambda: (_ for _ in ()).throw(RuntimeError("db dead"))
return s
with patch("src.core.task_manager.persistence.TasksSessionLocal", broken_session):
self.service.add_logs("test-task-1", [entry]) # Should not raise
import pytest
with pytest.raises(RuntimeError, match="db dead"):
self.service.add_logs("test-task-1", [entry])
from src.models.task import TaskLogRecord
session = self.TestSessionLocal()