busya
f75c15dbc6
- 10 translate plugin test files (100% coverage on 12 modules) - assistant/handler tools: 85+ tests covering dispatch, registry, resolvers, routes, llm_planner, 13 tool handlers - clean release: artifact_catalog_loader, mappers, approval, publication tests - API routes: translate_helpers, validation_service extensions, datasets to 100% - notifications: providers/service tests - services: profile_preference_service - docs/orthogonal-test-report.md — full speckit.tests audit - Fixes: 3 git_base async mock failures, 4 assistant handler permission-check patches - .gitignore: coverage artifacts
219 lines
9.0 KiB
Python
219 lines
9.0 KiB
Python
# #region Test.PreviewPromptHelpers [C:3] [TYPE Module] [SEMANTICS test, translate, preview, token, budget, helpers]
|
|
# @BRIEF Tests for preview_prompt_helpers.py — estimate_token_budget_for_rows, compute_build_token_metadata.
|
|
# @RELATION BINDS_TO -> [preview_prompt_helpers]
|
|
# @TEST_CONTRACT: estimate_token_budget_for_rows -> dict | adjusts rows based on token budget
|
|
# @TEST_CONTRACT: compute_build_token_metadata -> dict | returns token estimates with cost warning
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent / "src"))
|
|
|
|
import os
|
|
os.environ.setdefault("SECRET_KEY", "test-secret-key-for-testing")
|
|
os.environ.setdefault("DATABASE_URL", "sqlite:///:memory:")
|
|
os.environ.setdefault("AUTH_DATABASE_URL", "sqlite:///:memory:")
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from src.plugins.translate.preview_prompt_helpers import (
|
|
estimate_token_budget_for_rows,
|
|
compute_build_token_metadata,
|
|
)
|
|
|
|
|
|
class TestEstimateTokenBudgetForRows:
|
|
"""estimate_token_budget_for_rows — adjust row count based on budget."""
|
|
|
|
def test_no_adjustment_needed(self):
|
|
"""When budget fits, rows unchanged."""
|
|
job = MagicMock()
|
|
job.translation_column = "name"
|
|
job.context_columns = []
|
|
source_rows = [{"name": "hello"}, {"name": "world"}]
|
|
|
|
with patch("src.plugins.translate.preview_prompt_helpers.estimate_token_budget") as mock_budget:
|
|
mock_budget.return_value = {
|
|
"batch_size_adjusted": 5,
|
|
"estimated_input_tokens": 100,
|
|
"estimated_output_tokens": 200,
|
|
"batch_size": 2,
|
|
}
|
|
result = estimate_token_budget_for_rows(source_rows, ["ru"], job, None)
|
|
|
|
assert result["actual_row_count"] == 2
|
|
assert result["source_rows"] == source_rows
|
|
assert result["token_budget"]["batch_size_adjusted"] == 5
|
|
|
|
def test_adjustment_reduces_rows(self):
|
|
"""When budget is tight, rows are truncated to fit."""
|
|
job = MagicMock()
|
|
job.translation_column = "name"
|
|
job.context_columns = []
|
|
source_rows = [{"name": f"row{i}"} for i in range(10)]
|
|
|
|
with patch("src.plugins.translate.preview_prompt_helpers.estimate_token_budget") as mock_budget:
|
|
# First call: batch_size_adjusted smaller than actual
|
|
# Second call: after truncation
|
|
mock_budget.side_effect = [
|
|
{
|
|
"batch_size_adjusted": 3,
|
|
"estimated_input_tokens": 1000,
|
|
"estimated_output_tokens": 2000,
|
|
"batch_size": 10,
|
|
},
|
|
{
|
|
"batch_size_adjusted": 3,
|
|
"estimated_input_tokens": 300,
|
|
"estimated_output_tokens": 600,
|
|
"batch_size": 3,
|
|
},
|
|
]
|
|
result = estimate_token_budget_for_rows(source_rows, ["ru", "de"], job, None)
|
|
|
|
assert result["actual_row_count"] == 3
|
|
assert len(result["source_rows"]) == 3
|
|
assert result["token_budget"]["batch_size"] == 3
|
|
|
|
def test_empty_rows(self):
|
|
"""Empty source rows handled."""
|
|
job = MagicMock()
|
|
job.translation_column = "name"
|
|
job.context_columns = []
|
|
|
|
with patch("src.plugins.translate.preview_prompt_helpers.estimate_token_budget") as mock_budget:
|
|
mock_budget.return_value = {
|
|
"batch_size_adjusted": 0,
|
|
"estimated_input_tokens": 0,
|
|
"estimated_output_tokens": 0,
|
|
"batch_size": 0,
|
|
}
|
|
result = estimate_token_budget_for_rows([], ["ru"], job, None)
|
|
|
|
assert result["actual_row_count"] == 0
|
|
assert result["source_rows"] == []
|
|
|
|
def test_provider_model_passed(self):
|
|
"""Provider model string passed to estimate_token_budget."""
|
|
job = MagicMock()
|
|
job.translation_column = "name"
|
|
job.context_columns = []
|
|
|
|
with patch("src.plugins.translate.preview_prompt_helpers.estimate_token_budget") as mock_budget:
|
|
mock_budget.return_value = {
|
|
"batch_size_adjusted": 2,
|
|
"estimated_input_tokens": 100,
|
|
"estimated_output_tokens": 200,
|
|
"batch_size": 2,
|
|
}
|
|
result = estimate_token_budget_for_rows(
|
|
[{"name": "hello"}], ["ru"], job, "gpt-4o-mini"
|
|
)
|
|
|
|
assert result["actual_row_count"] == 1
|
|
# verify provider_model was passed (checked via call args)
|
|
call_kwargs = mock_budget.call_args[1]
|
|
assert call_kwargs["provider_info"] == "gpt-4o-mini"
|
|
|
|
|
|
class TestComputeBuildTokenMetadata:
|
|
"""compute_build_token_metadata — token estimation metadata."""
|
|
|
|
def test_basic_metadata(self):
|
|
"""Returns all expected keys with estimates."""
|
|
with patch("src.plugins.translate.preview_prompt_helpers.TokenEstimator") as MockTE:
|
|
MockTE.estimate_prompt_tokens.return_value = 100
|
|
MockTE.estimate_output_tokens.return_value = 200
|
|
MockTE.estimate_cost.return_value = 0.0006
|
|
MockTE.check_cost_warning.return_value = None
|
|
|
|
result = compute_build_token_metadata(
|
|
prompt="translate this",
|
|
row_meta=[{"id": 1}],
|
|
target_languages=["ru"],
|
|
num_languages=1,
|
|
dictionary_section="## Dictionary\nterm -> word",
|
|
actual_row_count=5,
|
|
sample_size=5,
|
|
)
|
|
|
|
assert result["prompt"] == "translate this"
|
|
assert result["row_meta"] == [{"id": 1}]
|
|
assert result["target_languages"] == ["ru"]
|
|
assert result["num_languages"] == 1
|
|
assert result["dictionary_section"] == "## Dictionary\nterm -> word"
|
|
assert result["sample_prompt_tokens"] == 100
|
|
assert result["sample_output_tokens"] == 200
|
|
assert result["sample_total_tokens"] == 300
|
|
assert result["sample_cost"] == 0.0006
|
|
assert result["total_est_rows"] == 50 # sample_size * 10
|
|
assert result["total_est_tokens"] > 0
|
|
assert result["total_est_cost"] > 0
|
|
assert result["cost_warning"] is None
|
|
assert result["actual_row_count"] == 5
|
|
|
|
def test_with_cost_warning(self):
|
|
"""Cost warning returned when sample exceeds threshold."""
|
|
with patch("src.plugins.translate.preview_prompt_helpers.TokenEstimator") as MockTE:
|
|
MockTE.estimate_prompt_tokens.return_value = 500
|
|
MockTE.estimate_output_tokens.return_value = 2000
|
|
MockTE.estimate_cost.return_value = 0.005
|
|
MockTE.check_cost_warning.return_value = (
|
|
"Large preview — estimated 2500 tokens, ~$0.0050 cost (across 2 languages)"
|
|
)
|
|
|
|
result = compute_build_token_metadata(
|
|
prompt="translate all this text",
|
|
row_meta=[{"id": 1}, {"id": 2}],
|
|
target_languages=["ru", "de"],
|
|
num_languages=2,
|
|
dictionary_section="",
|
|
actual_row_count=2,
|
|
sample_size=100,
|
|
)
|
|
|
|
assert result["cost_warning"] is not None
|
|
assert "Large preview" in result["cost_warning"]
|
|
assert "2 languages" in result["cost_warning"]
|
|
|
|
def test_empty_prompt(self):
|
|
"""Empty prompt handled by TokenEstimator."""
|
|
with patch("src.plugins.translate.preview_prompt_helpers.TokenEstimator") as MockTE:
|
|
MockTE.estimate_prompt_tokens.return_value = 0
|
|
MockTE.estimate_output_tokens.return_value = 0
|
|
MockTE.estimate_cost.return_value = 0.0
|
|
MockTE.check_cost_warning.return_value = None
|
|
|
|
result = compute_build_token_metadata(
|
|
prompt="", row_meta=[], target_languages=[], num_languages=1,
|
|
dictionary_section="", actual_row_count=0, sample_size=0,
|
|
)
|
|
|
|
assert result["sample_prompt_tokens"] == 0
|
|
assert result["sample_output_tokens"] == 0
|
|
assert result["sample_total_tokens"] == 0
|
|
assert result["sample_cost"] == 0.0
|
|
|
|
def test_total_est_replaces_row_count_in_prompt(self):
|
|
"""total_est_tokens substitutes actual_row_count with total_est_rows in prompt."""
|
|
with patch("src.plugins.translate.preview_prompt_helpers.TokenEstimator") as MockTE:
|
|
MockTE.estimate_prompt_tokens.side_effect = lambda p: len(p)
|
|
MockTE.estimate_output_tokens.return_value = 200
|
|
MockTE.estimate_cost.return_value = 0.001
|
|
MockTE.check_cost_warning.return_value = None
|
|
|
|
result = compute_build_token_metadata(
|
|
prompt="Translate 5 rows",
|
|
row_meta=[{"id": 1}],
|
|
target_languages=["ru"],
|
|
num_languages=1,
|
|
dictionary_section="",
|
|
actual_row_count=5,
|
|
sample_size=5,
|
|
)
|
|
|
|
# Second call: prompt.replace("5", "50")
|
|
assert MockTE.estimate_prompt_tokens.call_count == 2
|
|
# #endregion Test.PreviewPromptHelpers
|