ss-tools/backend/tests/integration/test_translate_clickhouse.py

# #region TestTranslateClickHouseIntegration [C:3] [TYPE Module] [SEMANTICS test,translate,clickhouse,integration]
# @BRIEF Integration tests for ClickHouse insert path using testcontainers.
# @RELATION BINDS_TO -> [_batch_insert_module]
# @RELATION BINDS_TO -> [SQLGenerator]
#
# @TEST_CONTRACT ClickHouseInsert ->
#   SQL generation: clickhouse dialect -> valid INSERT SQL with backtick quoting
#   Rows with timestamps: timestamp values encode to YYYY-MM-DD format
#   Null handling: None values encode to NULL
#   UPSERT: clickhouse supports INSERT with key_cols (uses INSERT ... VALUES)
#
# @REQUIRES Docker daemon running (testcontainers starts clickhouse server)
import pytest
import uuid

from testcontainers.clickhouse import ClickHouseContainer

from src.plugins.translate.sql_generator import SQLGenerator, _encode_sql_value


# #region clickhouse_container [C:2] [TYPE Fixture]
@pytest.fixture(scope="module")
def clickhouse_container():
    """Start a ClickHouse server for the test module."""
    with ClickHouseContainer("clickhouse/clickhouse-server:24.3") as ch:
        yield ch
# #endregion clickhouse_container


# #region clickhouse_client [C:2] [TYPE Fixture]
@pytest.fixture(scope="module")
def clickhouse_client(clickhouse_container):
    """clickhouse-connect client connected to the ClickHouse container."""
    import clickhouse_connect

    client = clickhouse_connect.get_client(
        host=clickhouse_container.get_container_host_ip(),
        port=int(clickhouse_container.get_exposed_port(8123)),
        username="test",
        password="test",
    )
    yield client
    client.close()
# #endregion clickhouse_client


# #region TestClickHouseSQLGeneration [C:3] [TYPE Class]
class TestClickHouseSQLGeneration:
    """Verify SQL generation produces valid ClickHouse SQL."""

    def test_generate_insert(self):
        sql, count = SQLGenerator.generate(
            dialect="clickhouse",
            target_schema="",
            target_table="test_table",
            columns=["id", "name", "value"],
            rows=[
                {"id": 1, "name": "Alice", "value": "Hello"},
                {"id": 2, "name": "Bob", "value": "World"},
            ],
            key_columns=["id"],
            upsert_strategy="INSERT",
        )
        assert sql is not None
        assert "INSERT INTO" in sql
        assert "test_table" in sql
        assert "Alice" in sql
        assert "Bob" in sql
        assert count == 2

    def test_generate_with_backtick_quoting(self):
        sql, _ = SQLGenerator.generate(
            dialect="clickhouse",
            target_schema="",
            target_table="test_table",
            columns=["id", "translated-text"],
            rows=[{"id": 1, "translated-text": "hello"}],
            key_columns=["id"],
            upsert_strategy="INSERT",
        )
        assert "`translated-text`" in sql

    def test_encode_timestamp(self):
        result = _encode_sql_value("1726358400000.0", dialect="clickhouse")
        assert "2024" in result  # should produce a date string

    def test_encode_string(self):
        result = _encode_sql_value("Hello World", dialect="clickhouse")
        assert result == "'Hello World'"

    def test_encode_none(self):
        result = _encode_sql_value(None, dialect="clickhouse")
        assert result == "NULL"

    def test_encode_int(self):
        result = _encode_sql_value(42, dialect="clickhouse")
        assert result == "42"

    def test_encode_bool(self):
        result = _encode_sql_value(True, dialect="clickhouse")
        assert result == "TRUE"
# #endregion TestClickHouseSQLGeneration


# #region TestClickHouseRealExecution [C:3] [TYPE Class]
class TestClickHouseRealExecution:
    """Execute generated SQL against a real ClickHouse container."""

    @pytest.fixture(autouse=True)
    def setup_table(self, clickhouse_client):
        """Create test table before each test."""
        clickhouse_client.command("""
            CREATE TABLE IF NOT EXISTS test_translations (
                id UInt32,
                source_text String,
                translated_text String,
                lang_code String,
                is_original UInt8
            ) ENGINE = MergeTree()
            ORDER BY id
        """)
        yield
        clickhouse_client.command("DROP TABLE IF EXISTS test_translations")

    def test_insert_and_select(self, clickhouse_client):
        """Insert rows and verify they can be selected."""
        sql, count = SQLGenerator.generate(
            dialect="clickhouse",
            target_schema="",
            target_table="test_translations",
            columns=["id", "source_text", "translated_text", "lang_code", "is_original"],
            rows=[
                {"id": 1, "source_text": "Hello", "translated_text": "Привет", "lang_code": "ru", "is_original": 1},
                {"id": 1, "source_text": "Hello", "translated_text": "Привет", "lang_code": "ru", "is_original": 0},
            ],
            key_columns=["id"],
            upsert_strategy="INSERT",
        )
        assert sql is not None
        assert count == 2

        clickhouse_client.command(sql)

        result = clickhouse_client.query("SELECT count(*) FROM test_translations")
        assert result.result_rows[0][0] == 2

    def test_multiple_batches(self, clickhouse_client):
        """Insert in multiple chunks."""
        rows = [{"id": i, "source_text": f"text_{i}", "translated_text": f"trans_{i}", "lang_code": "ru", "is_original": 1} for i in range(10)]

        statements = SQLGenerator.generate_batch(
            dialect="clickhouse",
            target_schema="",
            target_table="test_translations",
            columns=["id", "source_text", "translated_text", "lang_code", "is_original"],
            rows=rows,
            key_columns=["id"],
            upsert_strategy="INSERT",
            max_rows_per_statement=3,
        )

        assert len(statements) == 4
        for sql, chunk_count in statements:
            clickhouse_client.command(sql)

        result = clickhouse_client.query("SELECT count(*) FROM test_translations")
        assert result.result_rows[0][0] == 10

    def test_drop_and_reinsert(self, clickhouse_client):
        """Drop rows to verify idempotency."""
        clickhouse_client.command("""
            INSERT INTO test_translations (id, source_text, translated_text, lang_code, is_original)
            VALUES (1, 'Hello', 'Привет', 'ru', 1)
        """)

        clickhouse_client.command("TRUNCATE TABLE test_translations")

        result = clickhouse_client.query("SELECT count(*) FROM test_translations")
        assert result.result_rows[0][0] == 0
# #endregion TestClickHouseRealExecution
# #endregion TestTranslateClickHouseIntegration