# #region TestTranslateClickHouseIntegration [C:3] [TYPE Module] [SEMANTICS test,translate,clickhouse,integration] # @BRIEF Integration tests for ClickHouse insert path using testcontainers. # @RELATION BINDS_TO -> [_batch_insert_module] # @RELATION BINDS_TO -> [SQLGenerator] # # @TEST_CONTRACT ClickHouseInsert -> # SQL generation: clickhouse dialect -> valid INSERT SQL with backtick quoting # Rows with timestamps: timestamp values encode to YYYY-MM-DD format # Null handling: None values encode to NULL # UPSERT: clickhouse supports INSERT with key_cols (uses INSERT ... VALUES) # # @REQUIRES Docker daemon running (testcontainers starts clickhouse server) import pytest import uuid from testcontainers.clickhouse import ClickHouseContainer from src.plugins.translate.sql_generator import SQLGenerator, _encode_sql_value # #region clickhouse_container [C:2] [TYPE Fixture] @pytest.fixture(scope="module") def clickhouse_container(): """Start a ClickHouse server for the test module.""" with ClickHouseContainer("clickhouse/clickhouse-server:24.3") as ch: yield ch # #endregion clickhouse_container # #region clickhouse_client [C:2] [TYPE Fixture] @pytest.fixture(scope="module") def clickhouse_client(clickhouse_container): """clickhouse-connect client connected to the ClickHouse container.""" import clickhouse_connect client = clickhouse_connect.get_client( host=clickhouse_container.get_container_host_ip(), port=int(clickhouse_container.get_exposed_port(8123)), username="test", password="test", ) yield client client.close() # #endregion clickhouse_client # #region TestClickHouseSQLGeneration [C:3] [TYPE Class] class TestClickHouseSQLGeneration: """Verify SQL generation produces valid ClickHouse SQL.""" def test_generate_insert(self): sql, count = SQLGenerator.generate( dialect="clickhouse", target_schema="", target_table="test_table", columns=["id", "name", "value"], rows=[ {"id": 1, "name": "Alice", "value": "Hello"}, {"id": 2, "name": "Bob", "value": "World"}, ], key_columns=["id"], upsert_strategy="INSERT", ) assert sql is not None assert "INSERT INTO" in sql assert "test_table" in sql assert "Alice" in sql assert "Bob" in sql assert count == 2 def test_generate_with_backtick_quoting(self): sql, _ = SQLGenerator.generate( dialect="clickhouse", target_schema="", target_table="test_table", columns=["id", "translated-text"], rows=[{"id": 1, "translated-text": "hello"}], key_columns=["id"], upsert_strategy="INSERT", ) assert "`translated-text`" in sql def test_encode_timestamp(self): result = _encode_sql_value("1726358400000.0", dialect="clickhouse") assert "2024" in result # should produce a date string def test_encode_string(self): result = _encode_sql_value("Hello World", dialect="clickhouse") assert result == "'Hello World'" def test_encode_none(self): result = _encode_sql_value(None, dialect="clickhouse") assert result == "NULL" def test_encode_int(self): result = _encode_sql_value(42, dialect="clickhouse") assert result == "42" def test_encode_bool(self): result = _encode_sql_value(True, dialect="clickhouse") assert result == "TRUE" # #endregion TestClickHouseSQLGeneration # #region TestClickHouseRealExecution [C:3] [TYPE Class] class TestClickHouseRealExecution: """Execute generated SQL against a real ClickHouse container.""" @pytest.fixture(autouse=True) def setup_table(self, clickhouse_client): """Create test table before each test.""" clickhouse_client.command(""" CREATE TABLE IF NOT EXISTS test_translations ( id UInt32, source_text String, translated_text String, lang_code String, is_original UInt8 ) ENGINE = MergeTree() ORDER BY id """) yield clickhouse_client.command("DROP TABLE IF EXISTS test_translations") def test_insert_and_select(self, clickhouse_client): """Insert rows and verify they can be selected.""" sql, count = SQLGenerator.generate( dialect="clickhouse", target_schema="", target_table="test_translations", columns=["id", "source_text", "translated_text", "lang_code", "is_original"], rows=[ {"id": 1, "source_text": "Hello", "translated_text": "Привет", "lang_code": "ru", "is_original": 1}, {"id": 1, "source_text": "Hello", "translated_text": "Привет", "lang_code": "ru", "is_original": 0}, ], key_columns=["id"], upsert_strategy="INSERT", ) assert sql is not None assert count == 2 clickhouse_client.command(sql) result = clickhouse_client.query("SELECT count(*) FROM test_translations") assert result.result_rows[0][0] == 2 def test_multiple_batches(self, clickhouse_client): """Insert in multiple chunks.""" rows = [{"id": i, "source_text": f"text_{i}", "translated_text": f"trans_{i}", "lang_code": "ru", "is_original": 1} for i in range(10)] statements = SQLGenerator.generate_batch( dialect="clickhouse", target_schema="", target_table="test_translations", columns=["id", "source_text", "translated_text", "lang_code", "is_original"], rows=rows, key_columns=["id"], upsert_strategy="INSERT", max_rows_per_statement=3, ) assert len(statements) == 4 for sql, chunk_count in statements: clickhouse_client.command(sql) result = clickhouse_client.query("SELECT count(*) FROM test_translations") assert result.result_rows[0][0] == 10 def test_drop_and_reinsert(self, clickhouse_client): """Drop rows to verify idempotency.""" clickhouse_client.command(""" INSERT INTO test_translations (id, source_text, translated_text, lang_code, is_original) VALUES (1, 'Hello', 'Привет', 'ru', 1) """) clickhouse_client.command("TRUNCATE TABLE test_translations") result = clickhouse_client.query("SELECT count(*) FROM test_translations") assert result.result_rows[0][0] == 0 # #endregion TestClickHouseRealExecution # #endregion TestTranslateClickHouseIntegration