485 lines
16 KiB
Python
485 lines
16 KiB
Python
# #region Test.SQLGenerator [C:3] [TYPE Module] [SEMANTICS test, translate, sql, generator, upsert, insert]
|
|
# @BRIEF Verify SQLGenerator contracts — _normalize_timestamp_value, _quote_identifier, _encode_sql_value,
|
|
# _build_values_clause, generate_insert_sql, generate_upsert_sql, SQLGenerator.generate, SQLGenerator.generate_batch.
|
|
# @RELATION BINDS_TO -> [SQLGenerator]
|
|
# @TEST_EDGE: empty_columns -> raises ValueError
|
|
# @TEST_EDGE: empty_rows -> raises ValueError
|
|
# @TEST_EDGE: empty_table -> raises ValueError
|
|
# @TEST_EDGE: key_columns_empty_for_upsert -> raises ValueError
|
|
# @TEST_EDGE: all_columns_are_keys -> DO NOTHING
|
|
# @TEST_EDGE: clickhouse_dialect -> Backtick quoting
|
|
# @TEST_EDGE: unknown_dialect -> ANSI fallback
|
|
# @TEST_EDGE: timestamp_normalization -> Converts Unix timestamps
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from src.plugins.translate.sql_generator import (
|
|
_normalize_timestamp_value,
|
|
_quote_identifier,
|
|
_encode_sql_value,
|
|
_build_values_clause,
|
|
generate_insert_sql,
|
|
generate_upsert_sql,
|
|
SQLGenerator,
|
|
POSTGRESQL_DIALECTS,
|
|
CLICKHOUSE_DIALECTS,
|
|
BACKTICK_DIALECTS,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _normalize_timestamp_value
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestNormalizeTimestampValue:
|
|
"""Verify _normalize_timestamp_value."""
|
|
|
|
def test_none_input(self):
|
|
assert _normalize_timestamp_value(None) is None
|
|
|
|
def test_string_not_numeric(self):
|
|
assert _normalize_timestamp_value("hello") is None
|
|
|
|
def test_unix_seconds(self):
|
|
# 2024-01-15 12:00:00 UTC = 1705320000
|
|
result = _normalize_timestamp_value(1705320000)
|
|
assert result == "2024-01-15"
|
|
|
|
def test_unix_milliseconds(self):
|
|
# 2024-01-15 12:00:00 UTC = 1705320000000 ms
|
|
result = _normalize_timestamp_value(1705320000000)
|
|
assert result == "2024-01-15"
|
|
|
|
def test_string_unix_timestamp(self):
|
|
result = _normalize_timestamp_value("1705320000")
|
|
assert result == "2024-01-15"
|
|
|
|
def test_string_unix_millis(self):
|
|
result = _normalize_timestamp_value("1705320000000.0")
|
|
assert result == "2024-01-15"
|
|
|
|
def test_out_of_range_small(self):
|
|
# Too small to be a Unix timestamp
|
|
assert _normalize_timestamp_value(123) is None
|
|
|
|
def test_out_of_range_large(self):
|
|
# Way too large
|
|
assert _normalize_timestamp_value(1e16) is None
|
|
|
|
def test_negative_value(self):
|
|
assert _normalize_timestamp_value(-1) is None
|
|
|
|
def test_overflow_error(self):
|
|
# datetime.fromtimestamp can raise OverflowError for extreme values
|
|
# 1e15 is still within range for most systems
|
|
result = _normalize_timestamp_value(9999999999) # ~2286
|
|
assert result is not None
|
|
|
|
def test_float_milliseconds(self):
|
|
result = _normalize_timestamp_value(1705320000000.0)
|
|
assert result == "2024-01-15"
|
|
|
|
def test_empty_string(self):
|
|
assert _normalize_timestamp_value("") is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _quote_identifier
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestQuoteIdentifier:
|
|
"""Verify _quote_identifier."""
|
|
|
|
def test_postgresql_double_quotes(self):
|
|
result = _quote_identifier("my_column", "postgresql")
|
|
assert result == '"my_column"'
|
|
|
|
def test_redshift_double_quotes(self):
|
|
result = _quote_identifier("my_column", "redshift")
|
|
assert result == '"my_column"'
|
|
|
|
def test_clickhouse_backticks(self):
|
|
result = _quote_identifier("my_column", "clickhouse")
|
|
assert result == "`my_column`"
|
|
|
|
def test_clickhousedb_backticks(self):
|
|
result = _quote_identifier("my_column", "clickhousedb")
|
|
assert result == "`my_column`"
|
|
|
|
def test_mysql_backticks(self):
|
|
result = _quote_identifier("my_column", "mysql")
|
|
assert result == "`my_column`"
|
|
|
|
def test_unknown_dialect_ansi_fallback(self):
|
|
result = _quote_identifier("my_column", "mssql")
|
|
assert result == '"my_column"'
|
|
|
|
def test_removes_existing_quotes(self):
|
|
result = _quote_identifier('"my_column"', "postgresql")
|
|
assert result == '"my_column"'
|
|
|
|
def test_removes_backticks(self):
|
|
result = _quote_identifier("`my_column`", "clickhouse")
|
|
assert result == "`my_column`"
|
|
|
|
def test_removes_brackets(self):
|
|
result = _quote_identifier("[my_column]", "postgresql")
|
|
assert result == '"my_column"'
|
|
|
|
def test_empty_identifier(self):
|
|
assert _quote_identifier("", "postgresql") == ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _encode_sql_value
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestEncodeSqlValue:
|
|
"""Verify _encode_sql_value."""
|
|
|
|
def test_none(self):
|
|
assert _encode_sql_value(None) == "NULL"
|
|
|
|
def test_bool_true(self):
|
|
assert _encode_sql_value(True) == "TRUE"
|
|
|
|
def test_bool_false(self):
|
|
assert _encode_sql_value(False) == "FALSE"
|
|
|
|
def test_integer(self):
|
|
assert _encode_sql_value(42) == "42"
|
|
|
|
def test_float(self):
|
|
assert _encode_sql_value(3.14) == "3.14"
|
|
|
|
def test_string(self):
|
|
assert _encode_sql_value("hello") == "'hello'"
|
|
|
|
def test_string_with_quote(self):
|
|
assert _encode_sql_value("it's") == "'it''s'"
|
|
|
|
def test_clickhouse_with_timestamp_string(self):
|
|
result = _encode_sql_value("1705320000", dialect="clickhouse")
|
|
assert result == "'2024-01-15'"
|
|
|
|
def test_clickhouse_with_timestamp_number(self):
|
|
result = _encode_sql_value(1705320000, dialect="clickhouse")
|
|
assert result == "'2024-01-15'"
|
|
|
|
def test_clickhouse_regular_string(self):
|
|
result = _encode_sql_value("hello", dialect="clickhouse")
|
|
assert result == "'hello'"
|
|
|
|
def test_clickhouse_no_timestamp_empty_string(self):
|
|
result = _encode_sql_value("", dialect="clickhouse")
|
|
assert result == "''"
|
|
|
|
def test_clickhouse_none(self):
|
|
assert _encode_sql_value(None, dialect="clickhouse") == "NULL"
|
|
|
|
def test_non_clickhouse_passes_through(self):
|
|
result = _encode_sql_value("1705320000", dialect="postgresql")
|
|
assert result == "'1705320000'" # Not normalized
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _build_values_clause
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestBuildValuesClause:
|
|
"""Verify _build_values_clause."""
|
|
|
|
def test_single_row(self):
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
result = _build_values_clause(["col1", "col2"], rows)
|
|
assert result == "('a', 1)"
|
|
|
|
def test_multiple_rows(self):
|
|
rows = [
|
|
{"col1": "a", "col2": 1},
|
|
{"col1": "b", "col2": 2},
|
|
]
|
|
result = _build_values_clause(["col1", "col2"], rows)
|
|
assert "('a', 1)" in result
|
|
assert "('b', 2)" in result
|
|
assert ",\n" in result
|
|
|
|
def test_quoted_column_lookup(self):
|
|
"""Columns may be quoted; row keys are unquoted."""
|
|
rows = [{"col1": "a"}]
|
|
result = _build_values_clause(['"col1"'], rows)
|
|
assert result == "('a',)" or result == "('a')"
|
|
|
|
def test_null_value(self):
|
|
rows = [{"col1": None}]
|
|
result = _build_values_clause(["col1"], rows)
|
|
assert "NULL" in result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# generate_insert_sql
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestGenerateInsertSql:
|
|
"""Verify generate_insert_sql."""
|
|
|
|
def test_basic_insert(self):
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql = generate_insert_sql("public", "my_table", ["col1", "col2"], rows)
|
|
assert sql.startswith("INSERT INTO public.my_table")
|
|
assert "VALUES" in sql
|
|
assert "('a', 1)" in sql
|
|
|
|
def test_no_schema(self):
|
|
rows = [{"col1": "a"}]
|
|
sql = generate_insert_sql(None, "my_table", ["col1"], rows)
|
|
assert sql.startswith("INSERT INTO my_table")
|
|
|
|
def test_empty_table_raises(self):
|
|
with pytest.raises(ValueError, match="target_table is required"):
|
|
generate_insert_sql(None, "", ["col1"], [{"col1": "a"}])
|
|
|
|
def test_empty_columns_raises(self):
|
|
with pytest.raises(ValueError, match="At least one column is required"):
|
|
generate_insert_sql(None, "t", [], [{"col1": "a"}])
|
|
|
|
def test_empty_rows_raises(self):
|
|
with pytest.raises(ValueError, match="At least one row is required"):
|
|
generate_insert_sql(None, "t", ["col1"], [])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# generate_upsert_sql
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestGenerateUpsertSql:
|
|
"""Verify generate_upsert_sql."""
|
|
|
|
def test_basic_upsert(self):
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql = generate_upsert_sql(
|
|
"public", "my_table", ["col1", "col2"], ["col1"], rows,
|
|
)
|
|
assert "INSERT INTO" in sql
|
|
assert "ON CONFLICT" in sql
|
|
assert "DO UPDATE SET" in sql
|
|
assert "col2 = EXCLUDED.col2" in sql
|
|
|
|
def test_all_columns_are_keys_do_nothing(self):
|
|
"""When all columns are conflict keys, use DO NOTHING."""
|
|
rows = [{"col1": "a"}]
|
|
sql = generate_upsert_sql(
|
|
None, "my_table", ["col1"], ["col1"], rows,
|
|
)
|
|
assert "ON CONFLICT" in sql
|
|
assert "DO NOTHING" in sql
|
|
assert "DO UPDATE" not in sql
|
|
|
|
def test_no_schema(self):
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql = generate_upsert_sql(
|
|
None, "my_table", ["col1", "col2"], ["col1"], rows,
|
|
)
|
|
assert sql.startswith("INSERT INTO my_table")
|
|
|
|
def test_empty_table_raises(self):
|
|
with pytest.raises(ValueError, match="target_table is required"):
|
|
generate_upsert_sql(None, "", ["col1"], ["col1"], [{"col1": "a"}])
|
|
|
|
def test_empty_columns_raises(self):
|
|
with pytest.raises(ValueError, match="At least one column is required"):
|
|
generate_upsert_sql(None, "t", [], ["col1"], [{"col1": "a"}])
|
|
|
|
def test_empty_key_columns_raises(self):
|
|
with pytest.raises(ValueError, match="key_columns are required"):
|
|
generate_upsert_sql(None, "t", ["col1"], [], [{"col1": "a"}])
|
|
|
|
def test_empty_rows_raises(self):
|
|
with pytest.raises(ValueError, match="At least one row is required"):
|
|
generate_upsert_sql(None, "t", ["col1"], ["col1"], [])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SQLGenerator.generate
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSQLGeneratorGenerate:
|
|
"""Verify SQLGenerator.generate."""
|
|
|
|
def test_postgresql_insert(self):
|
|
rows = [{"col1": "a"}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="postgresql",
|
|
target_schema="public",
|
|
target_table="my_table",
|
|
columns=["col1"],
|
|
rows=rows,
|
|
upsert_strategy="INSERT",
|
|
)
|
|
assert count == 1
|
|
assert '"public"."my_table"' in sql
|
|
assert '"col1"' in sql
|
|
|
|
def test_postgresql_upsert(self):
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="postgresql",
|
|
target_schema="public",
|
|
target_table="my_table",
|
|
columns=["col1", "col2"],
|
|
rows=rows,
|
|
key_columns=["col1"],
|
|
upsert_strategy="MERGE",
|
|
)
|
|
assert count == 1
|
|
assert "ON CONFLICT" in sql
|
|
|
|
def test_clickhouse_insert(self):
|
|
rows = [{"col1": "a"}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="clickhouse",
|
|
target_schema="default",
|
|
target_table="my_table",
|
|
columns=["col1"],
|
|
rows=rows,
|
|
)
|
|
assert count == 1
|
|
assert "`default`.`my_table`" in sql
|
|
assert "`col1`" in sql
|
|
|
|
def test_clickhouse_upsert_fallsback_to_insert(self):
|
|
"""ClickHouse does not support ON CONFLICT; fallback to plain INSERT."""
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="clickhouse",
|
|
target_schema="default",
|
|
target_table="my_table",
|
|
columns=["col1", "col2"],
|
|
rows=rows,
|
|
key_columns=["col1"],
|
|
upsert_strategy="MERGE",
|
|
)
|
|
assert count == 1
|
|
assert "ON CONFLICT" not in sql
|
|
|
|
def test_unknown_dialect_insert(self):
|
|
rows = [{"col1": "a"}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="mssql",
|
|
target_schema="dbo",
|
|
target_table="my_table",
|
|
columns=["col1"],
|
|
rows=rows,
|
|
)
|
|
assert count == 1
|
|
assert '"dbo"."my_table"' in sql
|
|
|
|
def test_unknown_dialect_upsert_fallback(self):
|
|
"""Unknown dialect falls back to INSERT with warning."""
|
|
rows = [{"col1": "a", "col2": 1}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="mssql",
|
|
target_schema="dbo",
|
|
target_table="my_table",
|
|
columns=["col1", "col2"],
|
|
rows=rows,
|
|
key_columns=["col1"],
|
|
upsert_strategy="MERGE",
|
|
)
|
|
assert count == 1
|
|
assert "ON CONFLICT" not in sql
|
|
|
|
def test_no_schema(self):
|
|
rows = [{"col1": "a"}]
|
|
sql, count = SQLGenerator.generate(
|
|
dialect="postgresql",
|
|
target_schema=None,
|
|
target_table="my_table",
|
|
columns=["col1"],
|
|
rows=rows,
|
|
)
|
|
assert '"my_table"' in sql
|
|
|
|
def test_empty_table_raises(self):
|
|
with pytest.raises(ValueError, match="target_table is required"):
|
|
SQLGenerator.generate(
|
|
dialect="postgresql", target_schema=None,
|
|
target_table="", columns=["c"], rows=[{"c": "a"}],
|
|
)
|
|
|
|
def test_empty_columns_raises(self):
|
|
with pytest.raises(ValueError, match="At least one column is required"):
|
|
SQLGenerator.generate(
|
|
dialect="postgresql", target_schema=None,
|
|
target_table="t", columns=[], rows=[{"c": "a"}],
|
|
)
|
|
|
|
def test_empty_rows_raises(self):
|
|
with pytest.raises(ValueError, match="At least one row is required"):
|
|
SQLGenerator.generate(
|
|
dialect="postgresql", target_schema=None,
|
|
target_table="t", columns=["c"], rows=[],
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SQLGenerator.generate_batch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSQLGeneratorGenerateBatch:
|
|
"""Verify SQLGenerator.generate_batch."""
|
|
|
|
def test_empty_rows(self):
|
|
statements = SQLGenerator.generate_batch(
|
|
dialect="postgresql",
|
|
target_schema=None,
|
|
target_table="t",
|
|
columns=["c"],
|
|
rows=[],
|
|
)
|
|
assert statements == []
|
|
|
|
def test_splits_large_batch(self):
|
|
rows = [{"c": str(i)} for i in range(1500)]
|
|
statements = SQLGenerator.generate_batch(
|
|
dialect="postgresql",
|
|
target_schema=None,
|
|
target_table="t",
|
|
columns=["c"],
|
|
rows=rows,
|
|
max_rows_per_statement=500,
|
|
)
|
|
assert len(statements) == 3
|
|
for sql, count in statements:
|
|
assert count == 500
|
|
|
|
def test_single_batch(self):
|
|
rows = [{"c": "a"}, {"c": "b"}]
|
|
statements = SQLGenerator.generate_batch(
|
|
dialect="postgresql",
|
|
target_schema=None,
|
|
target_table="t",
|
|
columns=["c"],
|
|
rows=rows,
|
|
max_rows_per_statement=500,
|
|
)
|
|
assert len(statements) == 1
|
|
assert statements[0][1] == 2
|
|
|
|
def test_with_upsert(self):
|
|
rows = [{"c": "a", "d": 1}]
|
|
statements = SQLGenerator.generate_batch(
|
|
dialect="postgresql",
|
|
target_schema="public",
|
|
target_table="t",
|
|
columns=["c", "d"],
|
|
rows=rows,
|
|
key_columns=["c"],
|
|
upsert_strategy="MERGE",
|
|
)
|
|
assert len(statements) == 1
|
|
assert "ON CONFLICT" in statements[0][0]
|
|
# #endregion Test.SQLGenerator
|