From 880691c76110138331c312326a728d45601a6b59 Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Sat, 14 Nov 2026 01:00:00 +0000 Subject: [PATCH 1/3] feat(postgres): bulk-load DataFrames with COPY Load pandas DataFrames from Python models and seeds into Postgres with COPY FROM STDIN through a staging table, instead of batched INSERT ... VALUES. On 500k rows locally this took 3.1s instead of 56.2s. - Type the staging table from the DataFrame dtypes and cast into the target with _casted_columns, matching the VALUES path's results. Floats are staged as NUMERIC so rounding matches VALUES literals. - Stream CSV chunks of batch_size rows through cursor.copy_expert, with a random per-load NULL marker. Only _copy_df touches the driver API. - Fall back to VALUES for unsupported dtypes or values, a single-column frame containing `\.`, drivers without copy_expert, and RisingWave. - Drop the staging table after the load, and also when the COPY fails inside query_factory, which replace_query calls outside the SourceQuery context. - Make _get_temp_table respect MAX_IDENTIFIER_LENGTH by truncating the middle of the name, so long physical names get valid staging names. Fixes #5830 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Vamsi-klu --- docs/integrations/engines/postgres.md | 12 + sqlmesh/core/engine_adapter/base.py | 12 +- sqlmesh/core/engine_adapter/postgres.py | 194 ++++- sqlmesh/core/engine_adapter/risingwave.py | 4 + .../integration/test_integration_postgres.py | 788 +++++++++++++++++- tests/core/engine_adapter/test_base.py | 34 + tests/core/engine_adapter/test_postgres.py | 292 +++++++ tests/core/engine_adapter/test_risingwave.py | 15 + 8 files changed, 1345 insertions(+), 6 deletions(-) diff --git a/docs/integrations/engines/postgres.md b/docs/integrations/engines/postgres.md index cf1d3e4ce8..be2d3c6b70 100644 --- a/docs/integrations/engines/postgres.md +++ b/docs/integrations/engines/postgres.md @@ -23,3 +23,15 @@ pip install "sqlmesh[postgres]" | `role` | The role to use for authentication with the Postgres server | string | N | | `sslmode` | The security of the connection to the Postgres server | string | N | | `application_name` | The name of the application to use for the connection | string | N | + +### DataFrame loading + +When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those returned by Python models and used for seeds) with `COPY ... FROM STDIN (FORMAT csv)` instead of batched `INSERT ... VALUES` statements, which is much faster for large DataFrames. The data is copied into a temporary staging table, which is created in the same schema as the target table and then cast into the target table's column types. The staging table is dropped afterwards, including when the load fails. Because it is created in the target's schema, the connecting user needs the `CREATE` privilege on that schema, as for other SQLMesh DDL. + +No configuration is required and there is no option to disable this behavior. SQLMesh automatically falls back to `INSERT ... VALUES` when: + +- A column has a type or values that cannot be loaded with `COPY` identically. Supported columns are integers (up to 64 bits signed, up to 32 bits unsigned), `float64`, booleans, timestamps (with or without time zone), and strings. Other types, such as `float32`, 64-bit unsigned integers, `Decimal`, dates, bytes, dicts, lists, and categoricals, trigger the fallback. +- The DataFrame has a single column and one of its values is `\.`. +- The driver does not support `copy_expert` (for example, a driver other than `psycopg2`). + +The RisingWave engine always uses `INSERT ... VALUES`. diff --git a/sqlmesh/core/engine_adapter/base.py b/sqlmesh/core/engine_adapter/base.py index 2dd234f70a..2e3160b7e1 100644 --- a/sqlmesh/core/engine_adapter/base.py +++ b/sqlmesh/core/engine_adapter/base.py @@ -2921,9 +2921,15 @@ def _get_temp_table( Returns the name of the temp table that should be used for the given table name. """ table = t.cast(exp.Table, exp.to_table(table).copy()) - table.set( - "this", exp.to_identifier(f"__temp_{table.name}_{random_id(short=True)}", quoted=quoted) - ) + prefix, suffix = "__temp_", f"_{random_id(short=True)}" + name = table.name + if self.MAX_IDENTIFIER_LENGTH is not None: + # Cut the middle of the name so that the temp table name is a valid identifier + keep = max(self.MAX_IDENTIFIER_LENGTH - len(prefix) - len(suffix), 0) + if len(name) > keep: + tail = keep // 2 + name = name[: keep - tail] + (name[-tail:] if tail else "") + table.set("this", exp.to_identifier(f"{prefix}{name}{suffix}", quoted=quoted)) if table_only: table.set("db", None) diff --git a/sqlmesh/core/engine_adapter/postgres.py b/sqlmesh/core/engine_adapter/postgres.py index 6794169322..82c9be4afb 100644 --- a/sqlmesh/core/engine_adapter/postgres.py +++ b/sqlmesh/core/engine_adapter/postgres.py @@ -1,5 +1,7 @@ from __future__ import annotations +import csv +import io import logging import re import typing as t @@ -14,14 +16,59 @@ logical_merge, GrantsFromInfoSchemaMixin, ) -from sqlmesh.core.engine_adapter.shared import set_catalog +from sqlmesh.core.engine_adapter.shared import SourceQuery, set_catalog +from sqlmesh.utils import random_id if t.TYPE_CHECKING: + import pandas as pd + from sqlmesh.core._typing import TableName - from sqlmesh.core.engine_adapter._typing import DF, QueryOrDF + from sqlmesh.core.engine_adapter._typing import DF, Query, QueryOrDF logger = logging.getLogger(__name__) +# pandas dtypes that can be bulk loaded with COPY, mapped to the type of the staging table column. +# The staging column types are derived from the DataFrame (not from the target table) and the final +# `SELECT` casts them to the target types, which mirrors how the generic `VALUES` implementation types +# literals by their Python value and then casts. In particular floats are staged as an unbounded +# NUMERIC, because the literals emitted by the `VALUES` implementation are NUMERIC and e.g. casting +# 2.5 to an integer rounds differently for NUMERIC (3) and DOUBLE PRECISION (2). +_COPY_DTYPE_TO_STAGING_TYPE: t.Dict[str, str] = { + **{ + dtype: "BIGINT" + for dtype in ( + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "Int8", + "Int16", + "Int32", + "Int64", + "UInt8", + "UInt16", + "UInt32", + ) + }, + "float64": "NUMERIC", + "Float64": "NUMERIC", + "bool": "BOOLEAN", + "boolean": "BOOLEAN", +} +# What pandas' `infer_dtype` reports for object / string columns that can be copied. Python numbers in an +# object column (e.g. seeds, where NaN is replaced with None) are staged as NUMERIC, which is lossless. +_COPY_INFERRED_TO_STAGING_TYPE: t.Dict[str, str] = { + "string": "TEXT", + "empty": "TEXT", # only nulls + "boolean": "BOOLEAN", + "integer": "NUMERIC", + "floating": "NUMERIC", + "mixed-integer-float": "NUMERIC", +} + @set_catalog() class PostgresEngineAdapter( @@ -85,6 +132,149 @@ def _fetch_native_df( self._connection_pool.commit() return df + def _df_to_source_queries( + self, + df: DF, + target_columns_to_types: t.Dict[str, exp.DataType], + batch_size: int, + target_table: TableName, + source_columns: t.Optional[t.List[str]] = None, + ) -> t.List[SourceQuery]: + """Bulk loads the DataFrame into a staging table with COPY instead of using `VALUES` batches. + + The generic implementation is used when the DataFrame cannot be loaded safely with COPY. + """ + import pandas as pd + + assert isinstance(df, pd.DataFrame) + + # pandas truncates `na_rep` to 32 characters when writing float columns, so keep the marker short + null_marker = f"__sqlmesh_null_{random_id(short=True)}__" + # Same column selection and order as the generic implementation + ordered_df = df[list(source_columns or target_columns_to_types)] + staging_columns_to_types = ( + None + if ordered_df.empty or not hasattr(self.cursor, "copy_expert") + else self._get_copy_staging_columns_to_types(ordered_df, null_marker) + ) + if staging_columns_to_types is None: + return super()._df_to_source_queries( + df, + target_columns_to_types, + batch_size, + target_table, + source_columns=source_columns, + ) + + staging_table = self._get_temp_table(target_table or "pandas") + + def query_factory() -> Query: + # The factory can be called multiple times. The staging table is only loaded once, which + # assumes that every call would have produced the same result. + if not self.table_exists(staging_table): + try: + self.create_table(staging_table, staging_columns_to_types) + self._copy_df(staging_table, ordered_df, null_marker, batch_size) + except Exception: + # Callers such as `replace_query` invoke the factory outside of `with source_query`, + # so `cleanup_func` won't run and a committed staging table would leak. + try: + self._drop_copy_staging_table(staging_table) + except Exception: + logger.warning("Failed to drop staging table %s", staging_table.sql()) + raise + return exp.select( + *self._casted_columns(target_columns_to_types, source_columns=source_columns) + ).from_(staging_table) + + return [ + SourceQuery( + query_factory=query_factory, + cleanup_func=lambda: self._drop_copy_staging_table(staging_table), + ) + ] + + def _drop_copy_staging_table(self, staging_table: exp.Table) -> None: + try: + self.drop_table(staging_table) + except Exception as ex: + # The cleanup runs while an exception is propagating. If it left the transaction aborted + # the DROP fails with "current transaction is aborted", which would hide the original + # error. The staging table was created in that transaction, so its rollback removes it. + sqlstate = getattr(ex, "pgcode", None) or getattr(ex, "sqlstate", None) + if sqlstate != "25P02" or not self._connection_pool.is_transaction_active: + raise + logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql()) + + @staticmethod + def _get_copy_staging_columns_to_types( + df: pd.DataFrame, null_marker: str + ) -> t.Optional[t.Dict[str, exp.DataType]]: + """Returns the staging table column types for the DataFrame, or None if it can't be loaded with COPY. + + None is returned when a column has a dtype or values that wouldn't be read back by COPY exactly + as the generic `VALUES` implementation would write them, e.g. dicts, bytes, Decimals, dates or + categoricals, or when a string would be misread by the CSV parser. + """ + import pandas as pd + + staging_columns_to_types: t.Dict[str, exp.DataType] = {} + for column, series in df.items(): + dtype = series.dtype + staging_type: t.Optional[str] + if isinstance(dtype, pd.DatetimeTZDtype): + staging_type = "TIMESTAMPTZ" + elif pd.api.types.is_datetime64_dtype(dtype): + staging_type = "TIMESTAMP" + elif str(dtype) in _COPY_DTYPE_TO_STAGING_TYPE: + staging_type = _COPY_DTYPE_TO_STAGING_TYPE[str(dtype)] + elif dtype == object or isinstance(dtype, pd.StringDtype): + staging_type = _COPY_INFERRED_TO_STAGING_TYPE.get( + pd.api.types.infer_dtype(series, skipna=True) + ) + if staging_type == "TEXT": + if series.eq(null_marker).any(): + return None + # On PostgreSQL <= 17 an unquoted `\.` line ends the data, which happens if it is the only column + if len(df.columns) == 1 and series.eq("\\.").any(): + return None + else: + staging_type = None + if staging_type is None: + return None + staging_columns_to_types[str(column)] = exp.DataType.build( + staging_type, dialect=PostgresEngineAdapter.DIALECT + ) + return staging_columns_to_types + + def _copy_df( + self, table: exp.Table, df: pd.DataFrame, null_marker: str, batch_size: int + ) -> None: + """Loads the DataFrame into the table with `COPY ... FROM STDIN` in chunks of `batch_size` rows. + + This is the only place that calls the driver's COPY API (`psycopg2`'s `copy_expert`). + The copy runs on the adapter's cursor so it is part of the current transaction. + """ + columns = ", ".join(self._to_sql(exp.column(c, quoted=True)) for c in df.columns) + copy_sql = ( + f"COPY {self._to_sql(table)} ({columns}) " + f"FROM STDIN WITH (FORMAT csv, NULL '{null_marker}')" + ) + chunk_size = batch_size or max(len(df.index), 1) + for start in range(0, len(df.index), chunk_size): + buffer = io.StringIO() + df.iloc[start : start + chunk_size].to_csv( + buffer, + header=False, + index=False, + na_rep=null_marker, + quoting=csv.QUOTE_MINIMAL, + lineterminator="\n", + ) + buffer.seek(0) + # A text buffer is encoded by the driver with the connection's client_encoding + self.cursor.copy_expert(copy_sql, buffer) + def _create_table_like( self, target_table_name: TableName, diff --git a/sqlmesh/core/engine_adapter/risingwave.py b/sqlmesh/core/engine_adapter/risingwave.py index 61b44f5bbb..6d330ca4af 100644 --- a/sqlmesh/core/engine_adapter/risingwave.py +++ b/sqlmesh/core/engine_adapter/risingwave.py @@ -6,6 +6,7 @@ from sqlglot import exp +from sqlmesh.core.engine_adapter.base import EngineAdapter from sqlmesh.core.engine_adapter.postgres import PostgresEngineAdapter from sqlmesh.core.engine_adapter.shared import ( set_catalog, @@ -34,6 +35,9 @@ class RisingwaveEngineAdapter(PostgresEngineAdapter): MAX_IDENTIFIER_LENGTH = None SUPPORTS_GRANTS = False + # COPY into a staging table hasn't been verified on RisingWave, so DataFrames keep using `VALUES` + _df_to_source_queries = EngineAdapter._df_to_source_queries + def columns( self, table_name: TableName, include_pseudo_columns: bool = False ) -> t.Dict[str, exp.DataType]: diff --git a/tests/core/engine_adapter/integration/test_integration_postgres.py b/tests/core/engine_adapter/integration/test_integration_postgres.py index 2bc1dcc2e7..db3ed7a917 100644 --- a/tests/core/engine_adapter/integration/test_integration_postgres.py +++ b/tests/core/engine_adapter/integration/test_integration_postgres.py @@ -1,10 +1,16 @@ import typing as t import uuid +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from unittest.mock import patch + +import numpy as np # noqa: TID253 +import pandas as pd # noqa: TID253 import pytest from pytest import FixtureRequest +from pytest_mock import MockerFixture from pathlib import Path -from sqlmesh.core.engine_adapter import PostgresEngineAdapter +from sqlmesh.core.engine_adapter import EngineAdapter, PostgresEngineAdapter from sqlmesh.core.config import Config, DuckDBConnectionConfig from sqlmesh.core.config.common import VirtualEnvironmentMode from tests.core.engine_adapter.integration import TestContext @@ -1230,3 +1236,783 @@ def test_grants_plan_scd_type_2_with_vde_dev_only( ) assert roles["analyst"]["username"] in dev_physical_grants.get("SELECT", []) assert roles["etl_user"]["username"] in dev_physical_grants.get("INSERT", []) + + +# -------------------------------------------------------------------------------------------- +# DataFrame bulk loading via COPY +# -------------------------------------------------------------------------------------------- + + +def _ts(values: t.Any) -> pd.DatetimeIndex: + return pd.to_datetime(values, format="ISO8601") + + +def _types(**columns: str) -> t.Dict[str, exp.DataType]: + return {name: exp.DataType.build(kind, dialect="postgres") for name, kind in columns.items()} + + +def _leftover_staging_tables( + engine_adapter: PostgresEngineAdapter, schema_like: str +) -> t.List[str]: + rows = engine_adapter.fetchall( + "SELECT table_schema || '.' || table_name FROM information_schema.tables " + f"WHERE table_name LIKE '\\_\\_temp\\_%' AND table_schema LIKE '%{schema_like}'" + ) + return [r[0] for r in rows] + + +@contextmanager +def _values_path(adapter: PostgresEngineAdapter) -> t.Iterator[None]: + """Forces DataFrames through the generic `VALUES (...)` implementation.""" + with patch.object( + PostgresEngineAdapter, "_df_to_source_queries", EngineAdapter._df_to_source_queries + ): + yield + + +def _assert_same_rows(adapter: PostgresEngineAdapter, a: exp.Table, b: exp.Table) -> None: + a_sql, b_sql = a.sql("postgres", identify=True), b.sql("postgres", identify=True) + (a_count,) = adapter.fetchone(f"SELECT COUNT(*) FROM {a_sql}") # type: ignore + (b_count,) = adapter.fetchone(f"SELECT COUNT(*) FROM {b_sql}") # type: ignore + assert a_count == b_count > 0 + for left, right in ((a_sql, b_sql), (b_sql, a_sql)): + extra = adapter.fetchall(f"SELECT * FROM {left} EXCEPT ALL SELECT * FROM {right}") + assert extra == [], f"rows only in {left}: {extra}" + + +_ALL_THE_STRINGS = [ + "", + None, + "\\N", + "NULL", + "null", + "\\", + "\\\\", + "\\x41", + 'a"b', + '"', + "a,b", + "a\nb", + "a\rb", + "a\r\nb", + "a\tb", + " padded ", + "h\u00e9llo w\u00f6rld", + "\u65e5\u672c\u8a9e", + "\U0001f600 emoji", + "it's", + "\\.", + "end\\.", + "\\.\nafter", + "a" * 5000, +] + +_TS_NAIVE = _ts( + [ + "2020-01-01 00:00:00", + "2020-06-15 12:34:56.123456", + "1969-12-31 23:59:59.999999", + "1900-01-01 00:00:00", + "1677-09-22 00:00:00", + "2262-04-11 23:47:16.854775", + "2024-02-29 00:00:00.000001", + ] +) + +# the extreme values can't be converted to every time zone +_TS_TZ = _TS_NAIVE[[0, 1, 2, 3, 6]] + +# (id, DataFrame, target types, source_columns, expect the COPY path to be used) +_PARITY_CASES: t.List[t.Any] = [ + pytest.param( + pd.DataFrame({"s": _ALL_THE_STRINGS, "i": range(len(_ALL_THE_STRINGS))}), + _types(s="text", i="int"), + None, + True, + id="strings-multi-column", + ), + pytest.param( + pd.DataFrame({"s": [x for x in _ALL_THE_STRINGS if x != "\\."]}), + _types(s="text"), + None, + True, + id="strings-single-column", + ), + pytest.param( + pd.DataFrame({"s": ["a", "\\.", "b"]}), + _types(s="text"), + None, + False, + id="strings-end-of-copy-marker-falls-back", + ), + pytest.param( + pd.DataFrame({"s": ["", "x", None, ""]}), + _types(s="varchar(10)"), + None, + True, + id="strings-single-column-empty-and-null", + ), + pytest.param( + pd.DataFrame({"i": [1, None, 3, None], "j": [5, 6, 7, 8]}), + _types(i="int", j="smallint"), + None, + True, + id="int-with-nulls-is-float-in-pandas", + ), + pytest.param( + pd.DataFrame( + { + "i": [2**63 - 1, -(2**63), 9007199254740993, 0, -1], + "j": [1, 2, 3, 4, 5], + } + ), + _types(i="bigint", j="int"), + None, + True, + id="int64-bounds-and-2^53+1", + ), + pytest.param( + pd.DataFrame( + {"i": np.array([1, -2, 3], dtype="int8"), "j": np.array([4, 5, 6], dtype="uint32")} + ), + _types(i="smallint", j="bigint"), + None, + True, + id="small-int-dtypes", + ), + pytest.param( + pd.DataFrame({"f": [1.5, 2.5, 0.5, -0.5, -1.5, -2.5, 3.4999999999, 1e10]}), + _types(f="bigint"), + None, + True, + id="float-into-bigint-rounds-like-values", + ), + pytest.param( + pd.DataFrame( + { + "f": [ + 0.1, + 1e-300, + 1e300, + 0.30000000000000004, + -0.0, + 5e-324, + 1.7976931348623157e308, + 123456789.123456789, + np.nan, + ] + } + ), + _types(f="double precision"), + None, + True, + id="float64-into-double", + ), + pytest.param( + pd.DataFrame( + {"f": [0.1, 0.30000000000000004, 16777217.0, 1e-20, 123456789.123456789, -2.5, np.nan]} + ), + _types(f="real"), + None, + True, + id="float64-into-real", + ), + pytest.param( + pd.DataFrame( + {"f": [0.1, 0.30000000000000004, 1e-12, 123456789.123456789, -2.5, 1e10, np.nan]} + ), + _types(f="numeric(38,20)"), + None, + True, + id="float64-into-numeric-high-scale", + ), + pytest.param( + pd.DataFrame({"b": [True, False, True], "c": [True, None, False]}), + _types(b="boolean", c="boolean"), + None, + True, + id="bool-and-bool-with-nulls", + ), + pytest.param( + pd.DataFrame({"ts": _TS_NAIVE, "tstz": _TS_NAIVE}), + _types(ts="timestamp", tstz="timestamptz"), + None, + True, + id="naive-timestamps", + ), + pytest.param( + pd.DataFrame( + { + "utc": _TS_TZ.tz_localize("UTC"), + "east": _TS_TZ.tz_localize("UTC").tz_convert("US/Eastern"), + "kolkata": _TS_TZ.tz_localize("UTC").tz_convert("Asia/Kolkata"), + } + ), + _types(utc="timestamptz", east="timestamp", kolkata="timestamptz"), + None, + True, + id="tz-aware-timestamps", + ), + pytest.param( + pd.DataFrame({"ts": _ts(["2020-01-01", "2020-01-02", "2020-01-03"])}), + _types(ts="timestamp"), + None, + True, + id="midnight-only-timestamps", + ), + pytest.param( + pd.DataFrame({"d": ["2020-01-01", "1969-12-31", "2024-02-29", None]}), + _types(d="date"), + None, + True, + id="date-from-strings", + ), + pytest.param( + pd.DataFrame({"n": ["1.5", "-0.000000001", "12345678901234567890.123456789", "0", None]}), + _types(n="numeric(38,9)"), + None, + True, + id="numeric-from-strings", + ), + pytest.param( + pd.DataFrame({"a": [1, 2, 3], "b": ["x", "y", "z"], "ignored": [0, 0, 0]}), + _types(a="int", b="text", c="int"), + ["a", "b"], + True, + id="source-columns-subset-with-missing-target-column", + ), + pytest.param( + pd.DataFrame( + { + "Order": [1, 2], + "ID": ["a", "b"], + "select": [0.5, 1.5], + "with space": [True, False], + 'we"ird': ["x", "y"], + "\u00fcber": [1, 2], + } + ), + _types( + **{ + "Order": "int", + "ID": "text", + "select": "double precision", + "with space": "boolean", + 'we"ird': "text", + "\u00fcber": "int", + } + ), + None, + True, + id="reserved-mixed-case-and-unicode-column-names", + ), + pytest.param( + pd.DataFrame( + { + "f": pd.Series([1.5, None, 0.1, 2.5], dtype=object), + "i": pd.Series([1, None, 3, 2**70], dtype=object), + "m": pd.Series([1, 2.5, None, 3], dtype=object), + "b": pd.Series([True, None, False, True], dtype=object), + } + ), + _types(f="double precision", i="numeric(30,0)", m="numeric(10,2)", b="boolean"), + None, + True, + id="object-columns-with-python-numbers-and-none", + ), + pytest.param( + pd.DataFrame({"s": [None, None], "i": [None, None]}), + _types(s="text", i="int"), + None, + True, + id="all-null-columns", + ), +] + + +@pytest.mark.parametrize("df, types, source_columns, expect_copy", _PARITY_CASES) +@pytest.mark.parametrize("chunk_size", [10000, 3]) +def test_df_copy_matches_values_path( + engine_adapter: PostgresEngineAdapter, + ctx: TestContext, + mocker: MockerFixture, + df: pd.DataFrame, + types: t.Dict[str, exp.DataType], + source_columns: t.Optional[t.List[str]], + expect_copy: bool, + chunk_size: int, +): + """Differential test: COPY and the generic VALUES implementation must produce identical rows.""" + copy_table = ctx.table("parity_copy") + values_table = ctx.table("parity_values") + for table in (copy_table, values_table): + engine_adapter.create_table(table, types) + + engine_adapter.DEFAULT_BATCH_SIZE = chunk_size + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + engine_adapter.insert_append( + copy_table, df, target_columns_to_types=types, source_columns=source_columns + ) + assert (spy.call_count > 0) is expect_copy + + with _values_path(engine_adapter): + engine_adapter.insert_append( + values_table, df, target_columns_to_types=types, source_columns=source_columns + ) + + _assert_same_rows(engine_adapter, copy_table, values_table) + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_exact_values(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + """Guards against the parity test passing because both paths are equally wrong.""" + table = ctx.table("exact") + types = _types( + id="int", + s="text", + i="int", + f="double precision", + b="boolean", + ts="timestamptz", + n="numeric(38,9)", + ) + engine_adapter.create_table(table, types) + df = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "s": ["", None, "\\N", 'a"b,c\nd'], + "i": [1, None, 3, 4], + "f": [0.1, np.nan, 1e-300, 2.5], + "b": [True, None, False, True], + "ts": _ts(["2020-01-01 00:00:00.123456", None, "1960-01-01", "2020-01-02"]).tz_localize( + "UTC" + ), + "n": ["1.5", None, "2", "-0.000000001"], + } + ) + engine_adapter.insert_append(table, df, target_columns_to_types=types) + + rows = engine_adapter.fetchall( + f"SELECT id, s, i, f, b, to_char(ts AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), n::text " + f"FROM {table.sql('postgres', identify=True)} ORDER BY id" + ) + assert rows == [ + (1, "", 1, 0.1, True, "2020-01-01 00:00:00.123456", "1.500000000"), + (2, None, None, None, None, None, None), + (3, "\\N", 3, 1e-300, False, "1960-01-01 00:00:00.000000", "2.000000000"), + (4, 'a"b,c\nd', 4, 2.5, True, "2020-01-02 00:00:00.000000", "-0.000000001"), + ] + + +def test_df_copy_nullable_pandas_dtypes(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + """The generic implementation can't load pd.NA at all, COPY handles the nullable dtypes.""" + table = ctx.table("nullable") + types = _types(i="int", j="bigint", f="double precision", b="boolean", s="text") + engine_adapter.create_table(table, types) + df = pd.DataFrame( + { + "i": pd.array([1, None, 3], dtype="Int64"), + "j": pd.array([2**62, -(2**62), None], dtype="Int64"), + "f": pd.array([0.1, None, 2.5], dtype="Float64"), + "b": pd.array([True, None, False], dtype="boolean"), + "s": pd.array(["x", None, ""], dtype="string"), + } + ) + engine_adapter.insert_append(table, df, target_columns_to_types=types) + rows = engine_adapter.fetchall( + f"SELECT * FROM {table.sql('postgres', identify=True)} ORDER BY i NULLS LAST" + ) + assert rows == [ + (1, 2**62, 0.1, True, "x"), + (3, None, 2.5, False, ""), + (None, -(2**62), None, None, None), + ] + + +def test_df_copy_source_columns_in_a_different_order( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + table = ctx.table("reordered") + types = _types(a="int", b="text", c="int") + engine_adapter.create_table(table, types) + df = pd.DataFrame({"b": ["x", "y"], "a": [1, 2], "ignored": [0, 0]}) + engine_adapter.insert_append( + table, df, target_columns_to_types=types, source_columns=["b", "a"] + ) + assert engine_adapter.fetchall( + f"SELECT a, b, c FROM {table.sql('postgres', identify=True)} ORDER BY a" + ) == [(1, "x", None), (2, "y", None)] + + +def test_df_copy_infinity(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + if engine_adapter.server_version < (14, 0): + pytest.skip("NUMERIC infinity requires PostgreSQL 14+") + table = ctx.table("inf") + types = _types(f="double precision") + engine_adapter.create_table(table, types) + df = pd.DataFrame({"f": [float("inf"), float("-inf"), 1.0]}) + engine_adapter.insert_append(table, df, target_columns_to_types=types) + rows = engine_adapter.fetchall( + f"SELECT f FROM {table.sql('postgres', identify=True)} ORDER BY f" + ) + assert rows == [(float("-inf"),), (1.0,), (float("inf"),)] + + +def test_df_copy_null_timestamps(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + table = ctx.table("nat") + types = _types(ts="timestamp", tstz="timestamptz") + engine_adapter.create_table(table, types) + df = pd.DataFrame( + { + "ts": _ts(["2020-01-01", None]), + "tstz": _ts(["2020-01-01", None]).tz_localize("UTC"), + } + ) + engine_adapter.insert_append(table, df, target_columns_to_types=types) + rows = engine_adapter.fetchall( + f"SELECT ts, tstz FROM {table.sql('postgres', identify=True)} ORDER BY ts NULLS LAST" + ) + assert rows[1] == (None, None) + assert rows[0][0] is not None and rows[0][1] is not None + + +def test_df_copy_in_non_utf8_client_encoding( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + connection = engine_adapter._connection_pool.get() + connection.set_client_encoding("LATIN1") + try: + table = ctx.table("latin1") + types = _types(s="text") + engine_adapter.create_table(table, types) + values = ["h\u00e9llo", "caf\u00e9 \u00fcber", "plain"] + engine_adapter.insert_append( + table, pd.DataFrame({"s": values}), target_columns_to_types=types + ) + rows = engine_adapter.fetchall( + f"SELECT s FROM {table.sql('postgres', identify=True)} ORDER BY s" + ) + assert sorted(r[0] for r in rows) == sorted(values) + finally: + connection.set_client_encoding("UTF8") + + +def test_df_copy_cleans_up_staging_table_on_failed_insert( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, mocker: MockerFixture +): + table = ctx.table("not_null_target") + engine_adapter.execute( + f"CREATE TABLE {table.sql('postgres', identify=True)} (a INT NOT NULL, b TEXT)" + ) + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + df = pd.DataFrame({"a": [1, None], "b": ["x", "y"]}) + + with pytest.raises(Exception, match='null value in column "a"'): + engine_adapter.insert_append(table, df, target_columns_to_types=_types(a="int", b="text")) + + assert spy.call_count == 1 + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + assert engine_adapter.fetchall(f"SELECT * FROM {table.sql('postgres', identify=True)}") == [] + # The connection must still be usable + assert engine_adapter.fetchone("select 1") == (1,) + + +def test_df_copy_cleans_up_staging_table_outside_transaction( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + table = ctx.table("outside_txn") + types = _types(a="int") + engine_adapter.create_table(table, types) + source_queries = engine_adapter._df_to_source_queries( + pd.DataFrame({"a": [1, 2]}), types, 10000, table + ) + with source_queries[0] as query: + assert ( + len( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) + ) + == 1 + ) + assert engine_adapter.fetchall(query) == [(1,), (2,)] + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_long_physical_table_name(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + name = ( + "sqlmesh_physical_" + "x" * 30 + "__" + "9" * 12 + ) # 61 characters, a typical physical name + table = ctx.table(name) + assert len(table.name) > 47 + types = _types(a="int") + engine_adapter.create_table(table, types) + engine_adapter.insert_append( + table, pd.DataFrame({"a": [1, 2, 3]}), target_columns_to_types=types + ) + assert engine_adapter.fetchall( + f"SELECT a FROM {table.sql('postgres', identify=True)} ORDER BY a" + ) == [(1,), (2,), (3,)] + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_concurrent_threads( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, config: Config +): + """Every thread has its own connection (and transaction); staging names must not collide.""" + connection_config = config.gateways[ctx.gateway].connection + assert connection_config + # `concurrent_tasks > 1` gives every thread its own connection, like during a plan + adapter = t.cast( + PostgresEngineAdapter, connection_config.create_engine_adapter(concurrent_tasks=6) + ) + types = _types(a="int", b="text") + tables = [ctx.table(f"concurrent_{i}") for i in range(6)] + for table in tables: + engine_adapter.create_table(table, types) + + adapter.DEFAULT_BATCH_SIZE = 300 + + def _load(i: int) -> None: + df = pd.DataFrame({"a": range(i * 1000, i * 1000 + 1000), "b": [f"t{i}"] * 1000}) + adapter.insert_append(tables[i], df, target_columns_to_types=types) + + try: + with ThreadPoolExecutor(max_workers=6) as pool: + list(pool.map(_load, range(6))) + finally: + adapter.close() + + for i, table in enumerate(tables): + rows = engine_adapter.fetchall( + f"SELECT MIN(a), MAX(a), COUNT(*), COUNT(DISTINCT b), MIN(b) FROM {table.sql('postgres', identify=True)}" + ) + assert rows == [(i * 1000, i * 1000 + 999, 1000, 1, f"t{i}")] + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_without_create_privilege_fails_like_other_ddl( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, config: Config +): + schema = exp.to_table(ctx.schema("no_create")).name + engine_adapter.create_schema(schema) + table = exp.to_table(f"{schema}.target") + types = _types(a="int") + engine_adapter.create_table(table, types) + with create_users(engine_adapter, "reader") as roles: + user = roles["reader"]["username"] + engine_adapter.execute(f'GRANT USAGE ON SCHEMA "{schema}" TO "{user}"') + engine_adapter.execute(f'GRANT INSERT, SELECT ON "{schema}"."target" TO "{user}"') + reader = create_engine_adapter_for_role(roles["reader"], ctx, config) + try: + with pytest.raises(Exception, match="permission denied for schema"): + reader.insert_append(table, pd.DataFrame({"a": [1]}), target_columns_to_types=types) + assert reader.fetchone("select 1") == (1,) + finally: + reader.close() + + +def test_df_copy_generated_and_identity_columns( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + """Only the listed columns are copied, so identity / generated columns keep working.""" + table = ctx.table("generated") + sql_table = table.sql("postgres", identify=True) + engine_adapter.execute( + f"CREATE TABLE {sql_table} (" + "id INT GENERATED ALWAYS AS IDENTITY, a INT, doubled INT GENERATED ALWAYS AS (a * 2) STORED, b TEXT)" + ) + engine_adapter.insert_append( + table, + pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}), + target_columns_to_types=_types(a="int", b="text"), + ) + assert engine_adapter.fetchall(f"SELECT id, a, doubled, b FROM {sql_table} ORDER BY a") == [ + (1, 1, 2, "x"), + (2, 2, 4, "y"), + ] + + +def test_df_copy_engine_adapter_operations( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, mocker: MockerFixture +): + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + types = _types(id="int", name="text", ds="text") + + def _read(table: exp.Table) -> t.List[t.Tuple]: + return engine_adapter.fetchall( + f"SELECT * FROM {table.sql('postgres', identify=True)} ORDER BY id" + ) + + # replace_query: creates the table, then replaces its content + replaced = ctx.table("replaced") + engine_adapter.replace_query( + replaced, + pd.DataFrame({"id": [1, 2], "name": ["a", "b"], "ds": ["2020-01-01"] * 2}), + target_columns_to_types=types, + ) + assert _read(replaced) == [(1, "a", "2020-01-01"), (2, "b", "2020-01-01")] + engine_adapter.replace_query( + replaced, + pd.DataFrame({"id": [3], "name": ["c"], "ds": ["2020-01-02"]}), + target_columns_to_types=types, + ) + assert _read(replaced) == [(3, "c", "2020-01-02")] + + # insert_append + engine_adapter.insert_append( + replaced, + pd.DataFrame({"id": [4], "name": [None], "ds": ["2020-01-03"]}), + target_columns_to_types=types, + ) + assert _read(replaced) == [(3, "c", "2020-01-02"), (4, None, "2020-01-03")] + + # ctas from a DataFrame + ctas = ctx.table("ctas") + engine_adapter.ctas( + ctas, + pd.DataFrame({"id": [7], "name": ["z"], "ds": ["2020-01-07"]}), + target_columns_to_types=types, + ) + assert _read(ctas) == [(7, "z", "2020-01-07")] + + # insert_overwrite_by_time_partition + partitioned = ctx.table("partitioned") + engine_adapter.create_table(partitioned, types) + engine_adapter.insert_append( + partitioned, + pd.DataFrame( + {"id": [1, 2, 3], "name": list("abc"), "ds": ["2020-01-01", "2020-01-02", "2020-01-03"]} + ), + target_columns_to_types=types, + ) + engine_adapter.insert_overwrite_by_time_partition( + partitioned, + pd.DataFrame({"id": [20, 30], "name": ["B", "C"], "ds": ["2020-01-02", "2020-01-03"]}), + start="2020-01-02", + end="2020-01-03", + time_formatter=lambda x, _: exp.Literal.string(to_ds(x)), + time_column="ds", + target_columns_to_types=types, + ) + assert _read(partitioned) == [ + (1, "a", "2020-01-01"), + (20, "B", "2020-01-02"), + (30, "C", "2020-01-03"), + ] + + # merge: native (PG >= 15) and the logical implementation used for PG < 15 + for version in (engine_adapter.server_version, (14, 0)): + merged = ctx.table(f"merged_{version[0]}") + engine_adapter.create_table(merged, types) + engine_adapter.insert_append( + merged, + pd.DataFrame({"id": [1, 2], "name": ["a", "b"], "ds": ["x", "x"]}), + target_columns_to_types=types, + ) + engine_adapter.server_version = version # type: ignore + engine_adapter.merge( + merged, + pd.DataFrame({"id": [2, 3], "name": ["B", "C"], "ds": ["y", "y"]}), + target_columns_to_types=types, + unique_key=[exp.to_column("id", quoted=True)], + ) + assert _read(merged) == [(1, "a", "x"), (2, "B", "y"), (3, "C", "y")] + + assert spy.call_count >= 8 + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_python_and_seed_models(ctx: TestContext, tmp_path: Path, mocker: MockerFixture): + engine_adapter = t.cast(PostgresEngineAdapter, ctx.engine_adapter) + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + schema = exp.to_table(ctx.schema(TEST_SCHEMA)).this + models_dir = tmp_path / "models" + seeds_dir = tmp_path / "seeds" + models_dir.mkdir() + seeds_dir.mkdir() + + (seeds_dir / "seed.csv").write_text('id,name,score\n1,alice,1.5\n2,"b,ob",\n3,\\N,3.25\n') + (models_dir / "seed_model.sql").write_text(f""" + MODEL ( + name {schema}.seed_model, + kind SEED (path '../seeds/seed.csv'), + columns (id int, name text, score double) + ); + """) + (models_dir / "py_model.py").write_text(f""" +import typing as t +from datetime import datetime + +import pandas as pd +from sqlmesh import ExecutionContext, model +from sqlmesh.core.model.kind import ModelKindName + + +@model( + "{schema}.py_model", + kind=dict(name=ModelKindName.INCREMENTAL_BY_TIME_RANGE, time_column="ds", batch_size=1), + columns={{"id": "int", "name": "text", "value": "double", "ds": "text"}}, + start="2020-01-01", + cron="@daily", +) +def execute(context: ExecutionContext, start: datetime, end: datetime, execution_time: datetime, **kwargs: t.Any) -> pd.DataFrame: + ds = start.strftime("%Y-%m-%d") + return pd.DataFrame( + {{ + "id": [1, 2, 3], + "name": [f"n{{ds}}", None, 'q"uote,'], + "value": [0.1, float("nan"), 2.5], + "ds": [ds] * 3, + }} + ) +""") + + sqlmesh = ctx.create_context(path=tmp_path) + with time_machine.travel("2020-01-04 00:00:00"): + sqlmesh.plan(auto_apply=True, no_prompts=True) + + py_rows = engine_adapter.fetchall( + f'SELECT ds, COUNT(*), COUNT(name), COUNT(value), MIN(id) FROM "{schema}"."py_model" GROUP BY ds ORDER BY ds' + ) + assert py_rows == [(f"2020-01-0{d}", 3, 2, 2, 1) for d in (1, 2, 3)] + assert engine_adapter.fetchall( + f'SELECT id, name, score FROM "{schema}"."seed_model" ORDER BY id' + ) == [ + (1, "alice", 1.5), + (2, "b,ob", None), + (3, "\\N", 3.25), + ] + assert spy.call_count >= 4 + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +@pytest.mark.parametrize("operation", ["replace_query", "insert_append"]) +def test_df_copy_failure_leaves_no_staging_table( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, operation: str +): + """A failing COPY (here: a NUL character) must not leak the staging table, outside a transaction too.""" + table = ctx.table("copy_failure") + types = _types(x="text", y="int") + engine_adapter.create_table(table, types) + df = pd.DataFrame({"x": ["a\x00b"], "y": [1]}) + + with pytest.raises(Exception, match="0x00|null character|NUL"): + getattr(engine_adapter, operation)(table, df, target_columns_to_types=types) + + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + assert engine_adapter.fetchone("select 1") == (1,) diff --git a/tests/core/engine_adapter/test_base.py b/tests/core/engine_adapter/test_base.py index 1971ba3bbc..4cff662313 100644 --- a/tests/core/engine_adapter/test_base.py +++ b/tests/core/engine_adapter/test_base.py @@ -3489,6 +3489,40 @@ def test_get_temp_table(mocker: MockerFixture, make_mocked_engine_adapter: t.Cal assert value.sql() == '"catalog"."db"."__temp_test_table_abcdefgh"' +@pytest.mark.parametrize( + "max_length, table_name, expected", + [ + # The limit is only enforced when the engine defines one + ( + None, + "a_very_long_table_name_" + "x" * 40, + "__temp_a_very_long_table_name_" + "x" * 40 + "_abcdefgh", + ), + # Exactly at the limit: unchanged + (30, "t" * 14, "__temp_" + "t" * 14 + "_abcdefgh"), + # Over the limit: prefix and random suffix are kept, the middle of the name is cut + (30, "aaaaaaaa_bbbbbbbb", "__temp_aaaaaaa" + "bbbbbbb" + "_abcdefgh"), + # The limit is so small that nothing of the name fits + (10, "name", "__temp__abcdefgh"), + ], +) +def test_get_temp_table_respects_max_identifier_length( + mocker: MockerFixture, + make_mocked_engine_adapter: t.Callable, + max_length: t.Optional[int], + table_name: str, + expected: str, +): + adapter = make_mocked_engine_adapter(EngineAdapter) + adapter.MAX_IDENTIFIER_LENGTH = max_length + mocker.patch("sqlmesh.core.engine_adapter.base.random_id", return_value="abcdefgh") + + value = adapter._get_temp_table(f"catalog.db.{table_name}") + + assert value.name == expected + assert (value.catalog, value.db) == ("catalog", "db") + + def test_get_data_objects_batching(mocker: MockerFixture, make_mocked_engine_adapter: t.Callable): adapter = make_mocked_engine_adapter(EngineAdapter) adapter._get_data_objects = mocker.Mock(return_value=[]) diff --git a/tests/core/engine_adapter/test_postgres.py b/tests/core/engine_adapter/test_postgres.py index ebcdd03f55..568d7e133c 100644 --- a/tests/core/engine_adapter/test_postgres.py +++ b/tests/core/engine_adapter/test_postgres.py @@ -1,5 +1,10 @@ +import re import typing as t +from datetime import date +from decimal import Decimal +import numpy as np # noqa: TID253 +import pandas as pd # noqa: TID253 import pytest from pytest_mock import MockFixture from pytest_mock.plugin import MockerFixture @@ -282,3 +287,290 @@ def test_sync_grants_config_with_default_schema( "WHERE table_schema = 'public' AND table_name = 'test_table' " "AND grantor = current_role AND grantee <> current_role" ) + + +# -------------------------------------------------------------------------------------------- +# DataFrame bulk loading via COPY +# -------------------------------------------------------------------------------------------- + +_NULL_MARKER_RE = re.compile(r"NULL '([^']+)'") + + +def _capture_copy_calls(adapter: PostgresEngineAdapter) -> t.List[t.Tuple[str, str]]: + """Records (sql, payload) for every `copy_expert` call; the payload is read at call time.""" + calls: t.List[t.Tuple[str, str]] = [] + + def _copy_expert(sql: str, file: t.Any) -> None: + calls.append((sql, file.read())) + + adapter.cursor.copy_expert.side_effect = _copy_expert + return calls + + +@pytest.fixture +def copy_adapter( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture, make_temp_table_name: t.Callable +) -> PostgresEngineAdapter: + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + mocker.patch.object(adapter, "table_exists", return_value=False) + mocker.patch( + "sqlmesh.core.engine_adapter.EngineAdapter._get_temp_table", + return_value=make_temp_table_name("tbl", "abcdefgh"), + ) + return adapter + + +def test_df_copy_chunks_sql_and_payload(copy_adapter: PostgresEngineAdapter): + calls = _capture_copy_calls(copy_adapter) + df = pd.DataFrame( + { + "a": [1, 2, 3], + "b": ["x", None, 'q"uote, comma'], + "c": [1.5, np.nan, 0.1], + "d": [True, False, True], + "e": pd.to_datetime( + ["2020-01-01 00:00:00.123456", None, "1960-01-01 12:00:00"], # type: ignore + format="ISO8601", + ), + } + ) + types = { + "a": exp.DataType.build("int"), + "b": exp.DataType.build("text"), + "c": exp.DataType.build("double"), + "d": exp.DataType.build("boolean"), + "e": exp.DataType.build("timestamp"), + } + + copy_adapter.DEFAULT_BATCH_SIZE = 2 + copy_adapter.insert_append("tbl", df, target_columns_to_types=types) + + # One COPY per chunk of `batch_size` rows + assert len(calls) == 2 + marker = _NULL_MARKER_RE.search(calls[0][0]).group(1) # type: ignore + assert len(marker) <= 32 # pandas truncates longer `na_rep` values in float columns + expected_sql = ( + 'COPY "__temp_tbl_abcdefgh" ("a", "b", "c", "d", "e") ' + f"FROM STDIN WITH (FORMAT csv, NULL '{marker}')" + ) + assert [sql for sql, _ in calls] == [expected_sql, expected_sql] + assert ( + calls[0][1] + == f"1,x,1.5,True,2020-01-01 00:00:00.123456\n2,{marker},{marker},False,{marker}\n" + ) + assert calls[1][1] == '3,"q""uote, comma",0.1,True,1960-01-01 12:00:00\n' + + assert to_sql_calls(copy_adapter) == [ + 'CREATE TABLE IF NOT EXISTS "__temp_tbl_abcdefgh" ' + '("a" BIGINT, "b" TEXT, "c" DECIMAL, "d" BOOLEAN, "e" TIMESTAMP)', + 'INSERT INTO "tbl" ("a", "b", "c", "d", "e") ' + 'SELECT CAST("a" AS INT) AS "a", CAST("b" AS TEXT) AS "b", CAST("c" AS DOUBLE PRECISION) AS "c", ' + 'CAST("d" AS BOOLEAN) AS "d", CAST("e" AS TIMESTAMP) AS "e" FROM "__temp_tbl_abcdefgh"', + 'DROP TABLE IF EXISTS "__temp_tbl_abcdefgh"', + ] + + +def test_df_copy_batch_size_zero_is_single_chunk(copy_adapter: PostgresEngineAdapter): + calls = _capture_copy_calls(copy_adapter) + df = pd.DataFrame({"a": [1, 2, 3]}) + copy_adapter.DEFAULT_BATCH_SIZE = 0 + copy_adapter.insert_append("tbl", df, target_columns_to_types={"a": exp.DataType.build("int")}) + assert len(calls) == 1 + assert calls[0][1] == "1\n2\n3\n" + + +def test_df_copy_reorders_columns_and_skips_missing_source_columns( + copy_adapter: PostgresEngineAdapter, +): + calls = _capture_copy_calls(copy_adapter) + df = pd.DataFrame({"b": ["x", "y"], "a": [1, 2], "unused": [9, 9]}) + copy_adapter.insert_append( + "tbl", + df, + target_columns_to_types={ + "a": exp.DataType.build("int"), + "b": exp.DataType.build("text"), + "c": exp.DataType.build("int"), + }, + source_columns=["b", "a"], + ) + + # Only the source columns are copied, in the source order + assert len(calls) == 1 + assert calls[0][0].startswith('COPY "__temp_tbl_abcdefgh" ("b", "a") FROM STDIN') + assert calls[0][1] == "x,1\ny,2\n" + assert to_sql_calls(copy_adapter)[1] == ( + 'INSERT INTO "tbl" ("a", "b", "c") SELECT CAST("a" AS INT) AS "a", CAST("b" AS TEXT) AS "b", ' + 'CAST(NULL AS INT) AS "c" FROM "__temp_tbl_abcdefgh"' + ) + + +def test_df_copy_staging_table_is_created_once( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + calls = _capture_copy_calls(copy_adapter) + df = pd.DataFrame({"a": [1, 2]}) + source_queries = copy_adapter._df_to_source_queries( + df, {"a": exp.DataType.build("int")}, 10000, "tbl" + ) + assert len(source_queries) == 1 + + # Once the staging table exists, calling the factory again must not reload the data + with source_queries[0]: + pass + copy_adapter.table_exists.return_value = True # type: ignore + with source_queries[0]: + pass + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "df, remove_copy_expert", + [ + (pd.DataFrame({"a": [1, 2]}), True), # driver without copy_expert (e.g. pg8000) + (pd.DataFrame({"a": [{"k": 1}, {"k": 2}]}), False), # dict values + (pd.DataFrame({"a": [[1], [2]]}), False), # list values + (pd.DataFrame({"a": ["x", 1]}), False), # mixed str / int + (pd.DataFrame({"a": [b"x", b"y"]}), False), # bytes + (pd.DataFrame({"a": [Decimal("1.5"), Decimal("2")]}), False), # Decimal + (pd.DataFrame({"a": [date(2020, 1, 1), date(2020, 1, 2)]}), False), # date objects + (pd.DataFrame({"a": pd.Categorical(["x", "y"])}), False), # unsupported dtype + (pd.DataFrame({"a": pd.to_timedelta([1, 2], unit="s")}), False), # unsupported dtype + (pd.DataFrame({"a": np.array([1, 2], dtype="uint64")}), False), # may exceed BIGINT + ( + pd.DataFrame({"a": np.array([1, 2], dtype="float32")}), + False, + ), # repr differs from float64 + (pd.DataFrame({"a": ["x", "\\."]}), False), # `\.` can end COPY data in CSV format + ], +) +def test_df_copy_falls_back_to_values( + copy_adapter: PostgresEngineAdapter, df: pd.DataFrame, remove_copy_expert: bool +): + if remove_copy_expert: + del copy_adapter.cursor.copy_expert + + copy_adapter.insert_append("tbl", df, target_columns_to_types={"a": exp.DataType.build("text")}) + + if not remove_copy_expert: + copy_adapter.cursor.copy_expert.assert_not_called() + sql_calls = to_sql_calls(copy_adapter) + assert len(sql_calls) == 1 + assert "VALUES" in sql_calls[0] + assert "__temp_" not in sql_calls[0] + + +def test_df_copy_falls_back_on_null_marker_collision( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + mocker.patch("sqlmesh.core.engine_adapter.postgres.random_id", return_value="collide") + marker = "__sqlmesh_null_collide__" + df = pd.DataFrame({"a": ["x", marker]}) + + copy_adapter.insert_append("tbl", df, target_columns_to_types={"a": exp.DataType.build("text")}) + + copy_adapter.cursor.copy_expert.assert_not_called() + sql_calls = to_sql_calls(copy_adapter) + assert len(sql_calls) == 1 + assert "VALUES" in sql_calls[0] + + +def test_df_copy_not_used_for_empty_dataframe(copy_adapter: PostgresEngineAdapter): + # The adapter itself must not create a staging table for an empty DataFrame + df = pd.DataFrame({"a": pd.Series([], dtype="int64")}) + source_queries = copy_adapter._df_to_source_queries( + df, {"a": exp.DataType.build("int")}, 10000, "tbl" + ) + assert source_queries == [] + copy_adapter.cursor.copy_expert.assert_not_called() + assert to_sql_calls(copy_adapter) == [] + + +def test_df_copy_driver_call_is_isolated_in_copy_df( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + """Only `_copy_df` may touch the driver's COPY API (keeps a psycopg3 migration small).""" + copy_df = mocker.patch.object(copy_adapter, "_copy_df") + df = pd.DataFrame({"a": [1, 2]}) + copy_adapter.insert_append("tbl", df, target_columns_to_types={"a": exp.DataType.build("int")}) + copy_df.assert_called_once() + copy_adapter.cursor.copy_expert.assert_not_called() + + +def test_temp_table_name_respects_max_identifier_length( + make_mocked_engine_adapter: t.Callable, +): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + table = f"sqlmesh__schema.{'x' * 30}__{'y' * 28}" # 60 character table name + assert len(exp.to_table(table).name) == 60 + + first = adapter._get_temp_table(table) + second = adapter._get_temp_table(table) + + assert first.db == "sqlmesh__schema" + for temp in (first, second): + assert len(temp.name) <= 63 + assert temp.name.startswith("__temp_xxxxxx") # head of the name is kept + assert temp.name.rsplit("_", 1)[0].endswith("yyyyyy") # tail of the name is kept + assert first.name != second.name # random suffix keeps names unique + # Identifier-length validation on DDL must accept the staging name + adapter._check_identifier_length(exp.Create(this=first, kind="TABLE")) + + +def test_temp_table_name_unchanged_when_short(make_mocked_engine_adapter: t.Callable): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + name = "z" * 47 # 7 prefix + 47 + 1 + 8 suffix == 63 + temp = adapter._get_temp_table(f"s.{name}") + assert temp.name.startswith(f"__temp_{name}_") + assert len(temp.name) == 63 + + +class _FakeDbError(Exception): + def __init__(self, pgcode: t.Optional[str]) -> None: + super().__init__("boom") + self.pgcode = pgcode + + +@pytest.mark.parametrize( + "pgcode, in_transaction, raises", + [ + ("25P02", True, False), # aborted transaction: its rollback removes the staging table + ("25P02", False, True), + ("42501", True, True), + (None, True, True), + ], +) +def test_drop_copy_staging_table_error_handling( + copy_adapter: PostgresEngineAdapter, + mocker: MockerFixture, + pgcode: t.Optional[str], + in_transaction: bool, + raises: bool, +): + mocker.patch.object(copy_adapter, "drop_table", side_effect=_FakeDbError(pgcode)) + mocker.patch.object( + type(copy_adapter._connection_pool), + "is_transaction_active", + new_callable=mocker.PropertyMock, + return_value=in_transaction, + ) + table = exp.to_table("tbl") + if raises: + with pytest.raises(_FakeDbError): + copy_adapter._drop_copy_staging_table(table) + else: + copy_adapter._drop_copy_staging_table(table) + + +def test_df_copy_failure_drops_staging_table_and_reraises( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + mocker.patch.object(copy_adapter, "_copy_df", side_effect=ValueError("copy failed")) + df = pd.DataFrame({"a": [1, 2]}) + source_queries = copy_adapter._df_to_source_queries( + df, {"a": exp.DataType.build("int")}, 10000, "tbl" + ) + # `replace_query` calls the factory outside of `with source_query`, so the factory must clean up + with pytest.raises(ValueError, match="copy failed"): + source_queries[0].query_factory() + assert to_sql_calls(copy_adapter)[-1] == 'DROP TABLE IF EXISTS "__temp_tbl_abcdefgh"' diff --git a/tests/core/engine_adapter/test_risingwave.py b/tests/core/engine_adapter/test_risingwave.py index ed3cd77a3f..2711b5af16 100644 --- a/tests/core/engine_adapter/test_risingwave.py +++ b/tests/core/engine_adapter/test_risingwave.py @@ -84,3 +84,18 @@ def test_drop_view(adapter: t.Callable): call('DROP VIEW IF EXISTS "db"."view"'), ] ) + + +def test_df_is_not_loaded_with_copy(adapter: t.Callable): + """RisingWave inherits from Postgres but must keep using the VALUES path.""" + import pandas as pd # noqa: TID253 + + risingwave = adapter + risingwave.insert_append( + "tbl", pd.DataFrame({"a": [1, 2]}), target_columns_to_types={"a": exp.DataType.build("int")} + ) + + risingwave.cursor.copy_expert.assert_not_called() + sql_calls = [call_[0][0] for call_ in risingwave.cursor.execute.call_args_list] + assert len(sql_calls) == 1 + assert "VALUES" in str(sql_calls[0]) and "__temp_" not in str(sql_calls[0]) From 889bcc21b3845e3abd2d1167e14b358d3026b42d Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Sun, 15 Nov 2026 08:26:21 +0000 Subject: [PATCH 2/3] fix(postgres): handle UTF-8 truncation, cast compatibility, and transaction safety in COPY staging Address CodeRabbit review feedback for PR #6217: - Enforce 63-byte identifier limit in UTF-8 bytes with middle-truncation along Unicode code point boundaries in PostgresEngineAdapter._get_temp_table and _check_identifier_length, preserving prefix and random suffix. - Keep RisingwaveEngineAdapter MAX_IDENTIFIER_LENGTH = None and exempt from length truncation. - Detect staging-to-target cast incompatibilities in _get_copy_staging_columns_to_types, falling back to VALUES when target type is UNKNOWN, when BIGINT targets BOOLEAN, or when NUMERIC targets BOOLEAN, BIT, or OID/REG* types. - Stage bounded object-dtype integer series within signed 64-bit int range as BIGINT when target is not BOOLEAN. - Enclose staging table creation and dataframe copying in an atomic transaction block within query_factory, ensuring rollback without leaking staging tables even when pre_ping=False. - Isolate test table names as merged_{idx}_{version[0]} and restore original server_version in PostgreSQL 14 integration test. - Document cast incompatibility fallback in docs/integrations/engines/postgres.md. Signed-off-by: Vamsi-klu --- docs/integrations/engines/postgres.md | 1 + sqlmesh/core/engine_adapter/postgres.py | 167 ++++++++++++++++-- .../integration/test_integration_postgres.py | 106 ++++++++--- tests/core/engine_adapter/test_postgres.py | 134 +++++++++++++- tests/core/engine_adapter/test_risingwave.py | 8 + 5 files changed, 379 insertions(+), 37 deletions(-) diff --git a/docs/integrations/engines/postgres.md b/docs/integrations/engines/postgres.md index be2d3c6b70..ae39b3523f 100644 --- a/docs/integrations/engines/postgres.md +++ b/docs/integrations/engines/postgres.md @@ -31,6 +31,7 @@ When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those No configuration is required and there is no option to disable this behavior. SQLMesh automatically falls back to `INSERT ... VALUES` when: - A column has a type or values that cannot be loaded with `COPY` identically. Supported columns are integers (up to 64 bits signed, up to 32 bits unsigned), `float64`, booleans, timestamps (with or without time zone), and strings. Other types, such as `float32`, 64-bit unsigned integers, `Decimal`, dates, bytes, dicts, lists, and categoricals, trigger the fallback. +- A column's staging type is incompatible with the target column's cast (for example, an integer or numeric DataFrame column targeting a `BOOLEAN`, `BIT`, or `OID`/`REG*` type). - The DataFrame has a single column and one of its values is `\.`. - The driver does not support `copy_expert` (for example, a driver other than `psycopg2`). diff --git a/sqlmesh/core/engine_adapter/postgres.py b/sqlmesh/core/engine_adapter/postgres.py index 82c9be4afb..9f5e6088b6 100644 --- a/sqlmesh/core/engine_adapter/postgres.py +++ b/sqlmesh/core/engine_adapter/postgres.py @@ -18,6 +18,7 @@ ) from sqlmesh.core.engine_adapter.shared import SourceQuery, set_catalog from sqlmesh.utils import random_id +from sqlmesh.utils.errors import SQLMeshError if t.TYPE_CHECKING: import pandas as pd @@ -155,7 +156,9 @@ def _df_to_source_queries( staging_columns_to_types = ( None if ordered_df.empty or not hasattr(self.cursor, "copy_expert") - else self._get_copy_staging_columns_to_types(ordered_df, null_marker) + else self._get_copy_staging_columns_to_types( + ordered_df, null_marker, target_columns_to_types + ) ) if staging_columns_to_types is None: return super()._df_to_source_queries( @@ -173,8 +176,9 @@ def query_factory() -> Query: # assumes that every call would have produced the same result. if not self.table_exists(staging_table): try: - self.create_table(staging_table, staging_columns_to_types) - self._copy_df(staging_table, ordered_df, null_marker, batch_size) + with self.transaction(): + self.create_table(staging_table, staging_columns_to_types) + self._copy_df(staging_table, ordered_df, null_marker, batch_size) except Exception: # Callers such as `replace_query` invoke the factory outside of `with source_query`, # so `cleanup_func` won't run and a committed staging table would leak. @@ -194,6 +198,67 @@ def query_factory() -> Query: ) ] + def _get_temp_table( + self, table: TableName, table_only: bool = False, quoted: bool = True + ) -> exp.Table: + if self.MAX_IDENTIFIER_LENGTH is None: + return super()._get_temp_table(table, table_only=table_only, quoted=quoted) + + table = t.cast(exp.Table, exp.to_table(table).copy()) + prefix, suffix = "__temp_", f"_{random_id(short=True)}" + name = table.name + prefix_bytes = len(prefix.encode("utf-8")) + suffix_bytes = len(suffix.encode("utf-8")) + max_bytes = self.MAX_IDENTIFIER_LENGTH + total_bytes = prefix_bytes + len(name.encode("utf-8")) + suffix_bytes + + if total_bytes > max_bytes: + budget = max(max_bytes - prefix_bytes - suffix_bytes, 0) + if budget == 0: + name = "" + else: + target_tail_bytes = budget // 2 + target_head_bytes = budget - target_tail_bytes + + head = [] + head_bytes = 0 + for ch in name: + ch_b = len(ch.encode("utf-8")) + if head_bytes + ch_b > target_head_bytes: + break + head.append(ch) + head_bytes += ch_b + + remaining_tail_budget = budget - head_bytes + tail = [] + tail_bytes = 0 + for ch in reversed(name[len(head) :]): + ch_b = len(ch.encode("utf-8")) + if tail_bytes + ch_b > remaining_tail_budget: + break + tail.append(ch) + tail_bytes += ch_b + + name = "".join(head) + "".join(reversed(tail)) + + table.set("this", exp.to_identifier(f"{prefix}{name}{suffix}", quoted=quoted)) + if table_only: + table.set("db", None) + table.set("catalog", None) + return table + + def _check_identifier_length(self, expression: exp.Expr) -> None: + if self.MAX_IDENTIFIER_LENGTH is None or not isinstance(expression, exp.DDL): + return + + for identifier in expression.find_all(exp.Identifier): + name = identifier.name + name_bytes = len(name.encode("utf-8")) + if name_bytes > self.MAX_IDENTIFIER_LENGTH: + raise SQLMeshError( + f"Identifier name '{name}' (length {name_bytes} bytes) exceeds {self.dialect.capitalize()}'s max identifier limit of {self.MAX_IDENTIFIER_LENGTH} bytes" + ) + def _drop_copy_staging_table(self, staging_table: exp.Table) -> None: try: self.drop_table(staging_table) @@ -207,19 +272,59 @@ def _drop_copy_staging_table(self, staging_table: exp.Table) -> None: logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql()) @staticmethod + def _is_oid_or_reg_type(data_type: exp.DataType) -> bool: + type_str = str(data_type.this).lower() + sql_str = data_type.sql("postgres").lower() + return ( + type_str + in ( + "oid", + "regclass", + "regtype", + "regproc", + "regprocedure", + "regoper", + "regoperator", + "regnamespace", + "regrole", + "regconfig", + "regdictionary", + "regcollation", + ) + or sql_str == "oid" + or sql_str.startswith("reg") + ) + + @classmethod def _get_copy_staging_columns_to_types( - df: pd.DataFrame, null_marker: str + cls, + df: pd.DataFrame, + null_marker: str, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, ) -> t.Optional[t.Dict[str, exp.DataType]]: """Returns the staging table column types for the DataFrame, or None if it can't be loaded with COPY. None is returned when a column has a dtype or values that wouldn't be read back by COPY exactly as the generic `VALUES` implementation would write them, e.g. dicts, bytes, Decimals, dates or - categoricals, or when a string would be misread by the CSV parser. + categoricals, or when a string would be misread by the CSV parser, or when the staging type is + incompatible with the target column type. """ import pandas as pd + if target_columns_to_types is None: + return None + staging_columns_to_types: t.Dict[str, exp.DataType] = {} for column, series in df.items(): + col_name = str(column) + target_type = target_columns_to_types.get(col_name) + if target_type is None: + return None + if not isinstance(target_type, exp.DataType): + target_type = exp.DataType.build(target_type, dialect=cls.DIALECT) + if target_type.is_type(exp.DataType.Type.UNKNOWN): + return None + dtype = series.dtype staging_type: t.Optional[str] if isinstance(dtype, pd.DatetimeTZDtype): @@ -228,21 +333,55 @@ def _get_copy_staging_columns_to_types( staging_type = "TIMESTAMP" elif str(dtype) in _COPY_DTYPE_TO_STAGING_TYPE: staging_type = _COPY_DTYPE_TO_STAGING_TYPE[str(dtype)] - elif dtype == object or isinstance(dtype, pd.StringDtype): - staging_type = _COPY_INFERRED_TO_STAGING_TYPE.get( - pd.api.types.infer_dtype(series, skipna=True) - ) - if staging_type == "TEXT": - if series.eq(null_marker).any(): + if staging_type == "BIGINT" and target_type.is_type(exp.DataType.Type.BOOLEAN): + return None + if staging_type == "NUMERIC": + if target_type.is_type( + exp.DataType.Type.BOOLEAN, exp.DataType.Type.BIT + ) or cls._is_oid_or_reg_type(target_type): return None - # On PostgreSQL <= 17 an unquoted `\.` line ends the data, which happens if it is the only column - if len(df.columns) == 1 and series.eq("\\.").any(): + elif dtype == object or isinstance(dtype, pd.StringDtype): + inferred = pd.api.types.infer_dtype(series, skipna=True) + if inferred == "integer": + if target_type.is_type(exp.DataType.Type.BOOLEAN): return None + non_nulls = series.dropna() + try: + fits_bigint = bool( + ( + (non_nulls >= -9223372036854775808) + & (non_nulls <= 9223372036854775807) + ).all() + ) + except Exception: + fits_bigint = False + + if fits_bigint: + staging_type = "BIGINT" + else: + staging_type = "NUMERIC" + if target_type.is_type(exp.DataType.Type.BIT) or cls._is_oid_or_reg_type( + target_type + ): + return None + else: + staging_type = _COPY_INFERRED_TO_STAGING_TYPE.get(inferred) + if staging_type == "TEXT": + if series.eq(null_marker).any(): + return None + # On PostgreSQL <= 17 an unquoted `\.` line ends the data, which happens if it is the only column + if len(df.columns) == 1 and series.eq("\\.").any(): + return None + elif staging_type == "NUMERIC": + if target_type.is_type( + exp.DataType.Type.BOOLEAN, exp.DataType.Type.BIT + ) or cls._is_oid_or_reg_type(target_type): + return None else: staging_type = None if staging_type is None: return None - staging_columns_to_types[str(column)] = exp.DataType.build( + staging_columns_to_types[col_name] = exp.DataType.build( staging_type, dialect=PostgresEngineAdapter.DIALECT ) return staging_columns_to_types diff --git a/tests/core/engine_adapter/integration/test_integration_postgres.py b/tests/core/engine_adapter/integration/test_integration_postgres.py index db3ed7a917..89a825be55 100644 --- a/tests/core/engine_adapter/integration/test_integration_postgres.py +++ b/tests/core/engine_adapter/integration/test_integration_postgres.py @@ -1910,22 +1910,26 @@ def _read(table: exp.Table) -> t.List[t.Tuple]: ] # merge: native (PG >= 15) and the logical implementation used for PG < 15 - for version in (engine_adapter.server_version, (14, 0)): - merged = ctx.table(f"merged_{version[0]}") - engine_adapter.create_table(merged, types) - engine_adapter.insert_append( - merged, - pd.DataFrame({"id": [1, 2], "name": ["a", "b"], "ds": ["x", "x"]}), - target_columns_to_types=types, - ) - engine_adapter.server_version = version # type: ignore - engine_adapter.merge( - merged, - pd.DataFrame({"id": [2, 3], "name": ["B", "C"], "ds": ["y", "y"]}), - target_columns_to_types=types, - unique_key=[exp.to_column("id", quoted=True)], - ) - assert _read(merged) == [(1, "a", "x"), (2, "B", "y"), (3, "C", "y")] + original_version = engine_adapter.server_version + try: + for idx, version in enumerate((original_version, (14, 0))): + merged = ctx.table(f"merged_{idx}_{version[0]}") + engine_adapter.create_table(merged, types) + engine_adapter.insert_append( + merged, + pd.DataFrame({"id": [1, 2], "name": ["a", "b"], "ds": ["x", "x"]}), + target_columns_to_types=types, + ) + engine_adapter.server_version = version # type: ignore + engine_adapter.merge( + merged, + pd.DataFrame({"id": [2, 3], "name": ["B", "C"], "ds": ["y", "y"]}), + target_columns_to_types=types, + unique_key=[exp.to_column("id", quoted=True)], + ) + assert _read(merged) == [(1, "a", "x"), (2, "B", "y"), (3, "C", "y")] + finally: + engine_adapter.server_version = original_version assert spy.call_count >= 8 assert ( @@ -2000,19 +2004,79 @@ def execute(context: ExecutionContext, start: datetime, end: datetime, execution @pytest.mark.parametrize("operation", ["replace_query", "insert_append"]) +@pytest.mark.parametrize("pre_ping", [True, False]) def test_df_copy_failure_leaves_no_staging_table( - engine_adapter: PostgresEngineAdapter, ctx: TestContext, operation: str + engine_adapter: PostgresEngineAdapter, ctx: TestContext, operation: str, pre_ping: bool ): """A failing COPY (here: a NUL character) must not leak the staging table, outside a transaction too.""" - table = ctx.table("copy_failure") + adapter = t.cast( + PostgresEngineAdapter, + engine_adapter if pre_ping else engine_adapter.with_settings(pre_ping=False), + ) + table = ctx.table(f"copy_failure_{operation}_{pre_ping}") types = _types(x="text", y="int") - engine_adapter.create_table(table, types) + adapter.create_table(table, types) df = pd.DataFrame({"x": ["a\x00b"], "y": [1]}) with pytest.raises(Exception, match="0x00|null character|NUL"): - getattr(engine_adapter, operation)(table, df, target_columns_to_types=types) + getattr(adapter, operation)(table, df, target_columns_to_types=types) + + assert _leftover_staging_tables(adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + assert adapter.fetchone("select 1") == (1,) + + +def test_df_copy_cjk_table_name_utf8(engine_adapter: PostgresEngineAdapter, ctx: TestContext): + """Temp staging table name for multi-byte CJK table must not exceed 63 bytes in PostgreSQL.""" + # Target table is 53 bytes (<= 63 bytes), but temp table (__temp_..._) would be 69 bytes without truncation + table = ctx.table(f"cjk_{'界' * 8}_{'世' * 8}") + types = _types(a="int", b="text") + engine_adapter.create_table(table, types) + df = pd.DataFrame({"a": [1, 2], "b": ["foo", "bar"]}) + engine_adapter.insert_append(table, df, target_columns_to_types=types) + assert engine_adapter.fetchall(f"SELECT * FROM {table.sql('postgres', identify=True)}") == [ + (1, "foo"), + (2, "bar"), + ] + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_cast_compatibility_live_pg( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + """Live PostgreSQL verifies integer->boolean falls back to VALUES, while object integer->bit(4) uses COPY.""" + # 1. Integer DataFrame into boolean column (falls back to VALUES) + bool_table = ctx.table("target_bool") + bool_types = _types(id="int", flag="boolean") + engine_adapter.create_table(bool_table, bool_types) + df_bool = pd.DataFrame({"id": [1, 2], "flag": [0, 1]}) + engine_adapter.insert_append(bool_table, df_bool, target_columns_to_types=bool_types) + assert engine_adapter.fetchall( + f"SELECT * FROM {bool_table.sql('postgres', identify=True)}" + ) == [ + (1, False), + (2, True), + ] + + # 2. Object-dtype integer DataFrame into boolean column (falls back to VALUES) + df_obj_bool = pd.DataFrame({"id": [3, 4], "flag": pd.Series([1, 0], dtype=object)}) + engine_adapter.insert_append(bool_table, df_obj_bool, target_columns_to_types=bool_types) + assert ( + len(engine_adapter.fetchall(f"SELECT * FROM {bool_table.sql('postgres', identify=True)}")) + == 4 + ) + # 3. Object-dtype integer DataFrame into bit(4) column (uses COPY via BIGINT) + bit_table = ctx.table("target_bit") + bit_types = _types(id="int", b="bit(4)") + engine_adapter.create_table(bit_table, bit_types) + df_obj_bit = pd.DataFrame({"id": [1, 2], "b": pd.Series([1, 2], dtype=object)}) + engine_adapter.insert_append(bit_table, df_obj_bit, target_columns_to_types=bit_types) + assert engine_adapter.fetchall(f"SELECT * FROM {bit_table.sql('postgres', identify=True)}") == [ + (1, "0001"), + (2, "0010"), + ] assert ( _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] ) - assert engine_adapter.fetchone("select 1") == (1,) diff --git a/tests/core/engine_adapter/test_postgres.py b/tests/core/engine_adapter/test_postgres.py index 568d7e133c..10e2ead39a 100644 --- a/tests/core/engine_adapter/test_postgres.py +++ b/tests/core/engine_adapter/test_postgres.py @@ -124,7 +124,9 @@ def test_merge_version_lt_15( adapter = make_mocked_engine_adapter(PostgresEngineAdapter) adapter.server_version = (14, 0) - temp_table_mock = mocker.patch("sqlmesh.core.engine_adapter.EngineAdapter._get_temp_table") + temp_table_mock = mocker.patch( + "sqlmesh.core.engine_adapter.postgres.PostgresEngineAdapter._get_temp_table" + ) table_name = "test" temp_table_id = "abcdefgh" temp_table_mock.return_value = make_temp_table_name(table_name, temp_table_id) @@ -314,7 +316,7 @@ def copy_adapter( adapter = make_mocked_engine_adapter(PostgresEngineAdapter) mocker.patch.object(adapter, "table_exists", return_value=False) mocker.patch( - "sqlmesh.core.engine_adapter.EngineAdapter._get_temp_table", + "sqlmesh.core.engine_adapter.postgres.PostgresEngineAdapter._get_temp_table", return_value=make_temp_table_name("tbl", "abcdefgh"), ) return adapter @@ -574,3 +576,131 @@ def test_df_copy_failure_drops_staging_table_and_reraises( with pytest.raises(ValueError, match="copy failed"): source_queries[0].query_factory() assert to_sql_calls(copy_adapter)[-1] == 'DROP TABLE IF EXISTS "__temp_tbl_abcdefgh"' + + +def test_temp_table_name_respects_max_identifier_length_utf8( + make_mocked_engine_adapter: t.Callable, +): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + # CJK: 20 characters * 3 bytes/char = 60 bytes + cjk_table = f"s.{'界' * 10}__{'世' * 10}" + first = adapter._get_temp_table(cjk_table) + second = adapter._get_temp_table(cjk_table) + for temp in (first, second): + name_bytes = temp.name.encode("utf-8") + assert len(name_bytes) <= 63 + assert temp.name.startswith("__temp_界") + assert temp.name.rsplit("_", 1)[0].endswith("世") + # Validate that the string is valid UTF-8 and code points are intact + assert temp.name == name_bytes.decode("utf-8") + adapter._check_identifier_length(exp.Create(this=temp, kind="TABLE")) + assert first.name != second.name + + # 4-byte UTF-8 emojis + emoji_table = f"s.{'🚀' * 16}" + emoji_temp = adapter._get_temp_table(emoji_table) + assert len(emoji_temp.name.encode("utf-8")) <= 63 + assert emoji_temp.name.startswith("__temp_🚀") + assert emoji_temp.name.rsplit("_", 1)[0].endswith("🚀") + adapter._check_identifier_length(exp.Create(this=emoji_temp, kind="TABLE")) + + # Accented Latin: 35 characters * 2 bytes/char = 70 bytes + latin_table = f"s.{'é' * 35}" + latin_temp = adapter._get_temp_table(latin_table) + assert len(latin_temp.name.encode("utf-8")) <= 63 + assert latin_temp.name.startswith("__temp_é") + adapter._check_identifier_length(exp.Create(this=latin_temp, kind="TABLE")) + + +def test_check_identifier_length_utf8_raises(make_mocked_engine_adapter: t.Callable): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + # 22 CJK characters = 66 bytes > 63 bytes (even though char length is 22 < 63) + long_utf8 = "界" * 22 + with pytest.raises(SQLMeshError, match="exceeds Postgres's max identifier limit of 63 bytes"): + adapter._check_identifier_length(exp.Create(this=exp.to_table(long_utf8), kind="TABLE")) + + +def test_df_copy_incompatible_target_types_fall_back_to_values( + copy_adapter: PostgresEngineAdapter, +): + # BIGINT DataFrame column targeting BOOLEAN + df_int = pd.DataFrame({"a": [0, 1]}) + copy_adapter.insert_append( + "tbl", + df_int, + target_columns_to_types={"a": exp.DataType.build("boolean", dialect="postgres")}, + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + # Object-dtype integer column targeting BOOLEAN + df_obj_int = pd.DataFrame({"a": pd.Series([0, 1, None], dtype=object)}) + copy_adapter.insert_append( + "tbl", + df_obj_int, + target_columns_to_types={"a": exp.DataType.build("boolean", dialect="postgres")}, + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + # Huge integer (> 64-bit) staged as NUMERIC targeting BIT(4) or OID falls back to VALUES + df_huge = pd.DataFrame({"a": pd.Series([2**64, None], dtype=object)}) + copy_adapter.insert_append( + "tbl", + df_huge, + target_columns_to_types={"a": exp.DataType.build("bit(4)", dialect="postgres")}, + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + copy_adapter.insert_append( + "tbl", df_huge, target_columns_to_types={"a": exp.DataType.build("oid", dialect="postgres")} + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + # Float64 staged as NUMERIC targeting BIT(4) falls back to VALUES + df_float = pd.DataFrame({"a": [1.5, 2.5]}) + copy_adapter.insert_append( + "tbl", + df_float, + target_columns_to_types={"a": exp.DataType.build("bit(4)", dialect="postgres")}, + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + # UNKNOWN target type falls back to VALUES + copy_adapter.insert_append( + "tbl", + df_int, + target_columns_to_types={"a": exp.DataType.build("unknown", dialect="postgres")}, + ) + copy_adapter.cursor.copy_expert.assert_not_called() + assert "VALUES" in to_sql_calls(copy_adapter)[-1] + + +def test_df_copy_compatible_object_int_uses_copy( + copy_adapter: PostgresEngineAdapter, +): + # Object-dtype integer within 64-bit range targeting BIT(4) stages as BIGINT and uses COPY + df_obj_int = pd.DataFrame({"a": pd.Series([1, 2, None], dtype=object)}) + copy_adapter.insert_append( + "tbl", + df_obj_int, + target_columns_to_types={"a": exp.DataType.build("bit(4)", dialect="postgres")}, + ) + assert copy_adapter.cursor.copy_expert.call_count == 1 + assert "BIGINT" in to_sql_calls(copy_adapter)[0] + + +def test_df_copy_transaction_block_encloses_create_and_copy( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + tx_spy = mocker.spy(copy_adapter, "transaction") + df = pd.DataFrame({"a": [1, 2]}) + source_queries = copy_adapter._df_to_source_queries( + df, {"a": exp.DataType.build("int", dialect="postgres")}, 10000, "tbl" + ) + with source_queries[0]: + pass + assert tx_spy.call_count >= 1 diff --git a/tests/core/engine_adapter/test_risingwave.py b/tests/core/engine_adapter/test_risingwave.py index 2711b5af16..23a5d93595 100644 --- a/tests/core/engine_adapter/test_risingwave.py +++ b/tests/core/engine_adapter/test_risingwave.py @@ -99,3 +99,11 @@ def test_df_is_not_loaded_with_copy(adapter: t.Callable): sql_calls = [call_[0][0] for call_ in risingwave.cursor.execute.call_args_list] assert len(sql_calls) == 1 assert "VALUES" in str(sql_calls[0]) and "__temp_" not in str(sql_calls[0]) + + +def test_temp_table_name_not_truncated(adapter: t.Callable): + """RisingWave does not enforce MAX_IDENTIFIER_LENGTH, so temp table names are not truncated.""" + long_name = "x" * 70 + temp = adapter._get_temp_table(f"db.{long_name}") + assert temp.name.startswith(f"__temp_{long_name}_") + assert len(temp.name) > 70 From 4ca3a67fc143f4d1633a8ea83ab35be9cb1edb09 Mon Sep 17 00:00:00 2001 From: Vamsi-klu Date: Mon, 16 Nov 2026 06:01:09 +0000 Subject: [PATCH 3/3] fix(postgres): harden COPY staging cleanup, JSON fallback, and CR quoting - Wrap DataFrame replace_query, ctas, merge, and _scd_type_2 calls in a single transaction so COPY staging DDL, target writes, and cleanup commit or roll back atomically - Protect COMMENT ON statements inside active transactions with a SAVEPOINT so swallowed comment permission/policy failures do not leave psycopg2 in PQTRANS_INERROR and silently roll back DataFrame loads on commit - Track standalone staging table commits in _df_to_source_queries, roll back aborted 25P02 transactions before retrying DROP TABLE, and never mask propagating caller errors with secondary drop failures - Fall back to VALUES when non-TEXT staging columns target JSON or JSONB while preserving COPY for TEXT JSON strings - Use CRLF CSV row terminators in _copy_df so Python 3.9 csv.QUOTE_MINIMAL quotes bare carriage returns, and clarify staging table behavior in Postgres docs Signed-off-by: Vamsi-klu --- docs/integrations/engines/postgres.md | 4 +- sqlmesh/core/engine_adapter/postgres.py | 209 +++++++++- .../integration/test_integration_postgres.py | 360 ++++++++++++++++++ tests/core/engine_adapter/test_postgres.py | 277 +++++++++++++- 4 files changed, 825 insertions(+), 25 deletions(-) diff --git a/docs/integrations/engines/postgres.md b/docs/integrations/engines/postgres.md index ae39b3523f..2bc808eedd 100644 --- a/docs/integrations/engines/postgres.md +++ b/docs/integrations/engines/postgres.md @@ -26,12 +26,12 @@ pip install "sqlmesh[postgres]" ### DataFrame loading -When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those returned by Python models and used for seeds) with `COPY ... FROM STDIN (FORMAT csv)` instead of batched `INSERT ... VALUES` statements, which is much faster for large DataFrames. The data is copied into a temporary staging table, which is created in the same schema as the target table and then cast into the target table's column types. The staging table is dropped afterwards, including when the load fails. Because it is created in the target's schema, the connecting user needs the `CREATE` privilege on that schema, as for other SQLMesh DDL. +When using the `psycopg2` driver, SQLMesh loads pandas DataFrames (such as those returned by Python models and used for seeds) with `COPY ... FROM STDIN (FORMAT csv)` instead of batched `INSERT ... VALUES` statements, which is much faster for large DataFrames. The data is copied into a regular schema-qualified staging table (`__temp_...`, rather than a session-local PostgreSQL `TEMP` table) created in the same schema as the target table and then cast into the target table's column types. SQLMesh DataFrame loads run inside a transaction and drop the staging table at the end of the load (or roll back its creation if the load fails inside the transaction); if a staging query is run outside an enclosing transaction, default schema privileges apply while the table exists and an abrupt connection drop or process crash may leave a `__temp_*` table behind that operators can remove. Because the staging table is created in the target's schema, the connecting user needs the `CREATE` privilege on that schema, as for other SQLMesh DDL. No configuration is required and there is no option to disable this behavior. SQLMesh automatically falls back to `INSERT ... VALUES` when: - A column has a type or values that cannot be loaded with `COPY` identically. Supported columns are integers (up to 64 bits signed, up to 32 bits unsigned), `float64`, booleans, timestamps (with or without time zone), and strings. Other types, such as `float32`, 64-bit unsigned integers, `Decimal`, dates, bytes, dicts, lists, and categoricals, trigger the fallback. -- A column's staging type is incompatible with the target column's cast (for example, an integer or numeric DataFrame column targeting a `BOOLEAN`, `BIT`, or `OID`/`REG*` type). +- A column's staging type is incompatible with the target column's cast (for example, a non-text DataFrame column targeting `JSON` or `JSONB`, or an integer or numeric column targeting `BOOLEAN`, `BIT`, or `OID`/`REG*`). - The DataFrame has a single column and one of its values is `\.`. - The driver does not support `copy_expert` (for example, a driver other than `psycopg2`). diff --git a/sqlmesh/core/engine_adapter/postgres.py b/sqlmesh/core/engine_adapter/postgres.py index 9f5e6088b6..ccd8e8cf8f 100644 --- a/sqlmesh/core/engine_adapter/postgres.py +++ b/sqlmesh/core/engine_adapter/postgres.py @@ -1,9 +1,11 @@ from __future__ import annotations +import contextlib import csv import io import logging import re +import sys import typing as t from functools import cached_property, partial from sqlglot import exp @@ -170,20 +172,28 @@ def _df_to_source_queries( ) staging_table = self._get_temp_table(target_table or "pandas") + committed_standalone = False def query_factory() -> Query: + nonlocal committed_standalone # The factory can be called multiple times. The staging table is only loaded once, which # assumes that every call would have produced the same result. if not self.table_exists(staging_table): + committed_standalone = False + created_in_outer_tx = self._connection_pool.is_transaction_active try: with self.transaction(): self.create_table(staging_table, staging_columns_to_types) self._copy_df(staging_table, ordered_df, null_marker, batch_size) + if not created_in_outer_tx: + committed_standalone = True except Exception: # Callers such as `replace_query` invoke the factory outside of `with source_query`, # so `cleanup_func` won't run and a committed staging table would leak. try: - self._drop_copy_staging_table(staging_table) + self._drop_copy_staging_table( + staging_table, committed_standalone=committed_standalone + ) except Exception: logger.warning("Failed to drop staging table %s", staging_table.sql()) raise @@ -194,7 +204,9 @@ def query_factory() -> Query: return [ SourceQuery( query_factory=query_factory, - cleanup_func=lambda: self._drop_copy_staging_table(staging_table), + cleanup_func=lambda: self._drop_copy_staging_table( + staging_table, committed_standalone=committed_standalone + ), ) ] @@ -259,17 +271,51 @@ def _check_identifier_length(self, expression: exp.Expr) -> None: f"Identifier name '{name}' (length {name_bytes} bytes) exceeds {self.dialect.capitalize()}'s max identifier limit of {self.MAX_IDENTIFIER_LENGTH} bytes" ) - def _drop_copy_staging_table(self, staging_table: exp.Table) -> None: + def _drop_copy_staging_table( + self, staging_table: exp.Table, *, committed_standalone: bool = False + ) -> None: + exc_propagating = sys.exc_info()[0] is not None try: self.drop_table(staging_table) except Exception as ex: - # The cleanup runs while an exception is propagating. If it left the transaction aborted - # the DROP fails with "current transaction is aborted", which would hide the original - # error. The staging table was created in that transaction, so its rollback removes it. sqlstate = getattr(ex, "pgcode", None) or getattr(ex, "sqlstate", None) - if sqlstate != "25P02" or not self._connection_pool.is_transaction_active: - raise - logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql()) + if ( + sqlstate == "25P02" + and self._connection_pool.is_transaction_active + and not committed_standalone + ): + # The staging table was created in the active (aborted) outer transaction, so its + # rollback removes it without masking the caller's error. + logger.debug("Skipped dropping %s from an aborted transaction", staging_table.sql()) + return + if sqlstate == "25P02" and committed_standalone: + # The staging table was already committed in its own transaction before a subsequent + # statement aborted the connection's transaction, so rollback alone will not drop it. + was_in_tx = self._connection_pool.is_transaction_active + try: + self._connection_pool.rollback() + with self.transaction(): + self.drop_table(staging_table) + if was_in_tx and not exc_propagating: + self._connection_pool.begin() + return + except Exception: + if exc_propagating: + logger.warning( + "Failed to drop staging table %s after rolling back aborted transaction", + staging_table.sql(), + exc_info=True, + ) + return + raise + if exc_propagating: + logger.warning( + "Failed to drop staging table %s", + staging_table.sql(), + exc_info=True, + ) + return + raise @staticmethod def _is_oid_or_reg_type(data_type: exp.DataType) -> bool: @@ -381,6 +427,10 @@ def _get_copy_staging_columns_to_types( staging_type = None if staging_type is None: return None + if staging_type != "TEXT" and target_type.is_type( + exp.DataType.Type.JSON, exp.DataType.Type.JSONB + ): + return None staging_columns_to_types[col_name] = exp.DataType.build( staging_type, dialect=PostgresEngineAdapter.DIALECT ) @@ -408,12 +458,130 @@ def _copy_df( index=False, na_rep=null_marker, quoting=csv.QUOTE_MINIMAL, - lineterminator="\n", + lineterminator="\r\n", ) buffer.seek(0) # A text buffer is encoded by the driver with the connection's client_encoding self.cursor.copy_expert(copy_sql, buffer) + def replace_query( + self, + table_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + source_columns: t.Optional[t.List[str]] = None, + supports_replace_table_override: t.Optional[bool] = None, + **kwargs: t.Any, + ) -> None: + import pandas as pd + + with self.transaction(condition=isinstance(query_or_df, pd.DataFrame)): + return super().replace_query( + table_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + table_description=table_description, + column_descriptions=column_descriptions, + source_columns=source_columns, + supports_replace_table_override=supports_replace_table_override, + **kwargs, + ) + + def ctas( + self, + table_name: TableName, + query_or_df: QueryOrDF, + target_columns_to_types: t.Optional[t.Dict[str, exp.DataType]] = None, + exists: bool = True, + table_description: t.Optional[str] = None, + column_descriptions: t.Optional[t.Dict[str, str]] = None, + source_columns: t.Optional[t.List[str]] = None, + **kwargs: t.Any, + ) -> None: + import pandas as pd + + with self.transaction(condition=isinstance(query_or_df, pd.DataFrame)): + return super().ctas( + table_name, + query_or_df, + target_columns_to_types=target_columns_to_types, + exists=exists, + table_description=table_description, + column_descriptions=column_descriptions, + source_columns=source_columns, + **kwargs, + ) + + def _scd_type_2( + self, + target_table: TableName, + source_table: QueryOrDF, + *args: t.Any, + **kwargs: t.Any, + ) -> None: + import pandas as pd + + with self.transaction(condition=isinstance(source_table, pd.DataFrame)): + super()._scd_type_2(target_table, source_table, *args, **kwargs) + + @contextlib.contextmanager + def _comment_savepoint(self) -> t.Iterator[None]: + if ( + self._connection_pool.is_transaction_active is not True + or not self.SUPPORTS_TRANSACTIONS + ): + yield + return + + savepoint = "_sqlmesh_comment" + self.execute(f"SAVEPOINT {savepoint}") + try: + yield + except Exception: + try: + self.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") + self.execute(f"RELEASE SAVEPOINT {savepoint}") + except Exception: + logger.debug("Failed to roll back savepoint %s", savepoint, exc_info=True) + raise + else: + self.execute(f"RELEASE SAVEPOINT {savepoint}") + + def _create_table_comment( + self, table_name: TableName, table_comment: str, table_kind: str = "TABLE" + ) -> None: + table = exp.to_table(table_name) + try: + with self._comment_savepoint(): + self.execute(self._build_create_comment_table_exp(table, table_comment, table_kind)) + except Exception: + logger.warning( + f"Table comment for '{table.alias_or_name}' not registered - this may be due to limited permissions", + exc_info=True, + ) + + def _create_column_comments( + self, + table_name: TableName, + column_comments: t.Dict[str, str], + table_kind: str = "TABLE", + materialized_view: bool = False, + ) -> None: + table = exp.to_table(table_name) + for col, comment in column_comments.items(): + try: + with self._comment_savepoint(): + self.execute( + self._build_create_comment_column_exp(table, col, comment, table_kind) + ) + except Exception: + logger.warning( + f"Column comments for column '{col}' in table '{table.alias_or_name}' not registered - this may be due to limited permissions", + exc_info=True, + ) + def _create_table_like( self, target_table_name: TableName, @@ -448,18 +616,21 @@ def merge( source_columns: t.Optional[t.List[str]] = None, **kwargs: t.Any, ) -> None: + import pandas as pd + # Merge isn't supported until Postgres 15 major, minor = self.server_version merge_impl = super().merge if major >= 15 else partial(logical_merge, self) - merge_impl( # type: ignore - target_table, - source_table, - target_columns_to_types, - unique_key, - when_matched=when_matched, - merge_filter=merge_filter, - source_columns=source_columns, - ) + with self.transaction(condition=isinstance(source_table, pd.DataFrame)): + merge_impl( # type: ignore + target_table, + source_table, + target_columns_to_types, + unique_key, + when_matched=when_matched, + merge_filter=merge_filter, + source_columns=source_columns, + ) @cached_property def server_version(self) -> t.Tuple[int, int]: diff --git a/tests/core/engine_adapter/integration/test_integration_postgres.py b/tests/core/engine_adapter/integration/test_integration_postgres.py index 89a825be55..b36a6f9373 100644 --- a/tests/core/engine_adapter/integration/test_integration_postgres.py +++ b/tests/core/engine_adapter/integration/test_integration_postgres.py @@ -2080,3 +2080,363 @@ def test_df_copy_cast_compatibility_live_pg( assert ( _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] ) + + +@pytest.mark.parametrize( + "operation", + ["replace_query_new", "replace_query_existing", "ctas", "merge_native", "merge_logical"], +) +@pytest.mark.parametrize("pre_ping", [False, True]) +def test_df_copy_target_failure_cleans_up_staging_table_and_preserves_error( + engine_adapter: PostgresEngineAdapter, + ctx: TestContext, + mocker: MockerFixture, + operation: str, + pre_ping: bool, +): + """When COPY into `__temp_*` succeeds and the subsequent target write fails, the original DB + error must propagate (never `InFailedSqlTransaction`) and zero `__temp_*` tables may remain.""" + if operation == "merge_native" and engine_adapter.server_version < (15, 0): + pytest.skip("Native MERGE requires PostgreSQL 15+") + + adapter = t.cast( + PostgresEngineAdapter, + engine_adapter.with_settings(pre_ping=pre_ping), + ) + schema_name = exp.to_table(ctx.schema(TEST_SCHEMA)).name + table = ctx.table(f"target_fail_{operation}_{pre_ping}") + sql_table = table.sql("postgres", identify=True) + types = _types(id="int", val="int") + + exists_before = operation in ("replace_query_existing", "merge_native", "merge_logical") + if exists_before: + adapter.execute(f"CREATE TABLE {sql_table} (id INT, val INT CHECK (val >= 0))") + adapter.insert_append( + table, pd.DataFrame({"id": [1], "val": [100]}), target_columns_to_types=types + ) + + # Strings stage into TEXT via COPY without error, then fail when cast to INT in target SQL + bad_df = pd.DataFrame({"id": [1, 2], "val": ["10", "not_an_int"]}) + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + + original_version = adapter.server_version + try: + if operation == "merge_logical": + adapter.server_version = (14, 0) # type: ignore + + with pytest.raises(Exception) as exc_info: + if operation in ("replace_query_new", "replace_query_existing"): + adapter.replace_query(table, bad_df, target_columns_to_types=types) + elif operation == "ctas": + adapter.ctas(table, bad_df, target_columns_to_types=types) + else: + adapter.merge( + table, + bad_df, + target_columns_to_types=types, + unique_key=[exp.to_column("id", quoted=True)], + ) + finally: + adapter.server_version = original_version + + err_msg = str(exc_info.value) + assert "not_an_int" in err_msg + assert "current transaction is aborted" not in err_msg + assert spy.call_count == 1 + assert not adapter._connection_pool.is_transaction_active + assert _leftover_staging_tables(adapter, schema_name) == [] + assert adapter.fetchone("SELECT 1") == (1,) + + if exists_before: + assert adapter.fetchall(f"SELECT id, val FROM {sql_table}") == [(1, 100)] + else: + assert not adapter.table_exists(table) + + +def test_df_copy_check_constraint_failure_rolls_back_atomically( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, mocker: MockerFixture +): + """Check constraint failure on replace_query / merge with pre_ping=False leaves zero `__temp_*` tables.""" + adapter = t.cast(PostgresEngineAdapter, engine_adapter.with_settings(pre_ping=False)) + schema_name = exp.to_table(ctx.schema(TEST_SCHEMA)).name + table = ctx.table("check_violation") + sql_table = table.sql("postgres", identify=True) + types = _types(id="int", val="int") + adapter.execute(f"CREATE TABLE {sql_table} (id INT, val INT CHECK (val > 0))") + adapter.insert_append( + table, pd.DataFrame({"id": [1], "val": [10]}), target_columns_to_types=types + ) + + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + bad_df = pd.DataFrame({"id": [1, 2], "val": [5, -1]}) + + for fn in ( + lambda: adapter.replace_query(table, bad_df, target_columns_to_types=types), + lambda: adapter.merge( + table, + bad_df, + target_columns_to_types=types, + unique_key=[exp.to_column("id", quoted=True)], + ), + ): + with pytest.raises(Exception, match="check_violation_val_check|violates check constraint"): + fn() + assert _leftover_staging_tables(adapter, schema_name) == [] + assert adapter.fetchall(f"SELECT id, val FROM {sql_table}") == [(1, 10)] + assert adapter.fetchone("SELECT 1") == (1,) + + assert spy.call_count == 2 + + +def test_df_copy_standalone_source_query_cleanup_after_aborted_tx( + engine_adapter: PostgresEngineAdapter, ctx: TestContext +): + """Direct SourceQuery callers that commit `__temp_*` standalone still clean up on 25P02.""" + adapter = t.cast(PostgresEngineAdapter, engine_adapter.with_settings(pre_ping=False)) + schema_name = exp.to_table(ctx.schema(TEST_SCHEMA)).name + table = ctx.table("standalone_sq") + types = _types(a="int") + adapter.create_table(table, types) + + # 1. Implicit psycopg2 transaction aborted via raw cursor execution (is_transaction_active=False) + sq1 = adapter._df_to_source_queries(pd.DataFrame({"a": [1, 2]}), types, 1000, table)[0] + with pytest.raises(Exception, match="bad_implicit_int") as exc1: + with sq1: + assert len(_leftover_staging_tables(engine_adapter, schema_name)) == 1 + adapter.cursor.execute("SELECT 'bad_implicit_int'::int") + assert "current transaction is aborted" not in str(exc1.value) + assert _leftover_staging_tables(adapter, schema_name) == [] + assert _leftover_staging_tables(engine_adapter, schema_name) == [] + assert adapter.fetchone("SELECT 1") == (1,) + + # 2. Factory invoked standalone before an explicit outer transaction that then aborts + sq2 = adapter._df_to_source_queries(pd.DataFrame({"a": [3, 4]}), types, 1000, table)[0] + sq2.query_factory() + assert len(_leftover_staging_tables(engine_adapter, schema_name)) == 1 + with pytest.raises(Exception, match="bad_explicit_int") as exc2: + with adapter.transaction(): + with sq2: + adapter.execute("SELECT 'bad_explicit_int'::int") + assert "current transaction is aborted" not in str(exc2.value) + assert _leftover_staging_tables(adapter, schema_name) == [] + assert _leftover_staging_tables(engine_adapter, schema_name) == [] + assert adapter.fetchone("SELECT 1") == (1,) + + +def test_df_copy_json_and_jsonb_live_pg( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, mocker: MockerFixture +): + """Numeric DataFrames targeting JSON succeed via VALUES fallback; string JSON DataFrames use COPY.""" + schema_name = exp.to_table(ctx.schema(TEST_SCHEMA)).name + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + + # 1. Numeric DataFrames (int64, float64, object-int) targeting JSON succeed via VALUES fallback + json_num_table = ctx.table("target_json_num") + json_types = _types(id="int", payload="json") + engine_adapter.create_table(json_num_table, json_types) + engine_adapter.insert_append( + json_num_table, + pd.DataFrame({"id": [1, 2], "payload": [10, 20]}), + target_columns_to_types=json_types, + ) + engine_adapter.insert_append( + json_num_table, + pd.DataFrame({"id": [3, 4], "payload": [1.5, 2.5]}), + target_columns_to_types=json_types, + ) + engine_adapter.insert_append( + json_num_table, + pd.DataFrame({"id": [5, 6], "payload": pd.Series([100, 200], dtype=object)}), + target_columns_to_types=json_types, + ) + assert spy.call_count == 0 + assert engine_adapter.fetchall( + f"SELECT id, payload FROM {json_num_table.sql('postgres', identify=True)} ORDER BY id" + ) == [(1, 10), (2, 20), (3, 1.5), (4, 2.5), (5, 100), (6, 200)] + + # 2. Numeric DataFrames targeting JSONB also fall back to VALUES without creating staging tables + jsonb_types = _types(id="int", payload="jsonb") + jsonb_num_table = ctx.table("target_jsonb_num") + engine_adapter.create_table(jsonb_num_table, jsonb_types) + for num_df in ( + pd.DataFrame({"id": [1], "payload": [10]}), + pd.DataFrame({"id": [2], "payload": [1.5]}), + ): + with pytest.raises(Exception, match="cannot cast type"): + engine_adapter.insert_append( + jsonb_num_table, num_df, target_columns_to_types=jsonb_types + ) + assert spy.call_count == 0 + assert _leftover_staging_tables(engine_adapter, schema_name) == [] + + # 3. Valid JSON string DataFrames targeting JSON and JSONB succeed via COPY + for col_kind in ("json", "jsonb"): + str_table = ctx.table(f"target_{col_kind}_str") + str_types = _types(id="int", payload=col_kind) + engine_adapter.create_table(str_table, str_types) + df_str = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "payload": ['{"k": 1}', '[1, "two"]', "true", None], + } + ) + before_calls = spy.call_count + engine_adapter.insert_append(str_table, df_str, target_columns_to_types=str_types) + assert spy.call_count == before_calls + 1 + assert engine_adapter.fetchall( + f"SELECT id, payload FROM {str_table.sql('postgres', identify=True)} ORDER BY id" + ) == [ + (1, {"k": 1}), + (2, [1, "two"]), + (3, True), + (4, None), + ] + + assert _leftover_staging_tables(engine_adapter, schema_name) == [] + + +def test_df_copy_carriage_returns_and_newlines_round_trip( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, mocker: MockerFixture +): + """Strings with bare CR, bare LF, and CRLF must round-trip byte-for-byte via COPY.""" + spy = mocker.spy(PostgresEngineAdapter, "_copy_df") + values = [ + "a\rb", + "\r", + "\r\r", + "a\nb", + "\n", + "a\r\nb", + "\r\n", + "\n\r", + 'q"uote\rhere', + "c,omma\r\nhere", + "", + ] + + # Multi-column table (ordered by id) + multi_table = ctx.table("cr_lf_multi") + multi_types = _types(id="int", s="text") + engine_adapter.create_table(multi_table, multi_types) + engine_adapter.insert_append( + multi_table, + pd.DataFrame({"id": list(range(len(values))), "s": values}), + target_columns_to_types=multi_types, + ) + rows = engine_adapter.fetchall( + f"SELECT id, s, encode(convert_to(s, 'UTF8'), 'hex') " + f"FROM {multi_table.sql('postgres', identify=True)} ORDER BY id" + ) + assert [(r[0], r[1], r[2]) for r in rows] == [ + (idx, val, val.encode("utf-8").hex()) for idx, val in enumerate(values) + ] + + # Single-column table + single_table = ctx.table("cr_lf_single") + single_types = _types(s="text") + engine_adapter.create_table(single_table, single_types) + engine_adapter.insert_append( + single_table, + pd.DataFrame({"s": values}), + target_columns_to_types=single_types, + ) + single_hex = sorted( + r[0] + for r in engine_adapter.fetchall( + f"SELECT encode(convert_to(s, 'UTF8'), 'hex') " + f"FROM {single_table.sql('postgres', identify=True)}" + ) + ) + assert single_hex == sorted(v.encode("utf-8").hex() for v in values) + assert spy.call_count == 2 + assert ( + _leftover_staging_tables(engine_adapter, exp.to_table(ctx.schema(TEST_SCHEMA)).name) == [] + ) + + +def test_df_copy_swallowed_comment_failure_does_not_rollback_load( + engine_adapter: PostgresEngineAdapter, ctx: TestContext, config: Config +): + """A swallowed `COMMENT ON` error inside transactional `ctas(df)` / `replace_query(df)` must + roll back only to `_sqlmesh_comment` and still commit the table and DataFrame rows.""" + schema_name = exp.to_table(ctx.schema(TEST_SCHEMA)).name + types = _types(id="int", val="text") + df = pd.DataFrame({"id": [1, 2], "val": ["alpha", "beta"]}) + + # 1. Event trigger rejects COMMENT ON TABLE while allowing COMMENT ON COLUMN: + # both `ctas(df)` and `replace_query(df)` must commit the table, its rows, AND column comments! + fn_name = f"_block_tbl_comment_{random_id(short=True)}" + trg_name = f"_trg_block_tbl_comment_{random_id(short=True)}" + tag_prefix = f"comment_guard_{random_id(short=True)}" + engine_adapter.execute( + f""" + CREATE OR REPLACE FUNCTION {fn_name}() RETURNS event_trigger LANGUAGE plpgsql AS $$ + BEGIN + IF current_query() LIKE 'COMMENT ON TABLE%' AND current_query() LIKE '%{tag_prefix}%' THEN + RAISE EXCEPTION 'table comments blocked for {tag_prefix}'; + END IF; + END; + $$ + """ + ) + engine_adapter.execute( + f"CREATE EVENT TRIGGER {trg_name} ON ddl_command_start " + f"WHEN TAG IN ('COMMENT') EXECUTE FUNCTION {fn_name}()" + ) + try: + ctas_table = ctx.table(f"{tag_prefix}_ctas") + engine_adapter.ctas( + ctas_table, + df, + target_columns_to_types=types, + table_description="blocked table comment", + column_descriptions={"id": "id col comment", "val": "val col comment"}, + ) + replace_table = ctx.table(f"{tag_prefix}_replace") + engine_adapter.replace_query( + replace_table, + df, + target_columns_to_types=types, + table_description="blocked table comment", + column_descriptions={"id": "id col comment"}, + ) + finally: + engine_adapter.execute(f"DROP EVENT TRIGGER IF EXISTS {trg_name}") + engine_adapter.execute(f"DROP FUNCTION IF EXISTS {fn_name}()") + + for tbl in (ctas_table, replace_table): + sql_tbl = tbl.sql("postgres", identify=True) + assert engine_adapter.table_exists(tbl) + assert engine_adapter.fetchall(f"SELECT id, val FROM {sql_tbl} ORDER BY id") == [ + (1, "alpha"), + (2, "beta"), + ] + col_comment = engine_adapter.fetchone(f"SELECT col_description('{sql_tbl}'::regclass, 1)") + assert col_comment == ("id col comment",) + + # 2. Non-owner role (`42501: must be owner of table`) running inside an active transaction + # (`insert_append` + `ctas(..., exists=True, table_description=...)`) must keep inserted rows! + shared_table = ctx.table("shared_comment_perm") + engine_adapter.create_table(shared_table, types) + with create_users(engine_adapter, "writer") as roles: + writer_user = roles["writer"]["username"] + engine_adapter.execute(f'GRANT USAGE, CREATE ON SCHEMA "{schema_name}" TO "{writer_user}"') + engine_adapter.execute( + f'GRANT SELECT, INSERT ON {shared_table.sql("postgres", identify=True)} TO "{writer_user}"' + ) + with engine_adapter_for_role(roles["writer"], ctx, config) as writer_adapter: + with writer_adapter.transaction(): + writer_adapter.insert_append(shared_table, df, target_columns_to_types=types) + writer_adapter.ctas( + shared_table, + df, + target_columns_to_types=types, + exists=True, + table_description="non-owner comment fails with 42501", + column_descriptions={"id": "non-owner col comment fails with 42501"}, + ) + + assert engine_adapter.fetchall( + f"SELECT id, val FROM {shared_table.sql('postgres', identify=True)} ORDER BY id" + ) == [(1, "alpha"), (2, "beta")] + assert _leftover_staging_tables(engine_adapter, schema_name) == [] diff --git a/tests/core/engine_adapter/test_postgres.py b/tests/core/engine_adapter/test_postgres.py index 10e2ead39a..137804547d 100644 --- a/tests/core/engine_adapter/test_postgres.py +++ b/tests/core/engine_adapter/test_postgres.py @@ -358,9 +358,9 @@ def test_df_copy_chunks_sql_and_payload(copy_adapter: PostgresEngineAdapter): assert [sql for sql, _ in calls] == [expected_sql, expected_sql] assert ( calls[0][1] - == f"1,x,1.5,True,2020-01-01 00:00:00.123456\n2,{marker},{marker},False,{marker}\n" + == f"1,x,1.5,True,2020-01-01 00:00:00.123456\r\n2,{marker},{marker},False,{marker}\r\n" ) - assert calls[1][1] == '3,"q""uote, comma",0.1,True,1960-01-01 12:00:00\n' + assert calls[1][1] == '3,"q""uote, comma",0.1,True,1960-01-01 12:00:00\r\n' assert to_sql_calls(copy_adapter) == [ 'CREATE TABLE IF NOT EXISTS "__temp_tbl_abcdefgh" ' @@ -378,7 +378,7 @@ def test_df_copy_batch_size_zero_is_single_chunk(copy_adapter: PostgresEngineAda copy_adapter.DEFAULT_BATCH_SIZE = 0 copy_adapter.insert_append("tbl", df, target_columns_to_types={"a": exp.DataType.build("int")}) assert len(calls) == 1 - assert calls[0][1] == "1\n2\n3\n" + assert calls[0][1] == "1\r\n2\r\n3\r\n" def test_df_copy_reorders_columns_and_skips_missing_source_columns( @@ -400,7 +400,7 @@ def test_df_copy_reorders_columns_and_skips_missing_source_columns( # Only the source columns are copied, in the source order assert len(calls) == 1 assert calls[0][0].startswith('COPY "__temp_tbl_abcdefgh" ("b", "a") FROM STDIN') - assert calls[0][1] == "x,1\ny,2\n" + assert calls[0][1] == "x,1\r\ny,2\r\n" assert to_sql_calls(copy_adapter)[1] == ( 'INSERT INTO "tbl" ("a", "b", "c") SELECT CAST("a" AS INT) AS "a", CAST("b" AS TEXT) AS "b", ' 'CAST(NULL AS INT) AS "c" FROM "__temp_tbl_abcdefgh"' @@ -704,3 +704,272 @@ def test_df_copy_transaction_block_encloses_create_and_copy( with source_queries[0]: pass assert tx_spy.call_count >= 1 + + +def test_df_copy_csv_carriage_return_and_newline_quoting( + copy_adapter: PostgresEngineAdapter, +): + calls = _capture_copy_calls(copy_adapter) + df = pd.DataFrame( + { + "id": [1, 2, 3, 4], + "s": ["a\rb", "a\nb", "a\r\nb", "plain"], + } + ) + copy_adapter.insert_append( + "tbl", + df, + target_columns_to_types={ + "id": exp.DataType.build("int", dialect="postgres"), + "s": exp.DataType.build("text", dialect="postgres"), + }, + ) + assert len(calls) == 1 + assert calls[0][1] == '1,"a\rb"\r\n2,"a\nb"\r\n3,"a\r\nb"\r\n4,plain\r\n' + + +@pytest.mark.parametrize("json_type", ["json", "jsonb"]) +@pytest.mark.parametrize( + "series, expect_copy", + [ + (pd.Series([1, 2], dtype="int64"), False), + (pd.Series([1.5, 2.5], dtype="float64"), False), + (pd.Series([1, 2, None], dtype=object), False), + (pd.Series([2**64, None], dtype=object), False), + (pd.Series([1.5, None], dtype=object), False), + (pd.Series([True, False], dtype=object), False), + (pd.Series([True, False, None], dtype=object), False), + (pd.Series(pd.to_datetime(["2020-01-01", "2020-01-02"])), False), + (pd.Series(pd.to_datetime(["2020-01-01", "2020-01-02"]).tz_localize("UTC")), False), + (pd.Series(['{"a": 1}', "[1, 2]", None], dtype=object), True), + (pd.Series(['{"a": 1}', None], dtype="string"), True), + (pd.Series([None, None], dtype=object), True), + ], +) +def test_df_copy_json_and_jsonb_target_types( + copy_adapter: PostgresEngineAdapter, + json_type: str, + series: pd.Series, + expect_copy: bool, +): + df = pd.DataFrame({"j": series}) + target_types = {"j": exp.DataType.build(json_type, dialect="postgres")} + staging_types = copy_adapter._get_copy_staging_columns_to_types( + df, "__sqlmesh_null_test__", target_types + ) + if expect_copy: + assert staging_types == {"j": exp.DataType.build("TEXT", dialect="postgres")} + copy_adapter.insert_append("tbl", df, target_columns_to_types=target_types) + assert copy_adapter.cursor.copy_expert.call_count == 1 + assert "__temp_tbl_abcdefgh" in to_sql_calls(copy_adapter)[0] + else: + assert staging_types is None + copy_adapter.insert_append("tbl", df, target_columns_to_types=target_types) + copy_adapter.cursor.copy_expert.assert_not_called() + sql_calls = to_sql_calls(copy_adapter) + assert len(sql_calls) == 1 + assert "VALUES" in sql_calls[0] + assert "__temp_" not in sql_calls[0] + + # Numpy bool and nullable pandas dtypes also return None from staging type resolution + for non_text_series in ( + pd.Series([True, False], dtype="bool"), + pd.Series([True, None], dtype="boolean"), + pd.Series([1, None], dtype="Int64"), + pd.Series([1.5, None], dtype="Float64"), + ): + assert ( + copy_adapter._get_copy_staging_columns_to_types( + pd.DataFrame({"j": non_text_series}), "__sqlmesh_null_test__", target_types + ) + is None + ) + + +def test_df_high_level_operations_run_in_single_transaction( + make_mocked_engine_adapter: t.Callable, + mocker: MockerFixture, + make_temp_table_name: t.Callable, +): + from sqlmesh.core.engine_adapter.shared import DataObject, DataObjectType + + df = pd.DataFrame({"id": [1, 2], "val": ["a", "b"]}) + types = { + "id": exp.DataType.build("int", dialect="postgres"), + "val": exp.DataType.build("text", dialect="postgres"), + } + + for op in ("replace_new", "replace_existing", "ctas", "merge_15", "merge_14"): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + adapter.server_version = (14, 0) if op == "merge_14" else (15, 0) + mocker.patch.object(adapter, "table_exists", return_value=False) + mocker.patch( + "sqlmesh.core.engine_adapter.postgres.PostgresEngineAdapter._get_temp_table", + side_effect=[ + make_temp_table_name("tbl", "stage123"), + make_temp_table_name("tbl", "merge123"), + ], + ) + if op == "replace_existing": + mocker.patch.object( + adapter, + "get_data_object", + return_value=DataObject(schema="public", name="tbl", type=DataObjectType.TABLE), + ) + + begin_spy = mocker.spy(adapter._connection_pool, "begin") + commit_spy = mocker.spy(adapter._connection_pool, "commit") + + def _assert_in_tx(*_args: t.Any, **_kwargs: t.Any) -> None: + assert adapter._connection_pool.is_transaction_active + + adapter.cursor.copy_expert.side_effect = _assert_in_tx + + if op in ("replace_new", "replace_existing"): + adapter.replace_query("tbl", df, target_columns_to_types=types) + elif op == "ctas": + adapter.ctas("tbl", df, target_columns_to_types=types) + else: + adapter.merge( + "tbl", + df, + target_columns_to_types=types, + unique_key=[exp.to_column("id", quoted=True)], + ) + + # Entire staging CREATE + COPY + target write + staging DROP ran in a single transaction + assert begin_spy.call_count == 1 + assert commit_spy.call_count == 1 + assert not adapter._connection_pool.is_transaction_active + + +def test_sql_query_operations_do_not_add_extra_dataframe_transaction( + make_mocked_engine_adapter: t.Callable, mocker: MockerFixture +): + adapter = make_mocked_engine_adapter(PostgresEngineAdapter) + adapter.server_version = (15, 0) + query = t.cast(exp.Select, parse_one("SELECT 1 AS id, 'a' AS val")) + types = { + "id": exp.DataType.build("int", dialect="postgres"), + "val": exp.DataType.build("text", dialect="postgres"), + } + + tx_spy = mocker.spy(adapter, "transaction") + adapter.merge("tbl", query, target_columns_to_types=types, unique_key=[exp.to_column("id")]) + # Outer DataFrame guard passes condition=False, so only `execute()` enters a transaction + assert tx_spy.call_args_list[0].kwargs == {"condition": False} + + tx_spy.reset_mock() + adapter.ctas("tbl", query, target_columns_to_types=types) + assert tx_spy.call_args_list[0].kwargs == {"condition": False} + + tx_spy.reset_mock() + adapter.replace_query("tbl", query, target_columns_to_types=types) + assert tx_spy.call_args_list[0].kwargs == {"condition": False} + + +def test_drop_copy_staging_table_committed_standalone_aborted_tx( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + df = pd.DataFrame({"a": [1, 2]}) + types = {"a": exp.DataType.build("int", dialect="postgres")} + + # 1. Standalone SourceQuery whose subsequent raw cursor query aborts the implicit tx (25P02): + # cleanup rolls back the aborted implicit tx, drops the staging table in a committed tx, + # and lets the caller's original exception propagate unmasked. + source_queries = copy_adapter._df_to_source_queries(df, types, 10000, "tbl") + rollback_spy = mocker.spy(copy_adapter._connection_pool, "rollback") + commit_spy = mocker.spy(copy_adapter._connection_pool, "commit") + real_drop = copy_adapter.drop_table + drop_attempts = 0 + + def _flaky_drop(table_name: t.Any, *args: t.Any, **kwargs: t.Any) -> None: + nonlocal drop_attempts + drop_attempts += 1 + if drop_attempts == 1: + raise _FakeDbError("25P02") + real_drop(table_name, *args, **kwargs) + + mocker.patch.object(copy_adapter, "drop_table", side_effect=_flaky_drop) + + with pytest.raises(RuntimeError, match="caller query failed"): + with source_queries[0]: + # Commit count is 1 from standalone staging table load + assert commit_spy.call_count == 1 + assert not copy_adapter._connection_pool.is_transaction_active + raise RuntimeError("caller query failed") + + assert drop_attempts == 2 + assert rollback_spy.call_count == 1 + assert commit_spy.call_count == 2 # retry drop_table was also committed + assert to_sql_calls(copy_adapter)[-1] == 'DROP TABLE IF EXISTS "__temp_tbl_abcdefgh"' + + # 2. If the retry drop ALSO fails while an exception is propagating, the caller's original + # exception is still preserved (never masked). + source_queries_2 = copy_adapter._df_to_source_queries(df, types, 10000, "tbl") + mocker.patch.object(copy_adapter, "drop_table", side_effect=_FakeDbError("25P02")) + + with pytest.raises(RuntimeError, match="original failure"): + with source_queries_2[0]: + raise RuntimeError("original failure") + + # 3. If no caller exception is propagating and the retry drop fails, the retry error raises. + source_queries_3 = copy_adapter._df_to_source_queries(df, types, 10000, "tbl") + with pytest.raises(_FakeDbError): + with source_queries_3[0]: + pass + + +def test_drop_copy_staging_table_non_25p02_does_not_mask_propagating_error( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + """If the connection drops mid-load (OperationalError, pgcode=None), a secondary InterfaceError + from `drop_table` during `SourceQuery.__exit__` must be logged rather than masking the root error.""" + + class _FakeOperationalError(Exception): + pass + + class _FakeInterfaceError(Exception): + pass + + df = pd.DataFrame({"a": [1, 2]}) + types = {"a": exp.DataType.build("int", dialect="postgres")} + source_queries = copy_adapter._df_to_source_queries(df, types, 10000, "tbl") + mocker.patch.object( + copy_adapter, "drop_table", side_effect=_FakeInterfaceError("cursor already closed") + ) + + with pytest.raises(_FakeOperationalError, match="SSL connection closed"): + with source_queries[0]: + raise _FakeOperationalError("SSL connection closed") + + +def test_df_ctas_comments_use_savepoint_inside_transaction( + copy_adapter: PostgresEngineAdapter, mocker: MockerFixture +): + df = pd.DataFrame({"a": [1, 2]}) + types = {"a": exp.DataType.build("int", dialect="postgres")} + real_execute = copy_adapter._execute + + def _fail_table_comment(sql: str, *args: t.Any, **kwargs: t.Any) -> None: + if sql.startswith("COMMENT ON TABLE"): + raise _FakeDbError("42501") + real_execute(sql, *args, **kwargs) + + mocker.patch.object(copy_adapter, "_execute", side_effect=_fail_table_comment) + commit_spy = mocker.spy(copy_adapter._connection_pool, "commit") + + copy_adapter.ctas( + "tbl", + df, + target_columns_to_types=types, + table_description="tbl comment", + column_descriptions={"a": "col comment"}, + ) + + sql_calls = to_sql_calls(copy_adapter) + assert "SAVEPOINT _sqlmesh_comment" in sql_calls + assert "ROLLBACK TO SAVEPOINT _sqlmesh_comment" in sql_calls + assert "RELEASE SAVEPOINT _sqlmesh_comment" in sql_calls + assert """COMMENT ON COLUMN "tbl"."a" IS 'col comment'""" in sql_calls + assert commit_spy.call_count == 1