From 110f9a7b227612f94258cd9d03470c1fa2618743 Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 10:21:05 +0800 Subject: [PATCH 1/8] fix the compatibility issue --- .../sqlalchemy_mongodb/sqlalchemy_dialect.py | 31 ++++++ pyproject.toml | 1 + requirements-test.txt | 1 + tests/conftest.py | 8 ++ tests/test_pandas_sqlalchemy.py | 103 ++++++++++++++++++ tests/test_sqlalchemy_dialect.py | 14 +++ 6 files changed, 158 insertions(+) create mode 100644 tests/test_pandas_sqlalchemy.py diff --git a/pymongosql/sqlalchemy_mongodb/sqlalchemy_dialect.py b/pymongosql/sqlalchemy_mongodb/sqlalchemy_dialect.py index 8c89cbf..16f2c06 100644 --- a/pymongosql/sqlalchemy_mongodb/sqlalchemy_dialect.py +++ b/pymongosql/sqlalchemy_mongodb/sqlalchemy_dialect.py @@ -214,6 +214,37 @@ def __getattribute__(self, name): return pymongosql return super().__getattribute__(name) + @staticmethod + def _normalize_collection_name(statement: str) -> str: + """Extract a collection name from the compiler's DROP COLLECTION placeholder.""" + collection_name = statement[len("-- DROP COLLECTION ") :].strip() + if collection_name.startswith('"') and collection_name.endswith('"'): + return collection_name[1:-1] + return collection_name + + def _handle_ddl_placeholder(self, cursor, statement: str) -> bool: + """Handle SQLAlchemy DDL placeholders without routing them through the SQL parser.""" + if statement == "-- Collection will be created on first insert": + return True + + if statement.startswith("-- DROP COLLECTION "): + cursor.connection.database.drop_collection(self._normalize_collection_name(statement)) + return True + + return False + + def do_execute(self, cursor, statement, parameters, context=None): + """Execute statements, handling MongoDB DDL placeholders directly.""" + if self._handle_ddl_placeholder(cursor, statement): + return None + return super().do_execute(cursor, statement, parameters, context=context) + + def do_execute_no_params(self, cursor, statement, context=None): + """Execute parameterless statements, handling MongoDB DDL placeholders directly.""" + if self._handle_ddl_placeholder(cursor, statement): + return None + return super().do_execute_no_params(cursor, statement, context=context) + def create_connect_args(self, url: url.URL) -> Tuple[List[Any], Dict[str, Any]]: """Create connection arguments from SQLAlchemy URL. diff --git a/pyproject.toml b/pyproject.toml index d78614b..d68a596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dev = [ "pytest-cov>=4.0.0", "flake8>=6.0.0", "flake8-pyproject>=1.2.0", + "pandas>=2.2.0", "black>=23.0.0", "isort>=5.12.0", ] diff --git a/requirements-test.txt b/requirements-test.txt index 7b57803..1f9e163 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -6,3 +6,4 @@ pytest>=7.0.0 pytest-cov>=4.0.0 flake8>=6.0.0 flake8-pyproject>=1.2.0 +pandas>=2.2.0 diff --git a/tests/conftest.py b/tests/conftest.py index dc50af4..e6a8e5f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -115,6 +115,14 @@ class Base(DeclarativeBase): Base = None Session = None +try: + import pandas as pd + + HAS_PANDAS = True +except ImportError: + pd = None + HAS_PANDAS = False + # SQLAlchemy fixtures for dialect testing if HAS_SQLALCHEMY: diff --git a/tests/test_pandas_sqlalchemy.py b/tests/test_pandas_sqlalchemy.py new file mode 100644 index 0000000..1f52351 --- /dev/null +++ b/tests/test_pandas_sqlalchemy.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +import pytest + +from tests.conftest import HAS_PANDAS, HAS_SQLALCHEMY + +if HAS_PANDAS: + import pandas as pd + + +pytestmark = pytest.mark.skipif( + not (HAS_SQLALCHEMY and HAS_PANDAS), + reason="Pandas and SQLAlchemy are required for pandas SQLAlchemy integration tests", +) + + +class TestPandasSQLAlchemy: + """Compatibility tests for pandas through the SQLAlchemy engine.""" + + TEST_COLLECTION = "test_pandas_sqlalchemy" + + @pytest.fixture(autouse=True) + def setup_teardown(self, conn): + """Keep pandas write tests isolated from shared seeded collections.""" + db = conn.database + if self.TEST_COLLECTION in db.list_collection_names(): + db.drop_collection(self.TEST_COLLECTION) + yield + if self.TEST_COLLECTION in db.list_collection_names(): + db.drop_collection(self.TEST_COLLECTION) + + def test_read_sql_returns_dataframe(self, sqlalchemy_engine): + """pandas.read_sql should load a projected query into a DataFrame.""" + query = "SELECT _id, name, age FROM users LIMIT 5" + + dataframe = pd.read_sql(query, sqlalchemy_engine) + + assert isinstance(dataframe, pd.DataFrame) + assert not dataframe.empty + assert list(dataframe.columns) == ["_id", "name", "age"] + assert len(dataframe) <= 5 + assert dataframe["_id"].notna().all() + assert dataframe["name"].notna().all() + + def test_to_sql_append_writes_rows(self, sqlalchemy_engine, conn): + """pandas should support read_sql, iloc selection, and to_sql append.""" + db = conn.database + source = pd.read_sql( + "SELECT _id, name, age, city, active FROM users LIMIT 5", + sqlalchemy_engine, + ) + selected = source.iloc[[1, 3]].copy() + selected["source_collection"] = "users" + + selected.to_sql(self.TEST_COLLECTION, sqlalchemy_engine, if_exists="append", index=False) + + docs = list( + db[self.TEST_COLLECTION] + .find({}, {"_id": 1, "name": 1, "age": 1, "active": 1, "source_collection": 1}) + .sort("name", 1) + ) + + expected = ( + selected[["_id", "name", "age", "active", "source_collection"]].sort_values("name").to_dict("records") + ) + + assert len(docs) == len(expected) + assert docs == expected + + round_trip = pd.read_sql( + f"SELECT _id, name, age, active, source_collection FROM {self.TEST_COLLECTION}", + sqlalchemy_engine, + ) + round_trip = round_trip.sort_values("name").reset_index(drop=True) + expected_frame = ( + selected[["_id", "name", "age", "active", "source_collection"]].sort_values("name").reset_index(drop=True) + ) + + assert isinstance(round_trip, pd.DataFrame) + assert list(round_trip.columns) == ["_id", "name", "age", "active", "source_collection"] + assert round_trip.to_dict("records") == expected_frame.to_dict("records") + + def test_to_sql_replace_recreates_collection(self, sqlalchemy_engine, conn): + """pandas.to_sql with replace should drop and recreate the collection transparently.""" + db = conn.database + db[self.TEST_COLLECTION].insert_one({"_id": "stale", "name": "stale"}) + + replacement = pd.read_sql( + "SELECT _id, name, age, active FROM users LIMIT 2", + sqlalchemy_engine, + ).copy() + replacement["source_collection"] = "users" + + replacement.to_sql(self.TEST_COLLECTION, sqlalchemy_engine, if_exists="replace", index=False) + + docs = list( + db[self.TEST_COLLECTION] + .find({}, {"_id": 1, "name": 1, "age": 1, "active": 1, "source_collection": 1}) + .sort("_id", 1) + ) + expected = replacement.sort_values("_id").reset_index(drop=True) + + assert len(docs) == len(expected) + assert docs == expected.to_dict("records") diff --git a/tests/test_sqlalchemy_dialect.py b/tests/test_sqlalchemy_dialect.py index a89ddbf..e2fe7b2 100644 --- a/tests/test_sqlalchemy_dialect.py +++ b/tests/test_sqlalchemy_dialect.py @@ -392,6 +392,20 @@ def test_ddl_compiler(self): drop_result = PyMongoSQLDDLCompiler.visit_drop_table(mock_compiler, drop_mock) self.assertIn("DROP COLLECTION", drop_result) + def test_ddl_placeholder_handlers(self): + """Test that SQLAlchemy DDL placeholders are handled without SQL parsing.""" + mock_cursor = Mock() + mock_cursor.connection.database.drop_collection = Mock() + + self.assertTrue( + self.dialect._handle_ddl_placeholder(mock_cursor, "-- Collection will be created on first insert") + ) + mock_cursor.connection.database.drop_collection.assert_not_called() + + handled = self.dialect._handle_ddl_placeholder(mock_cursor, "-- DROP COLLECTION test_table") + self.assertTrue(handled) + mock_cursor.connection.database.drop_collection.assert_called_once_with("test_table") + class TestSQLAlchemyIntegration(unittest.TestCase): """Integration tests for SQLAlchemy functionality.""" From 3af2b53813440dcff5d92a3593a0b42a0883d47b Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 10:32:26 +0800 Subject: [PATCH 2/8] fix pandas and sqlalchmey version compatibility issue --- .github/workflows/ci.yml | 8 +++++--- pyproject.toml | 9 ++++++++- requirements-test-sqlalchemy14.txt | 5 +++++ requirements-test-sqlalchemy2.txt | 5 +++++ requirements-test.txt | 5 ++++- 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 requirements-test-sqlalchemy14.txt create mode 100644 requirements-test-sqlalchemy2.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcb5077..d0b46e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,10 +60,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - # Install the target SQLAlchemy version for this matrix entry first to ensure - # tests run against both 1.x and 2.x series. - pip install "SQLAlchemy==${{ matrix.sqlalchemy-version }}" pip install -r requirements-test.txt + if [ "${{ matrix.sqlalchemy-version }}" = "1.4.*" ]; then + pip install -r requirements-test-sqlalchemy14.txt + else + pip install -r requirements-test-sqlalchemy2.txt + fi pip install black isort - name: Wait for MongoDB to be ready diff --git a/pyproject.toml b/pyproject.toml index d68a596..96cd9eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,12 +41,19 @@ dependencies = [ [project.optional-dependencies] retry = ["tenacity>=9.0.0"] sqlalchemy = ["sqlalchemy>=1.4.0"] +pandas_sqlalchemy14 = [ + "sqlalchemy>=1.4.0,<2.0.0", + "pandas>=2.2.0,<3.0.0", +] +pandas_sqlalchemy2 = [ + "sqlalchemy>=2.0.0,<3.0.0", + "pandas>=3.0.0,<4.0.0", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "flake8>=6.0.0", "flake8-pyproject>=1.2.0", - "pandas>=2.2.0", "black>=23.0.0", "isort>=5.12.0", ] diff --git a/requirements-test-sqlalchemy14.txt b/requirements-test-sqlalchemy14.txt new file mode 100644 index 0000000..ca6a9bb --- /dev/null +++ b/requirements-test-sqlalchemy14.txt @@ -0,0 +1,5 @@ +-r requirements-test.txt + +# Supported compatibility pair for the SQLAlchemy 1.4 test lane. +SQLAlchemy>=1.4.0,<2.0.0 +pandas>=2.2.0,<3.0.0 \ No newline at end of file diff --git a/requirements-test-sqlalchemy2.txt b/requirements-test-sqlalchemy2.txt new file mode 100644 index 0000000..ae3375f --- /dev/null +++ b/requirements-test-sqlalchemy2.txt @@ -0,0 +1,5 @@ +-r requirements-test.txt + +# Supported compatibility pair for the SQLAlchemy 2.x test lane. +SQLAlchemy>=2.0.0,<3.0.0 +pandas>=3.0.0,<4.0.0 \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index 1f9e163..b353f4a 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,9 +1,12 @@ -r requirements.txt -r requirements-optional.txt +# Shared test tooling only. Version-coupled pandas/SQLAlchemy combinations live in +# requirements-test-sqlalchemy14.txt and requirements-test-sqlalchemy2.txt. +# This keeps local and CI installs explicit about which compatibility pair they use. + # Test dependencies pytest>=7.0.0 pytest-cov>=4.0.0 flake8>=6.0.0 flake8-pyproject>=1.2.0 -pandas>=2.2.0 From 0cc3d15a4838f4266f8fedd69755a0b553e296ec Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 15:37:52 +0800 Subject: [PATCH 3/8] patch pandas and python version compatibility --- .github/workflows/ci.yml | 2 +- pyproject.toml | 3 ++- requirements-test-sqlalchemy2.txt | 7 +++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0b46e4..6442b87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/pip - key: ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip-${{ hashFiles('**/requirements-test.txt', 'pyproject.toml') }} + key: ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip-${{ hashFiles('**/requirements-test.txt', '**/requirements-test-sqlalchemy14.txt', '**/requirements-test-sqlalchemy2.txt', 'pyproject.toml') }} restore-keys: | ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip- diff --git a/pyproject.toml b/pyproject.toml index 96cd9eb..bc17099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,8 @@ pandas_sqlalchemy14 = [ ] pandas_sqlalchemy2 = [ "sqlalchemy>=2.0.0,<3.0.0", - "pandas>=3.0.0,<4.0.0", + "pandas>=3.0.0,<4.0.0; python_version >= '3.10'", + "pandas>=2.2.0,<3.0.0; python_version < '3.10'", ] dev = [ "pytest>=7.0.0", diff --git a/requirements-test-sqlalchemy2.txt b/requirements-test-sqlalchemy2.txt index ae3375f..cc4ac08 100644 --- a/requirements-test-sqlalchemy2.txt +++ b/requirements-test-sqlalchemy2.txt @@ -1,5 +1,8 @@ -r requirements-test.txt -# Supported compatibility pair for the SQLAlchemy 2.x test lane. +# Supported compatibility for the SQLAlchemy 2.x test lane. SQLAlchemy>=2.0.0,<3.0.0 -pandas>=3.0.0,<4.0.0 \ No newline at end of file +# pandas 3 is not available on some older Python versions (e.g. 3.9), +# so keep a resolver-safe fallback for those interpreters. +pandas>=3.0.0,<4.0.0; python_version >= "3.10" +pandas>=2.2.0,<3.0.0; python_version < "3.10" From 41b4b307b54cc24f2ed84a3696d03eb2254b9e15 Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 17:27:08 +0800 Subject: [PATCH 4/8] Fix sqlalchemy 1.4 and pandas dependency issue --- pyproject.toml | 2 +- requirements-test-sqlalchemy14.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bc17099..aa5cc71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ retry = ["tenacity>=9.0.0"] sqlalchemy = ["sqlalchemy>=1.4.0"] pandas_sqlalchemy14 = [ "sqlalchemy>=1.4.0,<2.0.0", - "pandas>=2.2.0,<3.0.0", + "pandas>=2.1.0,<2.2.0", ] pandas_sqlalchemy2 = [ "sqlalchemy>=2.0.0,<3.0.0", diff --git a/requirements-test-sqlalchemy14.txt b/requirements-test-sqlalchemy14.txt index ca6a9bb..c505295 100644 --- a/requirements-test-sqlalchemy14.txt +++ b/requirements-test-sqlalchemy14.txt @@ -2,4 +2,4 @@ # Supported compatibility pair for the SQLAlchemy 1.4 test lane. SQLAlchemy>=1.4.0,<2.0.0 -pandas>=2.2.0,<3.0.0 \ No newline at end of file +pandas>=2.1.0,<2.2.0 \ No newline at end of file From df30effb77f80a64ed32466975c7c9d6c969465c Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 17:39:40 +0800 Subject: [PATCH 5/8] Fix pandas version issue --- pyproject.toml | 4 ++-- requirements-test-sqlalchemy2.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa5cc71..f7ac2c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ pandas_sqlalchemy14 = [ ] pandas_sqlalchemy2 = [ "sqlalchemy>=2.0.0,<3.0.0", - "pandas>=3.0.0,<4.0.0; python_version >= '3.10'", - "pandas>=2.2.0,<3.0.0; python_version < '3.10'", + "pandas>=3.0.0,<4.0.0; python_version >= '3.11'", + "pandas>=2.2.0,<3.0.0; python_version < '3.11'", ] dev = [ "pytest>=7.0.0", diff --git a/requirements-test-sqlalchemy2.txt b/requirements-test-sqlalchemy2.txt index cc4ac08..0e3d23a 100644 --- a/requirements-test-sqlalchemy2.txt +++ b/requirements-test-sqlalchemy2.txt @@ -2,7 +2,7 @@ # Supported compatibility for the SQLAlchemy 2.x test lane. SQLAlchemy>=2.0.0,<3.0.0 -# pandas 3 is not available on some older Python versions (e.g. 3.9), +# pandas 3 is not available on Python < 3.11, # so keep a resolver-safe fallback for those interpreters. -pandas>=3.0.0,<4.0.0; python_version >= "3.10" -pandas>=2.2.0,<3.0.0; python_version < "3.10" +pandas>=3.0.0,<4.0.0; python_version >= "3.11" +pandas>=2.2.0,<3.0.0; python_version < "3.11" From aa5367957d30a345800881db927288c03af59c81 Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 18:09:35 +0800 Subject: [PATCH 6/8] Fix dependency version issues --- pyproject.toml | 2 +- requirements-test-sqlalchemy14.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f7ac2c7..8e5f8ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ retry = ["tenacity>=9.0.0"] sqlalchemy = ["sqlalchemy>=1.4.0"] pandas_sqlalchemy14 = [ "sqlalchemy>=1.4.0,<2.0.0", - "pandas>=2.1.0,<2.2.0", + "pandas>=2.1.0,<2.2.0; python_version < '3.13'", ] pandas_sqlalchemy2 = [ "sqlalchemy>=2.0.0,<3.0.0", diff --git a/requirements-test-sqlalchemy14.txt b/requirements-test-sqlalchemy14.txt index c505295..d43e243 100644 --- a/requirements-test-sqlalchemy14.txt +++ b/requirements-test-sqlalchemy14.txt @@ -2,4 +2,6 @@ # Supported compatibility pair for the SQLAlchemy 1.4 test lane. SQLAlchemy>=1.4.0,<2.0.0 -pandas>=2.1.0,<2.2.0 \ No newline at end of file +# pandas>=2.2 has SQLAlchemy 1.4 engine compatibility issues in read_sql, +# while pandas 2.1.x does not provide wheels for Python >=3.13. +pandas>=2.1.0,<2.2.0; python_version < "3.13" \ No newline at end of file From 8ceeb64bb43e1700b3495b004b5232b928323dbb Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 18:15:43 +0800 Subject: [PATCH 7/8] Remove nodejs warn --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6442b87..00d8edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,6 @@ on: branches: [ main ] workflow_call: -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - jobs: test: runs-on: ubuntu-latest From bc05ceeea26f3f55010e41df681b8d396192f6f2 Mon Sep 17 00:00:00 2001 From: Peng Ren Date: Sun, 5 Jul 2026 18:24:38 +0800 Subject: [PATCH 8/8] Bump version to 0.7.2 --- pymongosql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongosql/__init__.py b/pymongosql/__init__.py index 5f318fc..ceb89c0 100644 --- a/pymongosql/__init__.py +++ b/pymongosql/__init__.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from .connection import Connection -__version__: str = "0.7.1" +__version__: str = "0.7.2" # Globals https://www.python.org/dev/peps/pep-0249/#globals apilevel: str = "2.0"