Feat/pandas compatibility

.github/workflows/ci.yml

@@ -7,9 +7,6 @@ on:
     branches: [ main ]
   workflow_call:
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
-
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -46,7 +43,7 @@ jobs:
       uses: actions/cache@v5
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip-${{ hashFiles('**/requirements-test.txt', 'pyproject.toml') }}
+        key: ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip-${{ hashFiles('**/requirements-test.txt', '**/requirements-test-sqlalchemy14.txt', '**/requirements-test-sqlalchemy2.txt', 'pyproject.toml') }}
         restore-keys: |
           ${{ runner.os }}-py${{ matrix.python-version }}-mongo${{ matrix.mongodb-version }}-sqlalchemy-${{ matrix.sqlalchemy-version }}-pip-
 
@@ -60,10 +57,12 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        # Install the target SQLAlchemy version for this matrix entry first to ensure
-        # tests run against both 1.x and 2.x series.
-        pip install "SQLAlchemy==${{ matrix.sqlalchemy-version }}"
         pip install -r requirements-test.txt
+        if [ "${{ matrix.sqlalchemy-version }}" = "1.4.*" ]; then
+          pip install -r requirements-test-sqlalchemy14.txt
+        else
+          pip install -r requirements-test-sqlalchemy2.txt
+        fi
         pip install black isort
 
     - name: Wait for MongoDB to be ready

pymongosql/__init__.py

@@ -6,7 +6,7 @@
 if TYPE_CHECKING:
     from .connection import Connection
 
-__version__: str = "0.7.1"
+__version__: str = "0.7.2"
 
 # Globals https://www.python.org/dev/peps/pep-0249/#globals
 apilevel: str = "2.0"

pymongosql/sqlalchemy_mongodb/sqlalchemy_dialect.py

@@ -214,6 +214,37 @@ def __getattribute__(self, name):
             return pymongosql
         return super().__getattribute__(name)
 
+    @staticmethod
+    def _normalize_collection_name(statement: str) -> str:
+        """Extract a collection name from the compiler's DROP COLLECTION placeholder."""
+        collection_name = statement[len("-- DROP COLLECTION ") :].strip()
+        if collection_name.startswith('"') and collection_name.endswith('"'):
+            return collection_name[1:-1]
+        return collection_name
+
+    def _handle_ddl_placeholder(self, cursor, statement: str) -> bool:
+        """Handle SQLAlchemy DDL placeholders without routing them through the SQL parser."""
+        if statement == "-- Collection will be created on first insert":
+            return True
+
+        if statement.startswith("-- DROP COLLECTION "):
+            cursor.connection.database.drop_collection(self._normalize_collection_name(statement))
+            return True
+
+        return False
+
+    def do_execute(self, cursor, statement, parameters, context=None):
+        """Execute statements, handling MongoDB DDL placeholders directly."""
+        if self._handle_ddl_placeholder(cursor, statement):
+            return None
+        return super().do_execute(cursor, statement, parameters, context=context)
+
+    def do_execute_no_params(self, cursor, statement, context=None):
+        """Execute parameterless statements, handling MongoDB DDL placeholders directly."""
+        if self._handle_ddl_placeholder(cursor, statement):
+            return None
+        return super().do_execute_no_params(cursor, statement, context=context)
+
     def create_connect_args(self, url: url.URL) -> Tuple[List[Any], Dict[str, Any]]:
         """Create connection arguments from SQLAlchemy URL.
 

pyproject.toml

@@ -41,6 +41,15 @@ dependencies = [
 [project.optional-dependencies]
 retry = ["tenacity>=9.0.0"]
 sqlalchemy = ["sqlalchemy>=1.4.0"]
+pandas_sqlalchemy14 = [
+    "sqlalchemy>=1.4.0,<2.0.0",
+    "pandas>=2.1.0,<2.2.0; python_version < '3.13'",
+]
+pandas_sqlalchemy2 = [
+    "sqlalchemy>=2.0.0,<3.0.0",
+    "pandas>=3.0.0,<4.0.0; python_version >= '3.11'",
+    "pandas>=2.2.0,<3.0.0; python_version < '3.11'",
+]
 dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0", 

requirements-test-sqlalchemy14.txt

@@ -0,0 +1,7 @@
+-r requirements-test.txt
+
+# Supported compatibility pair for the SQLAlchemy 1.4 test lane.
+SQLAlchemy>=1.4.0,<2.0.0
+# pandas>=2.2 has SQLAlchemy 1.4 engine compatibility issues in read_sql,
+# while pandas 2.1.x does not provide wheels for Python >=3.13.
+pandas>=2.1.0,<2.2.0; python_version < "3.13"

requirements-test-sqlalchemy2.txt

@@ -0,0 +1,8 @@
+-r requirements-test.txt
+
+# Supported compatibility for the SQLAlchemy 2.x test lane.
+SQLAlchemy>=2.0.0,<3.0.0
+# pandas 3 is not available on Python < 3.11,
+# so keep a resolver-safe fallback for those interpreters.
+pandas>=3.0.0,<4.0.0; python_version >= "3.11"
+pandas>=2.2.0,<3.0.0; python_version < "3.11"

requirements-test.txt

@@ -1,6 +1,10 @@
 -r requirements.txt
 -r requirements-optional.txt
 
+# Shared test tooling only. Version-coupled pandas/SQLAlchemy combinations live in
+# requirements-test-sqlalchemy14.txt and requirements-test-sqlalchemy2.txt.
+# This keeps local and CI installs explicit about which compatibility pair they use.
+
 # Test dependencies
 pytest>=7.0.0
 pytest-cov>=4.0.0

tests/conftest.py

@@ -115,6 +115,14 @@ class Base(DeclarativeBase):
     Base = None
     Session = None
 
+try:
+    import pandas as pd
+
+    HAS_PANDAS = True
+except ImportError:
+    pd = None
+    HAS_PANDAS = False
+
 # SQLAlchemy fixtures for dialect testing
 if HAS_SQLALCHEMY:
 

tests/test_pandas_sqlalchemy.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+import pytest
+
+from tests.conftest import HAS_PANDAS, HAS_SQLALCHEMY
+
+if HAS_PANDAS:
+    import pandas as pd
+
+
+pytestmark = pytest.mark.skipif(
+    not (HAS_SQLALCHEMY and HAS_PANDAS),
+    reason="Pandas and SQLAlchemy are required for pandas SQLAlchemy integration tests",
+)
+
+
+class TestPandasSQLAlchemy:
+    """Compatibility tests for pandas through the SQLAlchemy engine."""
+
+    TEST_COLLECTION = "test_pandas_sqlalchemy"
+
+    @pytest.fixture(autouse=True)
+    def setup_teardown(self, conn):
+        """Keep pandas write tests isolated from shared seeded collections."""
+        db = conn.database
+        if self.TEST_COLLECTION in db.list_collection_names():
+            db.drop_collection(self.TEST_COLLECTION)
+        yield
+        if self.TEST_COLLECTION in db.list_collection_names():
+            db.drop_collection(self.TEST_COLLECTION)
+
+    def test_read_sql_returns_dataframe(self, sqlalchemy_engine):
+        """pandas.read_sql should load a projected query into a DataFrame."""
+        query = "SELECT _id, name, age FROM users LIMIT 5"
+
+        dataframe = pd.read_sql(query, sqlalchemy_engine)
+
+        assert isinstance(dataframe, pd.DataFrame)
+        assert not dataframe.empty
+        assert list(dataframe.columns) == ["_id", "name", "age"]
+        assert len(dataframe) <= 5
+        assert dataframe["_id"].notna().all()
+        assert dataframe["name"].notna().all()
+
+    def test_to_sql_append_writes_rows(self, sqlalchemy_engine, conn):
+        """pandas should support read_sql, iloc selection, and to_sql append."""
+        db = conn.database
+        source = pd.read_sql(
+            "SELECT _id, name, age, city, active FROM users LIMIT 5",
+            sqlalchemy_engine,
+        )
+        selected = source.iloc[[1, 3]].copy()
+        selected["source_collection"] = "users"
+
+        selected.to_sql(self.TEST_COLLECTION, sqlalchemy_engine, if_exists="append", index=False)
+
+        docs = list(
+            db[self.TEST_COLLECTION]
+            .find({}, {"_id": 1, "name": 1, "age": 1, "active": 1, "source_collection": 1})
+            .sort("name", 1)
+        )
+
+        expected = (
+            selected[["_id", "name", "age", "active", "source_collection"]].sort_values("name").to_dict("records")
+        )
+
+        assert len(docs) == len(expected)
+        assert docs == expected
+
+        round_trip = pd.read_sql(
+            f"SELECT _id, name, age, active, source_collection FROM {self.TEST_COLLECTION}",
+            sqlalchemy_engine,
+        )
+        round_trip = round_trip.sort_values("name").reset_index(drop=True)
+        expected_frame = (
+            selected[["_id", "name", "age", "active", "source_collection"]].sort_values("name").reset_index(drop=True)
+        )
+
+        assert isinstance(round_trip, pd.DataFrame)
+        assert list(round_trip.columns) == ["_id", "name", "age", "active", "source_collection"]
+        assert round_trip.to_dict("records") == expected_frame.to_dict("records")
+
+    def test_to_sql_replace_recreates_collection(self, sqlalchemy_engine, conn):
+        """pandas.to_sql with replace should drop and recreate the collection transparently."""
+        db = conn.database
+        db[self.TEST_COLLECTION].insert_one({"_id": "stale", "name": "stale"})
+
+        replacement = pd.read_sql(
+            "SELECT _id, name, age, active FROM users LIMIT 2",
+            sqlalchemy_engine,
+        ).copy()
+        replacement["source_collection"] = "users"
+
+        replacement.to_sql(self.TEST_COLLECTION, sqlalchemy_engine, if_exists="replace", index=False)
+
+        docs = list(
+            db[self.TEST_COLLECTION]
+            .find({}, {"_id": 1, "name": 1, "age": 1, "active": 1, "source_collection": 1})
+            .sort("_id", 1)
+        )
+        expected = replacement.sort_values("_id").reset_index(drop=True)
+
+        assert len(docs) == len(expected)
+        assert docs == expected.to_dict("records")

tests/test_sqlalchemy_dialect.py

@@ -392,6 +392,20 @@ def test_ddl_compiler(self):
         drop_result = PyMongoSQLDDLCompiler.visit_drop_table(mock_compiler, drop_mock)
         self.assertIn("DROP COLLECTION", drop_result)
 
+    def test_ddl_placeholder_handlers(self):
+        """Test that SQLAlchemy DDL placeholders are handled without SQL parsing."""
+        mock_cursor = Mock()
+        mock_cursor.connection.database.drop_collection = Mock()
+
+        self.assertTrue(
+            self.dialect._handle_ddl_placeholder(mock_cursor, "-- Collection will be created on first insert")
+        )
+        mock_cursor.connection.database.drop_collection.assert_not_called()
+
+        handled = self.dialect._handle_ddl_placeholder(mock_cursor, "-- DROP COLLECTION test_table")
+        self.assertTrue(handled)
+        mock_cursor.connection.database.drop_collection.assert_called_once_with("test_table")
+
 
 class TestSQLAlchemyIntegration(unittest.TestCase):
     """Integration tests for SQLAlchemy functionality."""