From a87d43b8b2de3d00991cd24424456c8baf0f4a0b Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Tue, 25 Aug 2026 18:21:27 +0300 Subject: [PATCH 1/5] Make pyarrow an optional dependency, lazy-loaded in the file connector pyarrow is only needed for Parquet support in the file connector, but was imported eagerly, making it a hard dependency for importing wrangles at all. Load it lazily like every other connector-specific dependency (duckdb, pyodbc, boto3, etc.) and move it to requirements-full.txt. --- requirements-full.txt | 3 +++ requirements.txt | 1 - wrangles/connectors/file.py | 10 +++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/requirements-full.txt b/requirements-full.txt index 15406ae9..a61591da 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -12,3 +12,6 @@ duckdb>=1.0.0 pyodbc>=5.0.0 pymssql>=2.3.3 psycopg2-binary>=2.9.10 + +# Parquet file support (file connector) +pyarrow diff --git a/requirements.txt b/requirements.txt index 546d3c9e..68c95def 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ pandas>=2.0,<3.0 numpy numexpr polars==1.33.0 -pyarrow # Recipe engine & templating jinja2 diff --git a/wrangles/connectors/file.py b/wrangles/connectors/file.py index 370e0ac8..8dfffed1 100644 --- a/wrangles/connectors/file.py +++ b/wrangles/connectors/file.py @@ -5,17 +5,21 @@ """ from openpyxl.styles import Alignment as _Alignment import pandas as _pd -import pyarrow as _pa -import pyarrow.parquet as _pq import logging as _logging from typing import Union as _Union from io import BytesIO as _BytesIO import base64 as _base64 import os as _os import re as _re -from ..utils import wildcard_expansion as _wildcard_expansion +from ..utils import ( + wildcard_expansion as _wildcard_expansion, + LazyLoader as _LazyLoader, +) from ._formatting import file_format as _file_format +_pa = _LazyLoader('pyarrow') +_pq = _LazyLoader('pyarrow.parquet') + _schema = {} From a1edb48d8324a0f424901b80bd41c0d554c2a479 Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Wed, 26 Aug 2026 11:23:01 +0300 Subject: [PATCH 2/5] Update test_recipes.py --- tests/recipes/test_recipes.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/recipes/test_recipes.py b/tests/recipes/test_recipes.py index b2e09ef5..172d9fe5 100644 --- a/tests/recipes/test_recipes.py +++ b/tests/recipes/test_recipes.py @@ -131,7 +131,7 @@ def test_recipe_by_production_version(): """ Test running a recipe using a model ID and production version """ - df = wrangles.recipe.run("a6bac9e7-2388-4347") + df = wrangles.recipe.run("e954717c-fb9c-4c47") assert ( len(df) == 20 and list(df.columns) == ["header"] @@ -154,10 +154,10 @@ def test_recipe_by_production_semantic_version(mocker): return_value={"recipe": "{}"} ) - wrangles.recipe.run("a6bac9e7-2388-4347:production") + wrangles.recipe.run("e954717c-fb9c-4c47:production") model_content.assert_called_once_with( - "a6bac9e7-2388-4347", + "e954717c-fb9c-4c47", "production-version-id" ) @@ -178,9 +178,9 @@ def test_recipe_by_production_semantic_version_falls_back_to_latest( return_value={"recipe": "{}"} ) - wrangles.recipe.run("a6bac9e7-2388-4347:production") + wrangles.recipe.run("e954717c-fb9c-4c47:production") - model_content.assert_called_once_with("a6bac9e7-2388-4347", None) + model_content.assert_called_once_with("e954717c-fb9c-4c47", None) assert "No production version exists, defaulting to latest version" in caplog.text @@ -188,7 +188,7 @@ def test_recipe_by_version_latest(): """ Test running a recipe using a model ID and latest version """ - df = wrangles.recipe.run("a6bac9e7-2388-4347:latest") + df = wrangles.recipe.run("e954717c-fb9c-4c47:latest") assert ( len(df) == 10 and list(df.columns) == ["header"] @@ -198,7 +198,7 @@ def test_recipe_by_latest_version(): """ Test running a recipe using a model ID and latest version """ - df = wrangles.recipe.run("02fc0c63-1294-415b") + df = wrangles.recipe.run("1b41d016-7129-4b66") assert ( len(df) == 15 and list(df.columns) == ["header"] From 9d59adb7098ee256e1a47b79baf35e779f9c6f1e Mon Sep 17 00:00:00 2001 From: Mariia Borodii Date: Thu, 27 Aug 2026 06:57:06 +0000 Subject: [PATCH 3/5] Fix missing imports in file connector after merge with main The merge of main dropped the _unescape and _default_file_format imports in wrangles/connectors/file.py, causing NameError failures across the Excel/S3/file recipe test suites. --- wrangles/connectors/file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wrangles/connectors/file.py b/wrangles/connectors/file.py index 7bd17a6c..ed0170a1 100644 --- a/wrangles/connectors/file.py +++ b/wrangles/connectors/file.py @@ -14,7 +14,9 @@ wildcard_expansion as _wildcard_expansion, LazyLoader as _LazyLoader, ) +from openpyxl.utils.escape import unescape as _unescape from ._formatting import file_format as _file_format +from ._formatting import default_file_format as _default_file_format _pa = _LazyLoader('pyarrow') _pq = _LazyLoader('pyarrow.parquet') From 78a0cb93bb69252c5d223a544d3bf52b82b7934e Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Thu, 27 Aug 2026 14:10:20 +0300 Subject: [PATCH 4/5] Update README.md --- README.md | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4d6a9f39..27c039d7 100644 --- a/README.md +++ b/README.md @@ -57,29 +57,30 @@ The python package can be installed using [pip](https://pip.pypa.io/en/stable/ge pip install wrangles ``` -This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel and CSV file I/O, HTTP connectors, and SQLite. +This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel/CSV/JSON file I/O, HTTP connectors, SQLite, MongoDB, AWS S3, Salesforce, SFTP/SSH, notifications, and the OpenAI/Gemini/SerpAPI integrations. -### Optional dependencies +### Full install (adds SQL databases and Parquet files) -Connectors for databases, cloud storage, and external services require additional packages. Install only the ones you need: +A handful of connectors depend on heavier, more platform-specific packages (SQL database drivers) and aren't included by default: -| Capability | Install | +| Capability | Package(s) | |---|---| -| Microsoft SQL Server | `pip install pymssql sqlalchemy` | -| Microsoft Access | `pip install pyodbc` | -| DuckDB | `pip install duckdb` | -| PostgreSQL | `pip install psycopg2-binary sqlalchemy` | -| MySQL | `pip install pymysql sqlalchemy` | -| MongoDB | `pip install pymongo` | -| AWS S3 | `pip install boto3` | -| Salesforce | `pip install simple-salesforce` | -| SFTP / SSH | `pip install fabric` | -| Notifications | `pip install apprise` | -| OpenAI SDK | `pip install openai` | -| Google Gemini | `pip install google-generativeai` | -| SerpAPI (web search) | `pip install serpapi` | - -> If a connector is used without its required package installed, Wrangles will raise a clear `ImportError` with the exact `pip install` command needed. +| Microsoft SQL Server | `pymssql`, `sqlalchemy` | +| Microsoft Access | `pyodbc` | +| DuckDB | `duckdb` | +| PostgreSQL | `psycopg2-binary`, `sqlalchemy` | +| MySQL | `sqlalchemy` (`pymysql` itself is already in the core install) | +| Parquet files | `pyarrow` | + +Install just the ones you need (e.g. `pip install duckdb` for DuckDB only), or install all of them at once: + +```shell +pip install sqlalchemy duckdb pyodbc pymssql psycopg2-binary pyarrow +``` + +If you're working from a clone of this repository (e.g. for local development), `pip install -r requirements-full.txt` does the same thing, plus everything from the core install. + +> If a connector is used without its required package installed, Wrangles will raise a clear `ImportError` with the exact `pip install` command needed, so there's no harm in starting with the core install and adding packages only as you need them. Once installed, import the package into your code. ```python From ba737e72fe92c2a1f0b289f2ecd68ec441f57f39 Mon Sep 17 00:00:00 2001 From: mborodii-prog Date: Thu, 27 Aug 2026 14:43:36 +0300 Subject: [PATCH 5/5] add duckdb to requirements.txt --- README.md | 9 ++++----- requirements-full.txt | 1 - requirements.txt | 3 +++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 27c039d7..bf508d74 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ The python package can be installed using [pip](https://pip.pypa.io/en/stable/ge pip install wrangles ``` -This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel/CSV/JSON file I/O, HTTP connectors, SQLite, MongoDB, AWS S3, Salesforce, SFTP/SSH, notifications, and the OpenAI/Gemini/SerpAPI integrations. +This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel/CSV/JSON file I/O, HTTP connectors, SQLite, DuckDB, MongoDB, AWS S3, Salesforce, SFTP/SSH, notifications, and the OpenAI/Gemini/SerpAPI integrations. -### Full install (adds SQL databases and Parquet files) +### Full install (adds SQL Server, Access, PostgreSQL, MySQL, and Parquet files) A handful of connectors depend on heavier, more platform-specific packages (SQL database drivers) and aren't included by default: @@ -67,15 +67,14 @@ A handful of connectors depend on heavier, more platform-specific packages (SQL |---|---| | Microsoft SQL Server | `pymssql`, `sqlalchemy` | | Microsoft Access | `pyodbc` | -| DuckDB | `duckdb` | | PostgreSQL | `psycopg2-binary`, `sqlalchemy` | | MySQL | `sqlalchemy` (`pymysql` itself is already in the core install) | | Parquet files | `pyarrow` | -Install just the ones you need (e.g. `pip install duckdb` for DuckDB only), or install all of them at once: +Install just the ones you need (e.g. `pip install pyodbc` for Access only), or install all of them at once: ```shell -pip install sqlalchemy duckdb pyodbc pymssql psycopg2-binary pyarrow +pip install sqlalchemy pyodbc pymssql psycopg2-binary pyarrow ``` If you're working from a clone of this repository (e.g. for local development), `pip install -r requirements-full.txt` does the same thing, plus everything from the core install. diff --git a/requirements-full.txt b/requirements-full.txt index a61591da..ba0e521a 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,7 +8,6 @@ # SQL databases sqlalchemy>=2.0,<3.0 -duckdb>=1.0.0 pyodbc>=5.0.0 pymssql>=2.3.3 psycopg2-binary>=2.9.10 diff --git a/requirements.txt b/requirements.txt index b511e759..137c1d08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,9 @@ xlsxwriter # Cloud & storage boto3 +# Embedded database +duckdb>=1.0.0 + # NoSQL pymongo