diff --git a/README.md b/README.md index 4d6a9f39..bf508d74 100644 --- a/README.md +++ b/README.md @@ -57,29 +57,29 @@ The python package can be installed using [pip](https://pip.pypa.io/en/stable/ge pip install wrangles ``` -This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel and CSV file I/O, HTTP connectors, and SQLite. +This installs the core package, which covers the vast majority of use cases: all data wrangles, recipe execution, Excel/CSV/JSON file I/O, HTTP connectors, SQLite, DuckDB, MongoDB, AWS S3, Salesforce, SFTP/SSH, notifications, and the OpenAI/Gemini/SerpAPI integrations. -### Optional dependencies +### Full install (adds SQL Server, Access, PostgreSQL, MySQL, and Parquet files) -Connectors for databases, cloud storage, and external services require additional packages. Install only the ones you need: +A handful of connectors depend on heavier, more platform-specific packages (SQL database drivers) and aren't included by default: -| Capability | Install | +| Capability | Package(s) | |---|---| -| Microsoft SQL Server | `pip install pymssql sqlalchemy` | -| Microsoft Access | `pip install pyodbc` | -| DuckDB | `pip install duckdb` | -| PostgreSQL | `pip install psycopg2-binary sqlalchemy` | -| MySQL | `pip install pymysql sqlalchemy` | -| MongoDB | `pip install pymongo` | -| AWS S3 | `pip install boto3` | -| Salesforce | `pip install simple-salesforce` | -| SFTP / SSH | `pip install fabric` | -| Notifications | `pip install apprise` | -| OpenAI SDK | `pip install openai` | -| Google Gemini | `pip install google-generativeai` | -| SerpAPI (web search) | `pip install serpapi` | - -> If a connector is used without its required package installed, Wrangles will raise a clear `ImportError` with the exact `pip install` command needed. +| Microsoft SQL Server | `pymssql`, `sqlalchemy` | +| Microsoft Access | `pyodbc` | +| PostgreSQL | `psycopg2-binary`, `sqlalchemy` | +| MySQL | `sqlalchemy` (`pymysql` itself is already in the core install) | +| Parquet files | `pyarrow` | + +Install just the ones you need (e.g. `pip install pyodbc` for Access only), or install all of them at once: + +```shell +pip install sqlalchemy pyodbc pymssql psycopg2-binary pyarrow +``` + +If you're working from a clone of this repository (e.g. for local development), `pip install -r requirements-full.txt` does the same thing, plus everything from the core install. + +> If a connector is used without its required package installed, Wrangles will raise a clear `ImportError` with the exact `pip install` command needed, so there's no harm in starting with the core install and adding packages only as you need them. Once installed, import the package into your code. ```python diff --git a/requirements-full.txt b/requirements-full.txt index 15406ae9..ba0e521a 100644 --- a/requirements-full.txt +++ b/requirements-full.txt @@ -8,7 +8,9 @@ # SQL databases sqlalchemy>=2.0,<3.0 -duckdb>=1.0.0 pyodbc>=5.0.0 pymssql>=2.3.3 psycopg2-binary>=2.9.10 + +# Parquet file support (file connector) +pyarrow diff --git a/requirements.txt b/requirements.txt index f0b013c5..137c1d08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ pandas>=2.0,<3.0 numpy numexpr polars==1.33.0 -pyarrow # Fuzzy text similarity rapidfuzz>=3.0,<4.0 @@ -31,6 +30,9 @@ xlsxwriter # Cloud & storage boto3 +# Embedded database +duckdb>=1.0.0 + # NoSQL pymongo diff --git a/wrangles/connectors/file.py b/wrangles/connectors/file.py index f05b701c..ed0170a1 100644 --- a/wrangles/connectors/file.py +++ b/wrangles/connectors/file.py @@ -4,18 +4,22 @@ Supports Excel, CSV, JSON, JSONL and Parquet files. """ import pandas as _pd -import pyarrow as _pa -import pyarrow.parquet as _pq import logging as _logging from typing import Union as _Union from io import BytesIO as _BytesIO import base64 as _base64 import os as _os import re as _re +from ..utils import ( + wildcard_expansion as _wildcard_expansion, + LazyLoader as _LazyLoader, +) from openpyxl.utils.escape import unescape as _unescape -from ..utils import wildcard_expansion as _wildcard_expansion -from ._formatting import default_file_format as _default_file_format from ._formatting import file_format as _file_format +from ._formatting import default_file_format as _default_file_format + +_pa = _LazyLoader('pyarrow') +_pq = _LazyLoader('pyarrow.parquet') _schema = {}