From 49fada715bb838332376dc186a3755619123cc5f Mon Sep 17 00:00:00 2001 From: Flavio Hafner <55716026+f-hafner@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:25:52 +0200 Subject: [PATCH 1/3] Update download script --- ny-taxi/download.py | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/ny-taxi/download.py b/ny-taxi/download.py index 28384df..0dc4579 100644 --- a/ny-taxi/download.py +++ b/ny-taxi/download.py @@ -1,17 +1,28 @@ +"""Script to downlad NY Taxi data. + +- Uses `pooch` to manage and download files from DATA_BASE_URL +- TRIP_DATA_NAME is the name of the subdirectory where the parquet +files are stored. +""" + import pooch from pathlib import Path +DATA_BASE_URL = "https://d37ci6vzurychx.cloudfront.net/trip-data/" +TRIP_DATA_NAME = "trip-data" +REGISTRY_FILE = f"{TRIP_DATA_NAME}-registry.txt" + def taxi_filename(year: int = 2025, month: int = 1): return f"yellow_tripdata_{year:04}-{month:02}.parquet" def bootstrap_taxi_data( - path: Path = Path() / "data" / "trip-data", year: int = 2025 + path: Path, year: int = 2025 ) -> pooch.Pooch: return pooch.create( path=path, - base_url="https://d37ci6vzurychx.cloudfront.net/trip-data/", + base_url=DATA_BASE_URL, registry={ taxi_filename(year=year, month=month): None for month in range(1, 13) }, @@ -25,19 +36,17 @@ def download_all(p: pooch.Pooch): def make_registry(p: pooch.Pooch): path = Path(p.path) - registry_name = path.name + "-registry.txt" - pooch.make_registry(path, path.parent / registry_name) + pooch.make_registry(path, path.parent / REGISTRY_FILE) def taxi_data( - data_path: Path = Path() / "data", name: str = "trip-data" + data_path: Path, ) -> pooch.Pooch: - path = data_path / name - registry = data_path / (name + "-registry.txt") + registry = data_path / REGISTRY_FILE p = pooch.create( - path=path, - base_url="https://d37ci6vzurychx.cloudfront.net/trip-data/", + path=data_path / TRIP_DATA_NAME, + base_url=DATA_BASE_URL, registry=None, ) p.load_registry(registry) @@ -45,17 +54,19 @@ def taxi_data( def main(): - registry = Path() / "data" / "trip-data-registry.txt" + nyt_path = Path(__file__).parent + data_dir = "data" + registry = nyt_path / data_dir / REGISTRY_FILE if registry.exists(): - p = taxi_data() + p = taxi_data(data_path = nyt_path / data_dir) download_all(p) else: - p = bootstrap_taxi_data() + p = bootstrap_taxi_data(nyt_path / data_dir / TRIP_DATA_NAME) download_all(p) make_registry(p) if __name__ == "__main__": main() - +# TODO: add unit test that data can be read From 1558061ecaae58787c13bbd5ae863715b6601980 Mon Sep 17 00:00:00 2001 From: Flavio Hafner <55716026+f-hafner@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:04:55 +0200 Subject: [PATCH 2/3] Add test for NY taxi --- test/test_nytaxi.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 test/test_nytaxi.py diff --git a/test/test_nytaxi.py b/test/test_nytaxi.py new file mode 100644 index 0000000..cc0066e --- /dev/null +++ b/test/test_nytaxi.py @@ -0,0 +1,26 @@ + +import pytest +import polars as pl +from pathlib import Path + +@pytest.fixture +def data_info(): + data_dir = Path("ny-taxi/data/trip-data/") + parquet_files = list(data_dir.glob("*.parquet")) + return data_dir, parquet_files + +def test_files_exist(data_info): + data_dir, parquet_files = data_info + assert data_dir.exists(), "Missing data directory." + + n_files_expected = 12 + assert len(parquet_files) == n_files_expected, "Not the right number of parquet files" + +def test_single_file(data_info): + + _ , parquet_files = data_info + + first_df = pl.read_parquet(parquet_files[0], n_rows=1_000) + some_column_names = ["fare_amount", "trip_distance", "congestion_surcharge"] + assert set(some_column_names).issubset(set(first_df.columns)), "Expected column names not present" + From 3e5639df4866b93fdcc50bda850b8ea690ee9c5a Mon Sep 17 00:00:00 2001 From: Flavio Hafner <55716026+f-hafner@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:10:36 +0200 Subject: [PATCH 3/3] Update README --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b108c4..38aba67 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,13 @@ ![Python application](https://github.com/jhidding/parallel-python-workshop/workflows/Python%20application/badge.svg) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/escience-academy/parallel-python-workshop/HEAD) -Environment for the Parallel Python workshop. Lesson material can be found on the [Software Carpentry Incubator](https://carpentries-incubator.github.io/lesson-parallel-python/) +Environment and data for the Parallel Python workshop. Lesson material can be found on the [Software Carpentry Incubator](https://carpentries-incubator.github.io/lesson-parallel-python/) + +**Data used in this course:** +- New York taxi data ([description](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page)). The instructions +below help you download the data. + +## Setup instructions If the tests pass, your setup is good for the workshop. @@ -12,6 +18,7 @@ UV is our recommended tool to manage the Python environment. Please follow the [ ```bash uv sync +uv run ny-taxi/download.py uv run pytest ``` @@ -22,6 +29,7 @@ If you want to use Conda instead of UV, you can do the following: ```bash conda env create -f environment.yml conda activate parallel-python +python ny-taxi/download.py pytest ```