Update setup instructions

README.md

@@ -2,7 +2,13 @@
 ![Python application](https://github.com/jhidding/parallel-python-workshop/workflows/Python%20application/badge.svg)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/escience-academy/parallel-python-workshop/HEAD)
 
-Environment for the Parallel Python workshop. Lesson material can be found on the [Software Carpentry Incubator](https://carpentries-incubator.github.io/lesson-parallel-python/)
+Environment and data for the Parallel Python workshop. Lesson material can be found on the [Software Carpentry Incubator](https://carpentries-incubator.github.io/lesson-parallel-python/)
+
+**Data used in this course:**
+- New York taxi data ([description](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page)). The instructions
+below help you download the data.
+
+## Setup instructions
 
 If the tests pass, your setup is good for the workshop.
 
@@ -12,6 +18,7 @@ UV is our recommended tool to manage the Python environment. Please follow the [
 
 ```bash
 uv sync
+uv run ny-taxi/download.py
 uv run pytest
 ```
 
@@ -22,6 +29,7 @@ If you want to use Conda instead of UV, you can do the following:
 ```bash
 conda env create -f environment.yml
 conda activate parallel-python
+python ny-taxi/download.py
 pytest
 ```
 

ny-taxi/download.py

@@ -1,17 +1,28 @@
+"""Script to downlad NY Taxi data.
+
+- Uses `pooch` to manage and download files from DATA_BASE_URL
+- TRIP_DATA_NAME is the name of the subdirectory where the parquet
+files are stored.
+"""
+
 import pooch
 from pathlib import Path
 
 
+DATA_BASE_URL = "https://d37ci6vzurychx.cloudfront.net/trip-data/"
+TRIP_DATA_NAME = "trip-data"
+REGISTRY_FILE = f"{TRIP_DATA_NAME}-registry.txt"
+
 def taxi_filename(year: int = 2025, month: int = 1):
     return f"yellow_tripdata_{year:04}-{month:02}.parquet"
 
 
 def bootstrap_taxi_data(
-    path: Path = Path() / "data" / "trip-data", year: int = 2025
+    path: Path, year: int = 2025
 ) -> pooch.Pooch:
     return pooch.create(
         path=path,
-        base_url="https://d37ci6vzurychx.cloudfront.net/trip-data/",
+        base_url=DATA_BASE_URL,
         registry={
             taxi_filename(year=year, month=month): None for month in range(1, 13)
         },
@@ -25,37 +36,37 @@ def download_all(p: pooch.Pooch):
 
 def make_registry(p: pooch.Pooch):
     path = Path(p.path)
-    registry_name = path.name + "-registry.txt"
-    pooch.make_registry(path, path.parent / registry_name)
+    pooch.make_registry(path, path.parent / REGISTRY_FILE)
 
 
 def taxi_data(
-    data_path: Path = Path() / "data", name: str = "trip-data"
+    data_path: Path,
 ) -> pooch.Pooch:
-    path = data_path / name
-    registry = data_path / (name + "-registry.txt")
+    registry = data_path /  REGISTRY_FILE
 
     p = pooch.create(
-        path=path,
-        base_url="https://d37ci6vzurychx.cloudfront.net/trip-data/",
+        path=data_path / TRIP_DATA_NAME,
+        base_url=DATA_BASE_URL,
         registry=None,
     )
     p.load_registry(registry)
     return p
 
 
 def main():
-    registry = Path() / "data" / "trip-data-registry.txt"
+    nyt_path = Path(__file__).parent
+    data_dir = "data"
+    registry = nyt_path / data_dir / REGISTRY_FILE
 
     if registry.exists():
-        p = taxi_data()
+        p = taxi_data(data_path = nyt_path / data_dir)
         download_all(p)
     else:
-        p = bootstrap_taxi_data()
+        p = bootstrap_taxi_data(nyt_path / data_dir / TRIP_DATA_NAME)
         download_all(p)
         make_registry(p)
 
 
 if __name__ == "__main__":
     main()
-
+# TODO: add unit test that data can be read

test/test_nytaxi.py

@@ -0,0 +1,26 @@
+
+import pytest
+import polars as pl
+from pathlib import Path
+
+@pytest.fixture
+def data_info():
+    data_dir = Path("ny-taxi/data/trip-data/")
+    parquet_files = list(data_dir.glob("*.parquet"))
+    return data_dir, parquet_files
+
+def test_files_exist(data_info):
+    data_dir, parquet_files = data_info
+    assert data_dir.exists(), "Missing data directory."
+
+    n_files_expected = 12
+    assert len(parquet_files) == n_files_expected, "Not the right number of parquet files"
+
+def test_single_file(data_info):
+
+    _ , parquet_files = data_info
+
+    first_df = pl.read_parquet(parquet_files[0], n_rows=1_000)
+    some_column_names = ["fare_amount", "trip_distance", "congestion_surcharge"]
+    assert set(some_column_names).issubset(set(first_df.columns)), "Expected column names not present"
+