From c1d7d99256d925271a5d0411a247af2da0c91c7d Mon Sep 17 00:00:00 2001 From: mark-hammond Date: Fri, 5 Jun 2026 16:24:54 +0100 Subject: [PATCH] Fix UnicodeDecodeError reading the cp1252 DUKES failures cache data/generators/nominatim_cache_failures.txt is cp1252-encoded (Windows origin), so the default UTF-8 open() raises UnicodeDecodeError on a fresh macOS/Linux clone. Read it as cp1252 with errors='ignore'. --- scripts/generators/DUKES_generator_data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/generators/DUKES_generator_data.py b/scripts/generators/DUKES_generator_data.py index 483da07..09f4013 100644 --- a/scripts/generators/DUKES_generator_data.py +++ b/scripts/generators/DUKES_generator_data.py @@ -457,7 +457,10 @@ def geocode_from_nominatim(df: pd.DataFrame, cache_file: str = "data/generators/ # Load failure cache (stations that couldn't be geocoded) failed_stations = set() if Path(failure_cache_file).exists(): - with open(failure_cache_file, 'r') as f: + # The failures cache ships cp1252-encoded (Windows origin; e.g. the apostrophe + # in "Fiddler's Ferry"), which raises UnicodeDecodeError under the default UTF-8 + # reader on macOS/Linux. Read it as cp1252 and tolerate stray bytes. + with open(failure_cache_file, 'r', encoding='cp1252', errors='ignore') as f: failed_stations = set(line.strip() for line in f if line.strip()) logger.info(f"Loaded {len(failed_stations)} previously failed stations from cache")