From 8f5a7320558592ba714d9d83143c1d7f9c0bb91b Mon Sep 17 00:00:00 2001 From: Volker Hoffmann Date: Mon, 13 Apr 2026 12:48:42 +0200 Subject: [PATCH 1/5] feat(stacker): BASE_DIR from env --- tessera_preprocessing/s1_s2_stacker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tessera_preprocessing/s1_s2_stacker.sh b/tessera_preprocessing/s1_s2_stacker.sh index e2f5ff6..238e26a 100644 --- a/tessera_preprocessing/s1_s2_stacker.sh +++ b/tessera_preprocessing/s1_s2_stacker.sh @@ -12,8 +12,7 @@ set -u ####################################### # === Basic Configuration === -# BASE_DIR="/absolute/path/to/your/data_dir" -BASE_DIR="/scratch/zf281/tessera/data/cambridge/output/2024" + : "${BASE_DIR:=/absolute/path/to/your/data_dir}" OUT_DIR="${BASE_DIR}/data_processed" DOWNSAMPLE_RATE=1 From 9f20e1b37506c020b8ab281ea1596795a067058d Mon Sep 17 00:00:00 2001 From: Volker Hoffmann Date: Mon, 13 Apr 2026 12:51:02 +0200 Subject: [PATCH 2/5] feat(infer/QAT): base, python, nproc from env --- tessera_infer_QAT/infer_all_tiles.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tessera_infer_QAT/infer_all_tiles.sh b/tessera_infer_QAT/infer_all_tiles.sh index 78064b7..83da891 100755 --- a/tessera_infer_QAT/infer_all_tiles.sh +++ b/tessera_infer_QAT/infer_all_tiles.sh @@ -8,18 +8,18 @@ ############### This needs to be modified to your environment ############### # Main directory where preprocessed tiles and outputs are located -BASE_DATA_DIR="/absolute_path_to_data_dir" + : "${BASE_DATA_DIR:=/absolute_path_to_data_dir}" # Python environment with required dependencies -export PYTHON_ENV="/absolute_path_to_python_env/bin/python" + : "${PYTHON_ENV:=/absolute/path/to/your/python_env/bin/python}" # CPU:GPU split ratio (Format: CPU:GPU) # Examples: "1:1" (balanced), "1:0" (CPU only), "0:1" (GPU only) -CPU_GPU_SPLIT="1:0" + : "${CPU_GPU_SPLIT:=1:0}" # Max concurrent tile processes for CPU/GPU -MAX_CONCURRENT_PROCESSES_CPU=20 -MAX_CONCURRENT_PROCESSES_GPU=1 + : "${MAX_CONCURRENT_PROCESSES_CPU:=20}" + : "${MAX_CONCURRENT_PROCESSES_GPU:=1}" # CPU cores to use TOTAL_CPU_CORES=$(nproc) From acd9b2148fcee636917a4274f88cb5a36775f6cc Mon Sep 17 00:00:00 2001 From: Volker Hoffmann Date: Mon, 13 Apr 2026 12:56:12 +0200 Subject: [PATCH 3/5] feat(shp2tiff): shapefile as cl arg --- tessera_preprocessing/convert_shp_to_tiff.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tessera_preprocessing/convert_shp_to_tiff.py b/tessera_preprocessing/convert_shp_to_tiff.py index de68f36..a623c35 100644 --- a/tessera_preprocessing/convert_shp_to_tiff.py +++ b/tessera_preprocessing/convert_shp_to_tiff.py @@ -3,6 +3,7 @@ import fiona import rasterio import logging +import sys import numpy as np from rasterio.features import rasterize from rasterio.transform import from_origin @@ -261,7 +262,11 @@ def main(): Main function to run the conversion process. """ # Input shapefile path - shp_path = 'absolute_path_to_your_shp_file' + if len(sys.argv) < 2: + # uv run ./convert_shp_to_tiff.py /path/to/shapefile.shp + print("Usage: python ./convert_shp_to_tiff.py /path/to/shapefile.shp") + sys.exit(1) + shp_path = sys.argv[1] # Call the conversion function try: From be9e085d876d472d5f252a32ffe1fc11c1fe74f3 Mon Sep 17 00:00:00 2001 From: Volker Hoffmann Date: Tue, 5 May 2026 14:31:51 +0200 Subject: [PATCH 4/5] feat: add $BASE_LOG_DIR to infer_all_tiles.sh --- tessera_infer_QAT/infer_all_tiles.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tessera_infer_QAT/infer_all_tiles.sh b/tessera_infer_QAT/infer_all_tiles.sh index 83da891..c353051 100755 --- a/tessera_infer_QAT/infer_all_tiles.sh +++ b/tessera_infer_QAT/infer_all_tiles.sh @@ -13,6 +13,9 @@ # Python environment with required dependencies : "${PYTHON_ENV:=/absolute/path/to/your/python_env/bin/python}" +# Base directory for logfiles + : "${BASE_LOG_DIR:=.}" + # CPU:GPU split ratio (Format: CPU:GPU) # Examples: "1:1" (balanced), "1:0" (CPU only), "0:1" (GPU only) : "${CPU_GPU_SPLIT:=1:0}" @@ -156,7 +159,7 @@ trap cleanup SIGINT SIGTERM log_header "SETUP DIRECTORIES" mkdir -p "$OUTPUT_DIR" mkdir -p "src/tile_lists" -mkdir -p "logs" +mkdir -p "${BASE_LOG_DIR}/logs" log_success "Created necessary directories" log_header "SCANNING TILES" @@ -272,7 +275,7 @@ launch_gpu_processes() { log_header "GPU PROCESSING" for ((i=0; i "$log_file" $PYTHON_ENV "$PYTHON_SCRIPT" \ --config "$CONFIG_FILE" \ @@ -394,6 +397,6 @@ SCRIPT_END=$(date +%s) TOTAL_DURATION=$(calculate_time $SCRIPT_START $SCRIPT_END) log_header "COMPLETED IN $TOTAL_DURATION" log_info "Detailed logs:" -log_info " - CPU logs: logs/infer_qat_cpu_*.log" -log_info " - GPU logs: logs/infer_qat_gpu_*.log" +log_info " - CPU logs: ${BASE_LOG_DIR}/logs/infer_qat_cpu_*.log" +log_info " - GPU logs: ${BASE_LOG_DIR}/logs/infer_qat_gpu_*.log" From 68bdfc6406478fcb2952fb2f26209d72c2c596cf Mon Sep 17 00:00:00 2001 From: Volker Hoffmann Date: Tue, 28 Apr 2026 13:50:18 +0200 Subject: [PATCH 5/5] style(shp2tiff): run ruff --- tessera_preprocessing/convert_shp_to_tiff.py | 49 ++++++++++++-------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/tessera_preprocessing/convert_shp_to_tiff.py b/tessera_preprocessing/convert_shp_to_tiff.py index a623c35..b75503c 100644 --- a/tessera_preprocessing/convert_shp_to_tiff.py +++ b/tessera_preprocessing/convert_shp_to_tiff.py @@ -1,21 +1,26 @@ #!/usr/bin/env python3 -import os -import fiona -import rasterio import logging +import os import sys + +import fiona import numpy as np +import rasterio +from pyproj import Transformer +from rasterio.crs import CRS from rasterio.features import rasterize from rasterio.transform import from_origin -from rasterio.crs import CRS -from shapely.geometry import shape, mapping -from shapely.ops import transform as shp_transform, unary_union -from pyproj import Transformer +from shapely.geometry import mapping, shape +from shapely.ops import transform as shp_transform +from shapely.ops import unary_union # Set up logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) logger = logging.getLogger(__name__) + def determine_utm_zone(lon, lat): """ Determine the best UTM zone based on longitude and latitude. @@ -52,6 +57,7 @@ def determine_utm_zone(lon, lat): return epsg_code, zone_number, is_northern + def determine_best_utm_crs(geometries, src_crs): """ Determine the best UTM coordinate reference system based on the centroid of the geometry collection. @@ -91,6 +97,7 @@ def determine_best_utm_crs(geometries, src_crs): return CRS.from_epsg(epsg_code) + def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): """ Convert a shapefile to a TIFF raster. @@ -106,14 +113,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): """ # Set default output path if not provided if tiff_path is None: - tiff_path = os.path.splitext(shp_path)[0] + '.tiff' + tiff_path = os.path.splitext(shp_path)[0] + ".tiff" logger.info(f"Starting conversion of shapefile: {shp_path}") logger.info(f"Output TIFF will be saved as: {tiff_path}") logger.info(f"Using pixel size: {pixel_size} meters") # Open the shapefile and read geometries - with fiona.open(shp_path, 'r') as src: + with fiona.open(shp_path, "r") as src: # Get basic shapefile information num_features = len(src) src_driver = src.driver @@ -125,7 +132,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): logger.info(f" - Schema: {src_schema}") # Read all geometries - geometries = [feature['geometry'] for feature in src] + geometries = [feature["geometry"] for feature in src] logger.info(f"Read {len(geometries)} geometries from the shapefile") # Get source CRS, default to EPSG:4326 if undefined @@ -150,7 +157,9 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): # Reproject geometries to the target CRS try: - reprojected_geoms = [mapping(shp_transform(transformer, shape(geom))) for geom in geometries] + reprojected_geoms = [ + mapping(shp_transform(transformer, shape(geom))) for geom in geometries + ] logger.info(f"Successfully reprojected {len(reprojected_geoms)} geometries") except Exception as e: logger.error(f"Error reprojecting geometries: {str(e)}") @@ -193,7 +202,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): transform=transform_affine, fill=0, default_value=255, - dtype='uint8' + dtype="uint8", ) logger.info(f"Rasterization complete. Raster shape: {raster.shape}") except Exception as e: @@ -205,8 +214,8 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): try: with rasterio.open( tiff_path, - 'w', - driver='GTiff', + "w", + driver="GTiff", height=height, width=width, count=1, @@ -221,7 +230,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): raise # Create convex hull TIFF - hull_tiff_path = os.path.splitext(tiff_path)[0] + '_convex_hull.tiff' + hull_tiff_path = os.path.splitext(tiff_path)[0] + "_convex_hull.tiff" logger.info(f"Creating convex hull TIFF: {hull_tiff_path}") try: @@ -234,14 +243,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): out_shape=(height, width), transform=transform_affine, fill=0, - dtype='uint8' + dtype="uint8", ) # Write the convex hull to TIFF with rasterio.open( hull_tiff_path, - 'w', - driver='GTiff', + "w", + driver="GTiff", height=height, width=width, count=1, @@ -257,6 +266,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): return tiff_path, hull_tiff_path + def main(): """ Main function to run the conversion process. @@ -278,5 +288,6 @@ def main(): logger.error(f"Conversion failed: {str(e)}") raise + if __name__ == "__main__": main()