Minor patches to some CLI scripts

tessera_infer_QAT/infer_all_tiles.sh

@@ -8,18 +8,21 @@
 
 ############### This needs to be modified to your environment ###############
 # Main directory where preprocessed tiles and outputs are located
-BASE_DATA_DIR="/absolute_path_to_data_dir"
+ : "${BASE_DATA_DIR:=/absolute_path_to_data_dir}"
 
 # Python environment with required dependencies
-export PYTHON_ENV="/absolute_path_to_python_env/bin/python"
+ : "${PYTHON_ENV:=/absolute/path/to/your/python_env/bin/python}"
+
+# Base directory for logfiles
+ : "${BASE_LOG_DIR:=.}"
 
 # CPU:GPU split ratio (Format: CPU:GPU)
 # Examples: "1:1" (balanced), "1:0" (CPU only), "0:1" (GPU only)
-CPU_GPU_SPLIT="1:0"
+ : "${CPU_GPU_SPLIT:=1:0}"
 
 # Max concurrent tile processes for CPU/GPU
-MAX_CONCURRENT_PROCESSES_CPU=20
-MAX_CONCURRENT_PROCESSES_GPU=1
+ : "${MAX_CONCURRENT_PROCESSES_CPU:=20}"
+ : "${MAX_CONCURRENT_PROCESSES_GPU:=1}"
 
 # CPU cores to use
 TOTAL_CPU_CORES=$(nproc)
@@ -156,7 +159,7 @@ trap cleanup SIGINT SIGTERM
 log_header "SETUP DIRECTORIES"
 mkdir -p "$OUTPUT_DIR"
 mkdir -p "src/tile_lists"
-mkdir -p "logs"
+mkdir -p "${BASE_LOG_DIR}/logs"
 log_success "Created necessary directories"
 
 log_header "SCANNING TILES"
@@ -272,7 +275,7 @@ launch_gpu_processes() {
     log_header "GPU PROCESSING"
     for ((i=0; i<NUM_GPU_PROCESSES; i++)); do
         TILE_LIST="src/tile_lists/tiles_gpu_${i}.json"
-        LOG_FILE="logs/infer_qat_gpu_${i}.log"
+        LOG_FILE="${BASE_LOG_DIR}/logs/infer_qat_gpu_${i}.log"
         VERBOSE_GPU_FLAG=""
         [[ "$VERBOSE_GPU" == "true" ]] && VERBOSE_GPU_FLAG="--verbose_gpu"
         CMD="$PYTHON_ENV $PYTHON_SCRIPT --config $CONFIG_FILE --mode gpu --gpu_id $i --tile_list $TILE_LIST --process_id $i --checkpoint_path $CHECKPOINT_PATH --output_dir $OUTPUT_DIR --batch_size $GPU_BATCH_SIZE --num_workers $GPU_NUM_WORKERS --log_interval $LOG_INTERVAL $GPU_BF16_FLAG $VERBOSE_GPU_FLAG --simplified_logging"
@@ -287,7 +290,7 @@ launch_gpu_processes() {
 start_cpu_process() {
     local tile_path="$1"
     local process_id="$2"
-    local log_file="logs/infer_qat_cpu_${process_id}.log"
+    local log_file="${BASE_LOG_DIR}/logs/infer_qat_cpu_${process_id}.log"
     > "$log_file"
     $PYTHON_ENV "$PYTHON_SCRIPT" \
         --config "$CONFIG_FILE" \
@@ -394,6 +397,6 @@ SCRIPT_END=$(date +%s)
 TOTAL_DURATION=$(calculate_time $SCRIPT_START $SCRIPT_END)
 log_header "COMPLETED IN $TOTAL_DURATION"
 log_info "Detailed logs:"
-log_info "  - CPU logs: logs/infer_qat_cpu_*.log"
-log_info "  - GPU logs: logs/infer_qat_gpu_*.log"
+log_info "  - CPU logs: ${BASE_LOG_DIR}/logs/infer_qat_cpu_*.log"
+log_info "  - GPU logs: ${BASE_LOG_DIR}/logs/infer_qat_gpu_*.log"
 

tessera_preprocessing/convert_shp_to_tiff.py

@@ -1,20 +1,26 @@
 #!/usr/bin/env python3
+import logging
 import os
+import sys
+
 import fiona
-import rasterio
-import logging
 import numpy as np
+import rasterio
+from pyproj import Transformer
+from rasterio.crs import CRS
 from rasterio.features import rasterize
 from rasterio.transform import from_origin
-from rasterio.crs import CRS
-from shapely.geometry import shape, mapping
-from shapely.ops import transform as shp_transform, unary_union
-from pyproj import Transformer
+from shapely.geometry import mapping, shape
+from shapely.ops import transform as shp_transform
+from shapely.ops import unary_union
 
 # Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
 logger = logging.getLogger(__name__)
 
+
 def determine_utm_zone(lon, lat):
     """
     Determine the best UTM zone based on longitude and latitude.
@@ -51,6 +57,7 @@ def determine_utm_zone(lon, lat):
 
     return epsg_code, zone_number, is_northern
 
+
 def determine_best_utm_crs(geometries, src_crs):
     """
     Determine the best UTM coordinate reference system based on the centroid of the geometry collection.
@@ -90,6 +97,7 @@ def determine_best_utm_crs(geometries, src_crs):
 
     return CRS.from_epsg(epsg_code)
 
+
 def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
     """
     Convert a shapefile to a TIFF raster.
@@ -105,14 +113,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
     """
     # Set default output path if not provided
     if tiff_path is None:
-        tiff_path = os.path.splitext(shp_path)[0] + '.tiff'
+        tiff_path = os.path.splitext(shp_path)[0] + ".tiff"
 
     logger.info(f"Starting conversion of shapefile: {shp_path}")
     logger.info(f"Output TIFF will be saved as: {tiff_path}")
     logger.info(f"Using pixel size: {pixel_size} meters")
 
     # Open the shapefile and read geometries
-    with fiona.open(shp_path, 'r') as src:
+    with fiona.open(shp_path, "r") as src:
         # Get basic shapefile information
         num_features = len(src)
         src_driver = src.driver
@@ -124,7 +132,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
         logger.info(f"  - Schema: {src_schema}")
 
         # Read all geometries
-        geometries = [feature['geometry'] for feature in src]
+        geometries = [feature["geometry"] for feature in src]
         logger.info(f"Read {len(geometries)} geometries from the shapefile")
 
         # Get source CRS, default to EPSG:4326 if undefined
@@ -149,7 +157,9 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
 
     # Reproject geometries to the target CRS
     try:
-        reprojected_geoms = [mapping(shp_transform(transformer, shape(geom))) for geom in geometries]
+        reprojected_geoms = [
+            mapping(shp_transform(transformer, shape(geom))) for geom in geometries
+        ]
         logger.info(f"Successfully reprojected {len(reprojected_geoms)} geometries")
     except Exception as e:
         logger.error(f"Error reprojecting geometries: {str(e)}")
@@ -192,7 +202,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
             transform=transform_affine,
             fill=0,
             default_value=255,
-            dtype='uint8'
+            dtype="uint8",
         )
         logger.info(f"Rasterization complete. Raster shape: {raster.shape}")
     except Exception as e:
@@ -204,8 +214,8 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
     try:
         with rasterio.open(
             tiff_path,
-            'w',
-            driver='GTiff',
+            "w",
+            driver="GTiff",
             height=height,
             width=width,
             count=1,
@@ -220,7 +230,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
         raise
 
     # Create convex hull TIFF
-    hull_tiff_path = os.path.splitext(tiff_path)[0] + '_convex_hull.tiff'
+    hull_tiff_path = os.path.splitext(tiff_path)[0] + "_convex_hull.tiff"
     logger.info(f"Creating convex hull TIFF: {hull_tiff_path}")
 
     try:
@@ -233,14 +243,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
             out_shape=(height, width),
             transform=transform_affine,
             fill=0,
-            dtype='uint8'
+            dtype="uint8",
         )
 
         # Write the convex hull to TIFF
         with rasterio.open(
             hull_tiff_path,
-            'w',
-            driver='GTiff',
+            "w",
+            driver="GTiff",
             height=height,
             width=width,
             count=1,
@@ -256,12 +266,17 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None):
 
     return tiff_path, hull_tiff_path
 
+
 def main():
     """
     Main function to run the conversion process.
     """
     # Input shapefile path
-    shp_path = 'absolute_path_to_your_shp_file'
+    if len(sys.argv) < 2:
+        # uv run ./convert_shp_to_tiff.py /path/to/shapefile.shp
+        print("Usage: python ./convert_shp_to_tiff.py /path/to/shapefile.shp")
+        sys.exit(1)
+    shp_path = sys.argv[1]
 
     # Call the conversion function
     try:
@@ -273,5 +288,6 @@ def main():
         logger.error(f"Conversion failed: {str(e)}")
         raise
 
+
 if __name__ == "__main__":
     main()

tessera_preprocessing/s1_s2_stacker.sh

@@ -12,8 +12,7 @@ set -u
 #######################################
 
 # === Basic Configuration ===
-# BASE_DIR="/absolute/path/to/your/data_dir"
-BASE_DIR="/scratch/zf281/tessera/data/cambridge/output/2024"
+ : "${BASE_DIR:=/absolute/path/to/your/data_dir}"
 OUT_DIR="${BASE_DIR}/data_processed"
 DOWNSAMPLE_RATE=1