diff --git a/tessera_infer_QAT/infer_all_tiles.sh b/tessera_infer_QAT/infer_all_tiles.sh index 78064b7..c353051 100755 --- a/tessera_infer_QAT/infer_all_tiles.sh +++ b/tessera_infer_QAT/infer_all_tiles.sh @@ -8,18 +8,21 @@ ############### This needs to be modified to your environment ############### # Main directory where preprocessed tiles and outputs are located -BASE_DATA_DIR="/absolute_path_to_data_dir" + : "${BASE_DATA_DIR:=/absolute_path_to_data_dir}" # Python environment with required dependencies -export PYTHON_ENV="/absolute_path_to_python_env/bin/python" + : "${PYTHON_ENV:=/absolute/path/to/your/python_env/bin/python}" + +# Base directory for logfiles + : "${BASE_LOG_DIR:=.}" # CPU:GPU split ratio (Format: CPU:GPU) # Examples: "1:1" (balanced), "1:0" (CPU only), "0:1" (GPU only) -CPU_GPU_SPLIT="1:0" + : "${CPU_GPU_SPLIT:=1:0}" # Max concurrent tile processes for CPU/GPU -MAX_CONCURRENT_PROCESSES_CPU=20 -MAX_CONCURRENT_PROCESSES_GPU=1 + : "${MAX_CONCURRENT_PROCESSES_CPU:=20}" + : "${MAX_CONCURRENT_PROCESSES_GPU:=1}" # CPU cores to use TOTAL_CPU_CORES=$(nproc) @@ -156,7 +159,7 @@ trap cleanup SIGINT SIGTERM log_header "SETUP DIRECTORIES" mkdir -p "$OUTPUT_DIR" mkdir -p "src/tile_lists" -mkdir -p "logs" +mkdir -p "${BASE_LOG_DIR}/logs" log_success "Created necessary directories" log_header "SCANNING TILES" @@ -272,7 +275,7 @@ launch_gpu_processes() { log_header "GPU PROCESSING" for ((i=0; i "$log_file" $PYTHON_ENV "$PYTHON_SCRIPT" \ --config "$CONFIG_FILE" \ @@ -394,6 +397,6 @@ SCRIPT_END=$(date +%s) TOTAL_DURATION=$(calculate_time $SCRIPT_START $SCRIPT_END) log_header "COMPLETED IN $TOTAL_DURATION" log_info "Detailed logs:" -log_info " - CPU logs: logs/infer_qat_cpu_*.log" -log_info " - GPU logs: logs/infer_qat_gpu_*.log" +log_info " - CPU logs: ${BASE_LOG_DIR}/logs/infer_qat_cpu_*.log" +log_info " - GPU logs: ${BASE_LOG_DIR}/logs/infer_qat_gpu_*.log" diff --git a/tessera_preprocessing/convert_shp_to_tiff.py b/tessera_preprocessing/convert_shp_to_tiff.py index de68f36..b75503c 100644 --- a/tessera_preprocessing/convert_shp_to_tiff.py +++ b/tessera_preprocessing/convert_shp_to_tiff.py @@ -1,20 +1,26 @@ #!/usr/bin/env python3 +import logging import os +import sys + import fiona -import rasterio -import logging import numpy as np +import rasterio +from pyproj import Transformer +from rasterio.crs import CRS from rasterio.features import rasterize from rasterio.transform import from_origin -from rasterio.crs import CRS -from shapely.geometry import shape, mapping -from shapely.ops import transform as shp_transform, unary_union -from pyproj import Transformer +from shapely.geometry import mapping, shape +from shapely.ops import transform as shp_transform +from shapely.ops import unary_union # Set up logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) logger = logging.getLogger(__name__) + def determine_utm_zone(lon, lat): """ Determine the best UTM zone based on longitude and latitude. @@ -51,6 +57,7 @@ def determine_utm_zone(lon, lat): return epsg_code, zone_number, is_northern + def determine_best_utm_crs(geometries, src_crs): """ Determine the best UTM coordinate reference system based on the centroid of the geometry collection. @@ -90,6 +97,7 @@ def determine_best_utm_crs(geometries, src_crs): return CRS.from_epsg(epsg_code) + def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): """ Convert a shapefile to a TIFF raster. @@ -105,14 +113,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): """ # Set default output path if not provided if tiff_path is None: - tiff_path = os.path.splitext(shp_path)[0] + '.tiff' + tiff_path = os.path.splitext(shp_path)[0] + ".tiff" logger.info(f"Starting conversion of shapefile: {shp_path}") logger.info(f"Output TIFF will be saved as: {tiff_path}") logger.info(f"Using pixel size: {pixel_size} meters") # Open the shapefile and read geometries - with fiona.open(shp_path, 'r') as src: + with fiona.open(shp_path, "r") as src: # Get basic shapefile information num_features = len(src) src_driver = src.driver @@ -124,7 +132,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): logger.info(f" - Schema: {src_schema}") # Read all geometries - geometries = [feature['geometry'] for feature in src] + geometries = [feature["geometry"] for feature in src] logger.info(f"Read {len(geometries)} geometries from the shapefile") # Get source CRS, default to EPSG:4326 if undefined @@ -149,7 +157,9 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): # Reproject geometries to the target CRS try: - reprojected_geoms = [mapping(shp_transform(transformer, shape(geom))) for geom in geometries] + reprojected_geoms = [ + mapping(shp_transform(transformer, shape(geom))) for geom in geometries + ] logger.info(f"Successfully reprojected {len(reprojected_geoms)} geometries") except Exception as e: logger.error(f"Error reprojecting geometries: {str(e)}") @@ -192,7 +202,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): transform=transform_affine, fill=0, default_value=255, - dtype='uint8' + dtype="uint8", ) logger.info(f"Rasterization complete. Raster shape: {raster.shape}") except Exception as e: @@ -204,8 +214,8 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): try: with rasterio.open( tiff_path, - 'w', - driver='GTiff', + "w", + driver="GTiff", height=height, width=width, count=1, @@ -220,7 +230,7 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): raise # Create convex hull TIFF - hull_tiff_path = os.path.splitext(tiff_path)[0] + '_convex_hull.tiff' + hull_tiff_path = os.path.splitext(tiff_path)[0] + "_convex_hull.tiff" logger.info(f"Creating convex hull TIFF: {hull_tiff_path}") try: @@ -233,14 +243,14 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): out_shape=(height, width), transform=transform_affine, fill=0, - dtype='uint8' + dtype="uint8", ) # Write the convex hull to TIFF with rasterio.open( hull_tiff_path, - 'w', - driver='GTiff', + "w", + driver="GTiff", height=height, width=width, count=1, @@ -256,12 +266,17 @@ def shp_to_tiff(shp_path, tiff_path=None, pixel_size=100, force_crs=None): return tiff_path, hull_tiff_path + def main(): """ Main function to run the conversion process. """ # Input shapefile path - shp_path = 'absolute_path_to_your_shp_file' + if len(sys.argv) < 2: + # uv run ./convert_shp_to_tiff.py /path/to/shapefile.shp + print("Usage: python ./convert_shp_to_tiff.py /path/to/shapefile.shp") + sys.exit(1) + shp_path = sys.argv[1] # Call the conversion function try: @@ -273,5 +288,6 @@ def main(): logger.error(f"Conversion failed: {str(e)}") raise + if __name__ == "__main__": main() diff --git a/tessera_preprocessing/s1_s2_stacker.sh b/tessera_preprocessing/s1_s2_stacker.sh index e2f5ff6..238e26a 100644 --- a/tessera_preprocessing/s1_s2_stacker.sh +++ b/tessera_preprocessing/s1_s2_stacker.sh @@ -12,8 +12,7 @@ set -u ####################################### # === Basic Configuration === -# BASE_DIR="/absolute/path/to/your/data_dir" -BASE_DIR="/scratch/zf281/tessera/data/cambridge/output/2024" + : "${BASE_DIR:=/absolute/path/to/your/data_dir}" OUT_DIR="${BASE_DIR}/data_processed" DOWNSAMPLE_RATE=1