diff --git a/Dockerfile b/Dockerfile index ca21abd..1d4f490 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN mkdir -p /data # This is because outside of Docker we want to ensure # the Python virtualenv is set, but in Docker we don't # use a virtualenv, as docker *is* a virtualenv -ENV VIRTUAL_ENV=/ +ENV VIRTUAL_ENV=/usr ENV PYTHONPATH=/root/star RUN python3 -m pytest ./tests diff --git a/aoh-calculator b/aoh-calculator index d56e12f..c24def9 160000 --- a/aoh-calculator +++ b/aoh-calculator @@ -1 +1 @@ -Subproject commit d56e12f85255ad5ea14d1105febecd635c2afcee +Subproject commit c24def960799f170a9812af31d4c0e2dc5940dbf diff --git a/prepare_species/extract_species_data_psql.py b/prepare_species/extract_species_data_psql.py index 8ea70eb..0ceda78 100644 --- a/prepare_species/extract_species_data_psql.py +++ b/prepare_species/extract_species_data_psql.py @@ -509,7 +509,7 @@ def main() -> None: parser.add_argument( '--output', type=str, - help='Directory where per species Geojson is stored', + help='Directory where per species GeoJSON is stored', required=True, dest='output_directory_path', ) diff --git a/scripts/run.sh b/scripts/run.sh index ed380f6..25d5668 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -19,67 +19,96 @@ if [ -z "${VIRTUAL_ENV}" ]; then exit 1 fi +export CPUS=`getconf _NPROCESSORS_ONLN` +export THREADS=$(($CPUS / 2)) +echo "Using $THREADS threads." + declare -a TAXALIST=("AMPHIBIA" "AVES" "MAMMALIA" "REPTILIA") # Get habitat layer and prepare for use -if [ ! -f ${DATADIR}/habitat/raw.tif ]; then - reclaimer zenodo --zenodo_id 3939050 --filename PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif --output ${DATADIR}/habitat/raw.tif +if [ ! -d ${DATADIR}/habitat_layers ]; then + if [ ! -f ${DATADIR}/habitat/raw.tif ]; then + echo "Fetching habitat map..." + reclaimer zenodo --zenodo_id 3939050 --filename PROBAV_LC100_global_v3.0.1_2019-nrt_Discrete-Classification-map_EPSG-4326.tif --output ${DATADIR}/habitat/raw.tif + fi + + echo "Processing habitat map..." + python3 ./aoh-calculator/habitat_process.py --habitat ${DATADIR}/habitat/raw.tif \ + --scale 1000.0 \ + --projection "ESRI:54009" \ + --output ${DATADIR}/tmp_habitat_layers/current + mv ${DATADIR}/tmp_habitat_layers ${DATADIR}/habitat_layers fi -python3 ./aoh-calculator/habitat_process.py --habitat ${DATADIR}/habitat/raw.tif \ - --scale 1000.0 \ - --projection "ESRI:54009" \ - --output ${DATADIR}/habitat_layers/current - -python3 ./prepare_layers/make_masks.py --habitat_layers ${DATADIR}/habitat_layers/current \ - --output_directory ${DATADIR}/masks +if [ ! -d ${DATADIR}/masks ]; then + echo "Processing masks..." + python3 ./prepare_layers/make_masks.py --habitat_layers ${DATADIR}/habitat_layers/current \ + --output_directory ${DATADIR}/masks +fi # Fetch and prepare the elevation layers -if [ ! -f ${DATADIR}/elevation/elevation.tif ]; then - reclaimer zenodo --zenodo_id 5719984 --filename dem-100m-esri54017.tif --output ${DATADIR}/elevation/elevation.tif -fi -if [ ! -f ${DATADIR}/elevation/elevation-max-1k.tif ]; then - gdalwarp -t_srs ESRI:54009 -tr 1000 -1000 -r max -co COMPRESS=LZW -wo NUM_THREADS=40 ${DATADIR}/elevation/elevation.tif ${DATADIR}/elevation/elevation-max-1k.tif -fi -if [ ! -f ${DATADIR}/elevation/elevation-min-1k.tif ]; then - gdalwarp -t_srs ESRI:54009 -tr 1000 -1000 -r min -co COMPRESS=LZW -wo NUM_THREADS=40 ${DATADIR}/elevation/elevation.tif ${DATADIR}/elevation/elevation-min-1k.tif +if [[ ! -f ${DATADIR}/elevation/elevation-max-1k.tif || ! -f ${DATADIR}/elevation/elevation-min-1k.tif ]]; then + if [ ! -f ${DATADIR}/elevation/elevation.tif ]; then + echo "Fetching elevation map..." + reclaimer zenodo --zenodo_id 5719984 --filename dem-100m-esri54017.tif --output ${DATADIR}/elevation/elevation.tif + fi + if [ ! -f ${DATADIR}/elevation/elevation-max-1k.tif ]; then + echo "Generating elevation max layer..." + gdalwarp -t_srs ESRI:54009 -tr 1000 -1000 -r max -co COMPRESS=LZW -wo NUM_THREADS=40 ${DATADIR}/elevation/elevation.tif ${DATADIR}/elevation/elevation-max-1k.tif + fi + if [ ! -f ${DATADIR}/elevation/elevation-min-1k.tif ]; then + echo "Generating elevation min layer..." + gdalwarp -t_srs ESRI:54009 -tr 1000 -1000 -r min -co COMPRESS=LZW -wo NUM_THREADS=40 ${DATADIR}/elevation/elevation.tif ${DATADIR}/elevation/elevation-min-1k.tif + fi fi # Generate the crosswalk table if [ ! -f ${DATADIR}/crosswalk.csv ]; then + echo "Generating crosswalk table..." python3 ./prepare_layers/convert_crosswalk.py --original ${PWD}/data/crosswalk_bin_T.csv --output ${DATADIR}/crosswalk.csv fi # Get species data per taxa from IUCN data for TAXA in "${TAXALIST[@]}" do + echo "Extracting species data for ${TAXA}..." python3 ./prepare_species/extract_species_data_psql.py --class ${TAXA} --output ${DATADIR}/species-info/${TAXA}/ --projection "ESRI:54009" --excludes ${DATADIR}/SpeciesList_generalisedRangePolygons.csv done if [ -f data/BL_Species_Elevations_2023.csv ]; then + echo "Applying birdlife data..." python3 ./prepare_species/apply_birdlife_data.py --geojsons ${DATADIR}/species-info/AVES --overrides data/BL_Species_Elevations_2023.csv fi +echo "Generating AoH task list..." python3 ./utils/aoh_generator.py --input ${DATADIR}/species-info --datadir ${DATADIR} --output ${DATADIR}/aohbatch.csv -littlejohn -j 200 -o ${DATADIR}/aohbatch.log -c ${DATADIR}/aohbatch.csv ${VIRTUAL_ENV}/bin/python3 -- ./aoh-calculator/aohcalc.py +echo "Generating AoHs..." +littlejohn -j ${THREADS} -o ${DATADIR}/aohbatch.log -c ${DATADIR}/aohbatch.csv ${VIRTUAL_ENV}/bin/python3 -- ./aoh-calculator/aohcalc.py # Calculate predictors from AoHs +echo "Generating species richness..." python3 ./aoh-calculator/summaries/species_richness.py --aohs_folder ${DATADIR}/aohs/current/ \ --output ${DATADIR}/summaries/species_richness.tif +echo "Generating endemism..." python3 ./aoh-calculator/summaries/endemism.py --aohs_folder ${DATADIR}/aohs/current/ \ --species_richness ${DATADIR}/summaries/species_richness.tif \ --output ${DATADIR}/summaries/endemism.tif # Aoh Validation +echo "Collating validation data..." python3 ./aoh-calculator/validation/collate_data.py --aoh_results ${DATADIR}/aohs/current/ \ --output ${DATADIR}/validation/aohs.csv +echo "Calculating model validation..." python3 ./aoh-calculator/validation/validate_map_prevalence.py --collated_aoh_data ${DATADIR}/validation/aohs.csv \ --output ${DATADIR}/validation/model_validation.csv # Threats +echo "Generating threat task list..." python3 ./utils/threats_generator.py --input ${DATADIR}/species-info --datadir ${DATADIR} --output ${DATADIR}/threatbatch.csv -littlejohn -j 200 -o ${DATADIR}/threatbatch.log -c ${DATADIR}/threatcatch.csv ${VIRTUAL_ENV}/bin/python3 -- ./threats/threat_processing.py +echo "Generating threat rasters..." +littlejohn -j ${THREADS} -o ${DATADIR}/threatbatch.log -c ${DATADIR}/threatcatch.csv ${VIRTUAL_ENV}/bin/python3 -- ./threats/threat_processing.py +echo "Summarising threats..." python3 ./threats/threat_summation.py --threat_rasters ${DATADIR}/threat_rasters --output ${DATADIR}/threat_results