diff --git a/scripts/run.sh b/scripts/run.sh index d2d8115..37ea655 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -237,7 +237,7 @@ fi # Get species data per taxa from IUCN data for TAXA in "${TAXAS[@]}" do - if [ ! -f "${DATADIR}"/overrides.csv ]; then + if [ -f "${DATADIR}"/overrides.csv ]; then python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" \ --output "${DATADIR}"/species-info/"${TAXA}"/ \ --projection "EPSG:4326" \ @@ -259,7 +259,7 @@ python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" \ --scenarios "${SCENARIOS[@]}" # Calculate all the AoHs -littlejohn -j 700 -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat +littlejohn -j 30 -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv "${VIRTUAL_ENV}"/bin/aoh-calc -- --force-habitat # Generate validation summaries aoh-collate-data --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv @@ -277,7 +277,7 @@ aoh-endemism --aohs_folder "${DATADIR}"/aohs/current/ \ --output "${DATADIR}"/predictors/endemism.tif # Calculate the per species Delta P values -littlejohn -j 200 -o "${DATADIR}"/persistencebatch.log -c "${DATADIR}"/persistencebatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./deltap/global_code_residents_pixel.py +littlejohn -j 30 -o "${DATADIR}"/persistencebatch.log -c "${DATADIR}"/persistencebatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./deltap/global_code_residents_pixel.py for SCENARIO in "${SCENARIOS[@]}" do diff --git a/scripts/slurm.sh b/scripts/slurm.sh index e3db904..2b89169 100644 --- a/scripts/slurm.sh +++ b/scripts/slurm.sh @@ -8,6 +8,7 @@ # https://github.com/quantifyearth/littlejohn - used to run batch jobs in parallel set -e +set -x # shellcheck disable=SC1091 source "${HOME}"/venvs/life/bin/activate @@ -26,144 +27,231 @@ fi declare -a SCENARIOS=("arable" "restore" "restore_all" "urban" "pasture" "restore_agriculture") declare -a TAXAS=("AMPHIBIA" "AVES" "MAMMALIA" "REPTILIA") -export CURVE=gompertz +export CURVE=0.25 +export PIXEL_SCALE=0.016666666666667 + +check_scenario() { + local target="$1" + for scenario in "${SCENARIOS[@]}"; do + if [[ "$scenario" == "$target" ]]; then + return 0 + fi + done + return 1 +} -python3 ./prepare_layers/generate_crosswalk.py --output "${DATADIR}"/crosswalk.csv +if [ ! -f "${DATADIR}"/crosswalk.csv ]; then + python3 ./prepare_layers/generate_crosswalk.py --output "${DATADIR}"/crosswalk.csv +fi # Get habitat layer and prepare for use -reclaimer zenodo --zenodo_id 4058819 \ - --filename iucn_habitatclassification_composite_lvl2_ver004.zip \ - --extract \ - --output "${DATADIR}"/habitat/jung_l2_raw.tif - -reclaimer zenodo --zenodo_id 4058819 \ - --filename lvl2_changemasks_ver004.zip \ - --extract \ - --output "${DATADIR}"/habitat/ - -python3 ./prepare_layers/make_current_map.py --jung "${DATADIR}"/habitat/jung_l2_raw.tif \ - --update_masks "${DATADIR}"/habitat/lvl2_changemasks_ver004 \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/current_raw.tif \ - -j 16 - -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/current_raw.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/current/ +if [ ! -f "${DATADIR}"/habitat/current_raw.tif ]; then + if [ ! -f "${DATADIR}"/habitat/jung_l2_raw.tif ]; then + reclaimer zenodo --zenodo_id 4058819 \ + --filename iucn_habitatclassification_composite_lvl2_ver004.zip \ + --extract \ + --output "${DATADIR}"/habitat/jung_l2_raw.tif + fi + + if [ ! -d "${DATADIR}"/habitat/lvl2_changemasks_ver004 ]; then + reclaimer zenodo --zenodo_id 4058819 \ + --filename lvl2_changemasks_ver004.zip \ + --extract \ + --output "${DATADIR}"/habitat/ + fi + + python3 ./prepare_layers/make_current_map.py --jung "${DATADIR}"/habitat/jung_l2_raw.tif \ + --update_masks "${DATADIR}"/habitat/lvl2_changemasks_ver004 \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/current_raw.tif \ + -j 16 +fi -# Get PNV layer and prepare for use -reclaimer zenodo --zenodo_id 4038749 \ - --filename pnv_lvl1_004.zip \ - --extract \ - --output "${DATADIR}"/habitat/pnv_raw.tif +if [ ! -d "${DATADIR}"/habitat_maps/current ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/current_raw.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/current/ +fi -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/pnv_raw.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/pnv/ +# Get PNV layer and prepare for use +if [ ! -f "${DATADIR}"/habitat/pnv_raw.tif ]; then + reclaimer zenodo --zenodo_id 4038749 \ + --filename pnv_lvl1_004.zip \ + --extract \ + --output "${DATADIR}"/habitat/pnv_raw.tif +fi +if [ ! -d "${DATADIR}"/habitat_maps/pnv ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/pnv_raw.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/pnv/ +fi # Generate an area scaling map -python3 ./prepare_layers/make_area_map.py --scale 0.016666666666667 --output "${DATADIR}"/area-per-pixel.tif +if [ ! -f "${DATADIR}"/area-per-pixel.tif ]; then + python3 ./prepare_layers/make_area_map.py --scale "${PIXEL_SCALE}" --output "${DATADIR}"/area-per-pixel.tif +fi # Generate the arable scenario map -python3 ./prepare_layers/make_arable_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --output "${DATADIR}"/habitat/arable.tif - -aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/arable/ - -python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/arable.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/arable_diff_area.tif +if check_scenario "arable"; then + if [ ! -f "${DATADIR}"/habitat/arable.tif ]; then + python3 ./prepare_layers/make_arable_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --output "${DATADIR}"/habitat/arable.tif + fi + + if [ ! -d "${DATADIR}"/habitat_maps/arable ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/arable/ + fi + + if [ ! -f "${DATADIR}"/habitat/arable_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/arable.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/arable_diff_area.tif + fi +fi # Generate the pasture scenario map -python3 ./prepare_layers/make_pasture_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --output "${DATADIR}"/habitat/pasture.tif - -aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/pasture/ - -python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/pasture.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/pasture_diff_area.tif - -# Generate the restore map -python3 ./prepare_layers/make_restore_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore.tif - -aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore/ - -python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/restore_diff_area.tif +if check_scenario "pasture"; then + if [ ! -f "${DATADIR}"/habitat/pasture.tif ]; then + python3 ./prepare_layers/make_pasture_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --output "${DATADIR}"/habitat/pasture.tif + fi + + if [ ! -d "${DATADIR}"/habitat_maps/pasture ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/pasture/ + fi + + if [ ! -f "${DATADIR}"/habitat/pasture_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/pasture.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/pasture_diff_area.tif + fi +fi # Generate the restore map -python3 ./prepare_layers/make_restore_agriculture_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore_agriculture.tif - -aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore_agriculture/ +if check_scenario "restore"; then + if [ ! -f "${DATADIR}"/habitat/restore.tif ]; then + python3 ./prepare_layers/make_restore_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore.tif + fi + + if [ ! -d "${DATADIR}"/habitat_maps/restore ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore/ + fi + + if [ ! -f "${DATADIR}"/habitat/restore_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_diff_area.tif + fi +fi -python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore_agriculture.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/restore_agriculture_diff_area.tif +# Generate the restore_agriculture map +if check_scenario "restore_agriculture"; then + if [ ! -f "${DATADIR}"/habitat/restore_agriculture.tif ]; then + python3 ./prepare_layers/make_restore_agriculture_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore_agriculture.tif + fi + + if [ ! -d "${DATADIR}"/habitat_maps/restore_agriculture ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_agriculture/ + fi + + if [ ! -f "${DATADIR}"/habitat/restore_agriculture_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore_agriculture.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_agriculture_diff_area.tif + fi +fi # Generate the restore all map -python3 ./prepare_layers/make_restore_all_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore_all.tif - -aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore_all/ - -python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore_all.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/restore_all_diff_area.tif +if check_scenario "restore_all"; then + if [ ! -f "${DATADIR}"/habitat/restore_all.tif ]; then + python3 ./prepare_layers/make_restore_all_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore_all.tif + fi + + if [ ! -d "${DATADIR}"/habitat_maps/restore_all ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_all/ + fi + + if [ ! -f "${DATADIR}"/habitat/restore_all_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore_all.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_all_diff_area.tif + fi +fi # Generate urban all map -python3 ./prepare_layers/make_constant_habitat.py --examplar "${DATADIR}"/habitat_maps/arable/lcc_1401.tif \ - --habitat_code 14.5 \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat_maps/urban - -python3 ./prepare_layers/make_constant_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --habitat_code 14.5 \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat/urban_diff_area.tif +if check_scenario "urban"; then + if [ ! -d "${DATADIR}"/habitat_maps/urban ]; then + python3 ./prepare_layers/make_constant_habitat.py --examplar "${DATADIR}"/habitat_maps/arable/lcc_1401.tif \ + --habitat_code 14.5 \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat_maps/urban + fi + + if [ ! -f "${DATADIR}"/habitat/urban_diff_area.tif ]; then + python3 ./prepare_layers/make_constant_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --habitat_code 14.5 \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/urban_diff_area.tif + fi +fi # Fetch and prepare the elevation layers -reclaimer zenodo --zenodo_id 5719984 --filename dem-100m-esri54017.tif --output "${DATADIR}"/elevation.tif -gdalwarp -t_srs EPSG:4326 -tr 0.016666666666667 -0.016666666666667 -r max -co COMPRESS=LZW -wo NUM_THREADS=40 "${DATADIR}"/elevation.tif "${DATADIR}"/elevation-max.tif -gdalwarp -t_srs EPSG:4326 -tr 0.016666666666667 -0.016666666666667 -r min -co COMPRESS=LZW -wo NUM_THREADS=40 "${DATADIR}"/elevation.tif "${DATADIR}"/elevation-min.tif +if [ ! -f "${DATADIR}"/elevation.tif ]; then + reclaimer zenodo --zenodo_id 5719984 --filename dem-100m-esri54017.tif --output "${DATADIR}"/elevation.tif +fi +if [ ! -f "${DATADIR}"/elevation-max.tif ]; then + gdalwarp -t_srs EPSG:4326 -tr "${PIXEL_SCALE}" -"${PIXEL_SCALE}" -r max -co COMPRESS=LZW -wo NUM_THREADS=40 "${DATADIR}"/elevation.tif "${DATADIR}"/elevation-max.tif +fi +if [ ! -f "${DATADIR}"/elevation-min.tif ]; then + gdalwarp -t_srs EPSG:4326 -tr "${PIXEL_SCALE}" -"${PIXEL_SCALE}" -r min -co COMPRESS=LZW -wo NUM_THREADS=40 "${DATADIR}"/elevation.tif "${DATADIR}"/elevation-min.tif +fi # Get species data per taxa from IUCN data for TAXA in "${TAXAS[@]}" do - python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" --output "${DATADIR}"/species-info/"${TAXA}"/ --projection "EPSG:4326" + if [ -f "${DATADIR}"/overrides.csv ]; then + python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" \ + --output "${DATADIR}"/species-info/"${TAXA}"/ \ + --projection "EPSG:4326" \ + --overrides "${DATADIR}"/overrides.csv + else + python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" \ + --output "${DATADIR}"/species-info/"${TAXA}"/ \ + --projection "EPSG:4326" + fi done # Generate the batch job input CSVs @@ -176,7 +264,7 @@ python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" \ --scenarios "${SCENARIOS[@]}" # Calculate all the AoHs -littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat +littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv "${VIRTUAL_ENV}"/bin/aoh-calc -- --force-habitat # Generate validation summaries aoh-collate-data --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv