From bba531e35e7d5338469c0a5dbc432dc8ee3609ea Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Mon, 22 Sep 2025 12:54:30 +0000 Subject: [PATCH 1/9] Update LIFE to use aoh module from pip rather than git submodule. --- .gitmodules | 3 -- Dockerfile | 5 +-- aoh-calculator | 1 - prepare_species/common.py | 5 ++- requirements.txt | 9 ++++-- scripts/generate_food_map.sh | 6 ++-- scripts/run.sh | 60 ++++++++++++++++++------------------ scripts/slurm.sh | 48 ++++++++++++++--------------- 8 files changed, 68 insertions(+), 69 deletions(-) delete mode 160000 aoh-calculator diff --git a/.gitmodules b/.gitmodules index 30fd8a6..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "aoh-calculator"] - path = aoh-calculator - url = git@github.com:quantifyearth/aoh-calculator.git diff --git a/Dockerfile b/Dockerfile index d762380..1677173 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN git clone https://github.com/carboncredits/littlejohn.git WORKDIR littlejohn RUN go build -from ghcr.io/osgeo/gdal:ubuntu-small-3.10.1 +from ghcr.io/osgeo/gdal:ubuntu-small-3.11.4 COPY --from=littlejohn /go/littlejohn/littlejohn /bin/littlejohn @@ -12,6 +12,7 @@ RUN apt-get update -qqy && \ git \ libpq-dev \ python3-pip \ + R-base \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /var/cache/apt/* @@ -19,7 +20,7 @@ RUN apt-get update -qqy && \ # gdal's python bindings are sad. Pandas we full out as its slow # to build, and this means it'll be cached RUN pip install --break-system-packages numpy -RUN pip install --break-system-packages gdal[numpy]==3.10.1 +RUN pip install --break-system-packages gdal[numpy]==3.11.4 RUN pip install --break-system-packages pandas COPY requirements.txt /tmp/ diff --git a/aoh-calculator b/aoh-calculator deleted file mode 160000 index 262af5a..0000000 --- a/aoh-calculator +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 262af5addc305fa9ad6e753b4f2b738f1bf228e8 diff --git a/prepare_species/common.py b/prepare_species/common.py index 2d4be8e..3408d61 100644 --- a/prepare_species/common.py +++ b/prepare_species/common.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any, Dict, List, Set, Tuple +import aoh import geopandas as gpd import pyproj import shapely @@ -11,8 +12,6 @@ logging.basicConfig() logger.setLevel(logging.DEBUG) -aoh_cleaning = importlib.import_module("aoh-calculator.cleaning") - SEASON_NAME = { 1: "RESIDENT", 2: "BREEDING", @@ -205,7 +204,7 @@ def tidy_reproject_save( target_crs = src_crs #pyproj.CRS.from_string(target_projection) graw = gdf.loc[0].copy() - grow = aoh_cleaning.tidy_data(graw) + grow = aoh.tidy_data(graw) output_path = output_directory_path / f"{grow.id_no}_{grow.season}.geojson" res = gpd.GeoDataFrame(grow.to_frame().transpose(), crs=src_crs, geometry="geometry") res_projected = res.to_crs(target_crs) diff --git a/requirements.txt b/requirements.txt index 86c74f8..0da34ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +# general requirements geopandas numpy duckdb @@ -13,15 +14,17 @@ rasterio requests alive-progress matplotlib - +yirgacheffe +aoh gdal[numpy] - git+https://github.com/quantifyearth/iucn_modlib.git -git+https://github.com/quantifyearth/yirgacheffe git+https://github.com/quantifyearth/seasonality +# developer requirements pytest pylint mypy pandas-stubs types-requests +yirgacheffe[dev] +aoh[dev] diff --git a/scripts/generate_food_map.sh b/scripts/generate_food_map.sh index aa192c4..0398c20 100644 --- a/scripts/generate_food_map.sh +++ b/scripts/generate_food_map.sh @@ -61,9 +61,9 @@ fi # We need rescaled versions of the current data if [ ! -d "${DATADIR}"/food/current_layers ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/current_raw.tif \ - --scale 0.08333333333333333 \ - --output "${DATADIR}"/food/current_layers/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/current_raw.tif \ + --scale 0.08333333333333333 \ + --output "${DATADIR}"/food/current_layers/ fi # Combine GAEZ and HYDE data diff --git a/scripts/run.sh b/scripts/run.sh index 4df375e..1f59009 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -54,9 +54,9 @@ fi if [ ! -d "${DATADIR}"/habitat_maps/current ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/current_raw.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/current/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/current_raw.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/current/ fi # Get PNV layer and prepare for use @@ -68,9 +68,9 @@ if [ ! -f "${DATADIR}"/habitat/pnv_raw.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/pnv ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/pnv_raw.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/pnv/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/pnv_raw.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/pnv/ fi # Generate an area scaling map @@ -85,9 +85,9 @@ if [ ! -f "${DATADIR}"/habitat/arable.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/arable ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/arable.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/arable/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/arable/ fi if [ ! -f "${DATADIR}"/habitat/arable_diff_area.tif ]; then @@ -105,9 +105,9 @@ if [ ! -f "${DATADIR}"/habitat/pasture.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/pasture ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/pasture.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/pasture/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/pasture/ fi if [ ! -f "${DATADIR}"/habitat/pasture_diff_area.tif ]; then @@ -127,9 +127,9 @@ if [ ! -f "${DATADIR}"/habitat/restore.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/restore ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore/ fi if [ ! -f "${DATADIR}"/habitat/restore_diff_area.tif ]; then @@ -149,9 +149,9 @@ if [ ! -f "${DATADIR}"/habitat/restore_agriculture.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/restore_agriculture ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore_agriculture/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_agriculture/ fi if [ ! -f "${DATADIR}"/habitat/restore_agriculture_diff_area.tif ]; then @@ -171,9 +171,9 @@ if [ ! -f "${DATADIR}"/habitat/restore_all.tif ]; then fi if [ ! -d "${DATADIR}"/habitat_maps/restore_all ]; then - python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore_all.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore_all/ + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_all/ fi if [ ! -f "${DATADIR}"/habitat/restore_all_diff_area.tif ]; then @@ -223,22 +223,22 @@ python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" --output "${DATADIR}" python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" --curve "${CURVE}" --output "${DATADIR}"/persistencebatch.csv # Calculate all the AoHs -littlejohn -j 700 -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./aoh-calculator/aohcalc.py --force-habitat +littlejohn -j 700 -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat # Generate validation summaries -python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv -python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/pnv/ --output "${DATADIR}"/aohs/pnv.csv +aoh-collate-data --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv +aoh-collate-data --aoh_results "${DATADIR}"/aohs/pnv/ --output "${DATADIR}"/aohs/pnv.csv for SCENARIO in "${SCENARIOS[@]}" do - python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/"${SCENARIO}"/ --output "${DATADIR}"/aohs/"${SCENARIO}".csv + aoh-collate-data --aoh_results "${DATADIR}"/aohs/"${SCENARIO}"/ --output "${DATADIR}"/aohs/"${SCENARIO}".csv done # Calculate predictors from AoHs -python3 ./aoh-calculator/summaries/species_richness.py --aohs_folder "${DATADIR}"/aohs/current/ \ - --output "${DATADIR}"/predictors/species_richness.tif -python3 ./aoh-calculator/summaries/endemism.py --aohs_folder "${DATADIR}"/aohs/current/ \ - --species_richness "${DATADIR}"/predictors/species_richness.tif \ - --output "${DATADIR}"/predictors/endemism.tif +aoh-species-richness --aohs_folder "${DATADIR}"/aohs/current/ \ + --output "${DATADIR}"/predictors/species_richness.tif +aoh-endemism --aohs_folder "${DATADIR}"/aohs/current/ \ + --species_richness "${DATADIR}"/predictors/species_richness.tif \ + --output "${DATADIR}"/predictors/endemism.tif # Calculate the per species Delta P values littlejohn -j 200 -o "${DATADIR}"/persistencebatch.log -c "${DATADIR}"/persistencebatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./deltap/global_code_residents_pixel.py diff --git a/scripts/slurm.sh b/scripts/slurm.sh index 2d3e745..4884ce7 100644 --- a/scripts/slurm.sh +++ b/scripts/slurm.sh @@ -70,9 +70,9 @@ python3 ./prepare_layers/make_area_map.py --scale 0.016666666666667 --output "${ python3 ./prepare_layers/make_arable_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --output "${DATADIR}"/habitat/arable.tif -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/arable.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/arable/ +aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ + --scale 0.016666666666667 \ + --output "${DATADIR}"/habitat_maps/arable/ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --scenario "${DATADIR}"/habitat/arable.tif \ @@ -84,9 +84,9 @@ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current python3 ./prepare_layers/make_pasture_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --output "${DATADIR}"/habitat/pasture.tif -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/pasture.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/pasture/ +aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ + --scale 0.016666666666667 \ + --output "${DATADIR}"/habitat_maps/pasture/ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --scenario "${DATADIR}"/habitat/pasture.tif \ @@ -100,9 +100,9 @@ python3 ./prepare_layers/make_restore_map.py --pnv "${DATADIR}"/habitat/pnv_raw. --crosswalk "${DATADIR}"/crosswalk.csv \ --output "${DATADIR}"/habitat/restore.tif -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore/ +aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ + --scale 0.016666666666667 \ + --output "${DATADIR}"/habitat_maps/restore/ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --scenario "${DATADIR}"/habitat/restore.tif \ @@ -116,9 +116,9 @@ python3 ./prepare_layers/make_restore_agriculture_map.py --pnv "${DATADIR}"/habi --crosswalk "${DATADIR}"/crosswalk.csv \ --output "${DATADIR}"/habitat/restore_agriculture.tif -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore_agriculture/ +aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ + --scale 0.016666666666667 \ + --output "${DATADIR}"/habitat_maps/restore_agriculture/ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --scenario "${DATADIR}"/habitat/restore_agriculture.tif \ @@ -132,9 +132,9 @@ python3 ./prepare_layers/make_restore_all_map.py --pnv "${DATADIR}"/habitat/pnv_ --crosswalk "${DATADIR}"/crosswalk.csv \ --output "${DATADIR}"/habitat/restore_all.tif -python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/restore_all.tif \ - --scale 0.016666666666667 \ - --output "${DATADIR}"/habitat_maps/restore_all/ +aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ + --scale 0.016666666666667 \ + --output "${DATADIR}"/habitat_maps/restore_all/ python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ --scenario "${DATADIR}"/habitat/restore_all.tif \ @@ -171,22 +171,22 @@ python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" --output "${DATADIR}" python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" --curve "${CURVE}" --output "${DATADIR}"/persistencebatch.csv # Calculate all the AoHs -littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./aoh-calculator/aohcalc.py --force-habitat +littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat # Generate validation summaries -python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv -python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/pnv/ --output "${DATADIR}"/aohs/pnv.csv +aoh-collate-data --aoh_results "${DATADIR}"/aohs/current/ --output "${DATADIR}"/aohs/current.csv +aoh-collate-data --aoh_results "${DATADIR}"/aohs/pnv/ --output "${DATADIR}"/aohs/pnv.csv for SCENARIO in "${SCENARIOS[@]}" do - python3 ./aoh-calculator/validation/collate_data.py --aoh_results "${DATADIR}"/aohs/"${SCENARIO}"/ --output "${DATADIR}"/aohs/"${SCENARIO}".csv + aoh-collate-data --aoh_results "${DATADIR}"/aohs/"${SCENARIO}"/ --output "${DATADIR}"/aohs/"${SCENARIO}".csv done # Calculate predictors from AoHs -python3 ./aoh-calculator/summaries/species_richness.py --aohs_folder "${DATADIR}"/aohs/current/ \ - --output "${DATADIR}"/predictors/species_richness.tif -python3 ./aoh-calculator/summaries/endemism.py --aohs_folder "${DATADIR}"/aohs/current/ \ - --species_richness "${DATADIR}"/predictors/species_richness.tif \ - --output "${DATADIR}"/predictors/endemism.tif +aoh-species-richness --aohs_folder "${DATADIR}"/aohs/current/ \ + --output "${DATADIR}"/predictors/species_richness.tif +aoh-endemism --aohs_folder "${DATADIR}"/aohs/current/ \ + --species_richness "${DATADIR}"/predictors/species_richness.tif \ + --output "${DATADIR}"/predictors/endemism.tif # Calculate the per species Delta P values littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/persistencebatch.log -c "${DATADIR}"/persistencebatch.csv "${VIRTUAL_ENV}"/bin/python3 -- ./deltap/global_code_residents_pixel.py From 4108a2c9e685959fa311051deb90807a28ce6391 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Mon, 22 Sep 2025 13:35:17 +0000 Subject: [PATCH 2/9] Tweak how scenarios are passed to generator scripts. --- scripts/run.sh | 9 +++++++-- scripts/slurm.sh | 9 +++++++-- utils/persistencegenerator.py | 13 +++++++++++-- utils/speciesgenerator.py | 26 +++++++++++++++++++++----- 4 files changed, 46 insertions(+), 11 deletions(-) diff --git a/scripts/run.sh b/scripts/run.sh index 1f59009..78e1bae 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -219,8 +219,13 @@ do done # Generate the batch job input CSVs -python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" --output "${DATADIR}"/aohbatch.csv -python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" --curve "${CURVE}" --output "${DATADIR}"/persistencebatch.csv +python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" \ + --output "${DATADIR}"/aohbatch.csv \ + --scenarios "${SCENARIOS[@]}" +python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" \ + --curve "${CURVE}" \ + --output "${DATADIR}"/persistencebatch.csv \ + --scenarios "${SCENARIOS[@]}" # Calculate all the AoHs littlejohn -j 700 -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat diff --git a/scripts/slurm.sh b/scripts/slurm.sh index 4884ce7..e3db904 100644 --- a/scripts/slurm.sh +++ b/scripts/slurm.sh @@ -167,8 +167,13 @@ do done # Generate the batch job input CSVs -python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" --output "${DATADIR}"/aohbatch.csv -python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" --curve "${CURVE}" --output "${DATADIR}"/persistencebatch.csv +python3 ./utils/speciesgenerator.py --datadir "${DATADIR}" \ + --output "${DATADIR}"/aohbatch.csv \ + --scenarios "${SCENARIOS[@]}" +python3 ./utils/persistencegenerator.py --datadir "${DATADIR}" \ + --curve "${CURVE}" \ + --output "${DATADIR}"/persistencebatch.csv \ + --scenarios "${SCENARIOS[@]}" # Calculate all the AoHs littlejohn -j "${SLURM_JOB_CPUS_PER_NODE}" -o "${DATADIR}"/aohbatch.log -c "${DATADIR}"/aohbatch.csv aoh-calc -- --force-habitat diff --git a/utils/persistencegenerator.py b/utils/persistencegenerator.py index 81f323a..77fea17 100644 --- a/utils/persistencegenerator.py +++ b/utils/persistencegenerator.py @@ -8,6 +8,7 @@ def species_generator( data_dir: Path, curve: str, output_csv_path: Path, + scenarios: List[str], ): species_info_dir = data_dir / "species-info" taxas = [x.name for x in species_info_dir.iterdir()] @@ -19,7 +20,7 @@ def species_generator( for taxa in taxas: taxa_path = species_info_dir / taxa / "current" speciess = list(taxa_path.glob("*.geojson")) - for scenario in ['restore_all', 'urban', 'arable', 'pasture', 'restore', 'restore_agriculture']: + for scenario in scenarios: for species in speciess: res.append([ species, @@ -65,9 +66,17 @@ def main() -> None: required=True, dest="output" ) + parser.add_argument( + '--scenarios', + nargs='*', + type=str, + help="list of scenarios to calculate LIFE for" + required=True, + dest="scenarios", + ) args = parser.parse_args() - species_generator(args.data_dir, args.curve, args.output) + species_generator(args.data_dir, args.curve, args.output, args.scenarios) if __name__ == "__main__": main() diff --git a/utils/speciesgenerator.py b/utils/speciesgenerator.py index 435e305..95a1571 100644 --- a/utils/speciesgenerator.py +++ b/utils/speciesgenerator.py @@ -1,26 +1,35 @@ import argparse +import sys from pathlib import Path import pandas as pd -SCENARIOS = ['current', 'restore', 'arable', 'pnv', 'restore_all', 'urban', 'pasture', 'restore_agriculture'] +DEFAULT_SCENARIOS = ["current", "pnv"] def species_generator( data_dir: Path, output_csv_path: Path, + scenarios: List[str], ): species_info_dir = data_dir / "species-info" taxas = [x.name for x in species_info_dir.iterdir()] res = [] for taxa in taxas: - for scenario in SCENARIOS: + for scenario in scenarios: + habitat_maps_path = data_dir / "habitat_maps" / scenario + if not habitat_maps_path.exists(): + sys.exit(f"Expected to find habitat maps in {habitat_maps_path}") + source = 'historic' if scenario == 'pnv' else 'current' taxa_path = species_info_dir / taxa / source + if not taxa_path.exists(): + sys.exit(f"Expected to find list of species in {taxa_path}") + speciess = taxa_path.glob("*.geojson") for species in speciess: res.append([ - data_dir / "habitat_maps" / scenario, + habitat_maps_path, data_dir / "elevation-max.tif", data_dir / "elevation-min.tif", data_dir / "area-per-pixel.tif", @@ -40,7 +49,6 @@ def species_generator( ]) df.to_csv(output_csv_path, index=False) - def main() -> None: parser = argparse.ArgumentParser(description="Species and seasonality generator.") parser.add_argument( @@ -57,9 +65,17 @@ def main() -> None: required=True, dest="output" ) + parser.add_argument( + '--scenarios', + nargs='*', + type=str, + help="list of scenarios to calculate LIFE for" + required=True, + dest="scenarios", + ) args = parser.parse_args() - species_generator(args.data_dir, args.output) + species_generator(args.data_dir, args.output, args.scenarios + DEFAULT_SCENARIOS) if __name__ == "__main__": main() From 6732b470b411fa4cf7ff2710d460f636a28a4ed5 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Mon, 22 Sep 2025 15:54:20 +0000 Subject: [PATCH 3/9] WIP: add overrides for species info --- prepare_species/common.py | 2 +- prepare_species/extract_species_psql.py | 28 ++++++++++++++++++++++--- tests/test_species_filter.py | 11 ++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/prepare_species/common.py b/prepare_species/common.py index 3408d61..bd934e8 100644 --- a/prepare_species/common.py +++ b/prepare_species/common.py @@ -39,6 +39,7 @@ class SpeciesReport: "id_no", "assessment_id", "scientific_name", + "overriden", "has_systems", "not_marine", "has_habitats", @@ -75,7 +76,6 @@ def __getattr__(self, name: str) -> Any: def as_row(self) -> List: return [self.info[k] for k in self.REPORT_COLUMNS] - def process_systems( systems_data: List[Tuple], report: SpeciesReport, diff --git a/prepare_species/extract_species_psql.py b/prepare_species/extract_species_psql.py index 4a56e9f..264bd1a 100644 --- a/prepare_species/extract_species_psql.py +++ b/prepare_species/extract_species_psql.py @@ -4,7 +4,7 @@ from functools import partial from multiprocessing import Pool from pathlib import Path -from typing import Optional, Tuple +from typing import Optional, Set, Tuple # import pyshark # pylint: disable=W0611 import pandas as pd @@ -100,6 +100,7 @@ def process_row( class_name: str, + overrides: Set[int], output_directory_path: Path, presence: Tuple[int,...], row: Tuple, @@ -110,6 +111,8 @@ def process_row( (id_no, assessment_id, _elevation_lower, _elevation_upper, scientific_name, _family_name, _threat_code) = row report = SpeciesReport(id_no, assessment_id, scientific_name) + if id_no in overrides: + report.overriden = True cursor.execute(SYSTEMS_STATEMENT, (assessment_id,)) systems_data = cursor.fetchall() @@ -152,6 +155,7 @@ def extract_data_per_species( class_name: str, output_directory_path: Path, _target_projection: Optional[str], + overrides_path: Optional[Path], ) -> None: connection = psycopg2.connect(DB_CONFIG) @@ -164,13 +168,23 @@ def extract_data_per_species( logger.info("Found %d species in class %s", len(results), class_name) + if overrides_path is not None: + try: + overrides_df = pd.read_csv(overrides_path) + overrides = set(list(overrides_df.id_no)) + except FileNotFoundError: + sys.exit(f"Failed to find overrides at {overrides_path}") + logger.info("Found %d species overrides", len(overrides)) + else: + overrides={} + for era, presence in [("current", (1, 2)), ("historic", (1, 2, 4, 5))]: era_output_directory_path = output_directory_path / era os.makedirs(era_output_directory_path, exist_ok=True) # The limiting amount here is how many concurrent connections the database can take with Pool(processes=20) as pool: - reports = pool.map(partial(process_row, class_name, era_output_directory_path, presence), results) + reports = pool.map(partial(process_row, class_name, overrides, era_output_directory_path, presence), results) reports_df = pd.DataFrame( [x.as_row() for x in reports], @@ -202,12 +216,20 @@ def main() -> None: dest="target_projection", default="ESRI:54017" ) + parser.add_argument( + '--overrides', + type=Path, + help="CSV of species which should be included despite failing other checks.", + required=False, + dest="overrides_path", + ) args = parser.parse_args() extract_data_per_species( args.classname, args.output_directory_path, - args.target_projection + args.target_projection, + args.overrides_path, ) if __name__ == "__main__": diff --git a/tests/test_species_filter.py b/tests/test_species_filter.py index a27057e..058ab1b 100644 --- a/tests/test_species_filter.py +++ b/tests/test_species_filter.py @@ -8,6 +8,7 @@ def test_empty_report() -> None: assert report.id_no == 1 assert report.assessment_id == 2 assert report.scientific_name == "name" + assert not report.overriden assert not report.has_systems assert not report.not_marine assert not report.has_habitats @@ -237,6 +238,16 @@ def test_reject_if_marine_in_system(): assert report.has_systems assert not report.not_marine +def test_reject_if_marine_in_system_with_override(): + systems_data = [ + ("Terrestrial|Marine",) + ] + report = SpeciesReport(1, 2, "name") + report.overriden = True + process_systems(systems_data, report) + assert report.has_systems + assert not report.not_marine + def test_pass_if_marine_not_in_system(): systems_data = [ ("Terrestrial",) From 25d1027eb0bc72416b438fbc9a9328068d2a3607 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Mon, 22 Sep 2025 17:04:53 +0100 Subject: [PATCH 4/9] Fix typo in species option --- utils/persistencegenerator.py | 2 +- utils/speciesgenerator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/persistencegenerator.py b/utils/persistencegenerator.py index 77fea17..0b910b5 100644 --- a/utils/persistencegenerator.py +++ b/utils/persistencegenerator.py @@ -70,7 +70,7 @@ def main() -> None: '--scenarios', nargs='*', type=str, - help="list of scenarios to calculate LIFE for" + help="list of scenarios to calculate LIFE for", required=True, dest="scenarios", ) diff --git a/utils/speciesgenerator.py b/utils/speciesgenerator.py index 95a1571..ad0db02 100644 --- a/utils/speciesgenerator.py +++ b/utils/speciesgenerator.py @@ -69,7 +69,7 @@ def main() -> None: '--scenarios', nargs='*', type=str, - help="list of scenarios to calculate LIFE for" + help="list of scenarios to calculate LIFE for", required=True, dest="scenarios", ) From c48499120763d0eb89b57288610e66c52819b773 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Mon, 22 Sep 2025 17:20:46 +0100 Subject: [PATCH 5/9] Allow overrides for system check on species --- prepare_species/common.py | 26 +++++++++++++++----------- tests/test_species_filter.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/prepare_species/common.py b/prepare_species/common.py index bd934e8..eff943f 100644 --- a/prepare_species/common.py +++ b/prepare_species/common.py @@ -81,17 +81,21 @@ def process_systems( report: SpeciesReport, ) -> None: if len(systems_data) == 0: - raise ValueError("No systems found") - if len(systems_data) > 1: - raise ValueError("More than one systems aggregation found") - systems = systems_data[0][0] - if systems is None: - raise ValueError("no systems info") - report.has_systems = True - - if "Marine" in systems: - raise ValueError("Marine in systems") - report.not_marine = True + if not report.overriden: + raise ValueError("No systems found") + else: + if len(systems_data) > 1: + # We don't allow override on this as it's a programmer/database error + raise ValueError("More than one systems aggregation found") + systems = systems_data[0][0] + if systems is None and not report.overriden: + raise ValueError("no systems info") + report.has_systems = (len(systems_data) == 1) and (systems is not None) + + has_marine_in_systems = "Marine" in systems + if has_marine_in_systems and not report.overriden: + raise ValueError("Marine in systems") + report.not_marine = not has_marine_in_systems def process_habitats( habitats_data: List[Tuple], diff --git a/tests/test_species_filter.py b/tests/test_species_filter.py index 058ab1b..8631147 100644 --- a/tests/test_species_filter.py +++ b/tests/test_species_filter.py @@ -228,6 +228,23 @@ def test_inverted_13394_habitat_filter(): assert report.not_major_caves assert report.not_major_freshwater_lakes +def test_reject_if_no_systems(): + systems_data = [] + report = SpeciesReport(1, 2, "name") + with pytest.raises(ValueError): + process_systems(systems_data, report) + assert not report.has_systems + assert not report.not_marine + +def test_reject_if_no_systems_with_overriden(): + systems_data = [] + report = SpeciesReport(1, 2, "name") + report.overriden = True + process_systems(systems_data, report) + # No exception from above + assert not report.has_systems + assert not report.not_marine + def test_reject_if_marine_in_system(): systems_data = [ ("Terrestrial|Marine",) @@ -254,5 +271,6 @@ def test_pass_if_marine_not_in_system(): ] report = SpeciesReport(1, 2, "name") process_systems(systems_data, report) + # No exception from above assert report.has_systems assert report.not_marine From e0fe0a31e0c4df2bbf9e90c676bc9ff2fe51f04d Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Tue, 23 Sep 2025 08:31:59 +0100 Subject: [PATCH 6/9] Allow habitat overrides also --- prepare_species/common.py | 19 +++++++++++-------- tests/test_species_filter.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/prepare_species/common.py b/prepare_species/common.py index eff943f..4a14f77 100644 --- a/prepare_species/common.py +++ b/prepare_species/common.py @@ -155,14 +155,17 @@ def process_habitats( major_habitats_lvl_1 = {k: {int(v) for v in x} for k, x in major_habitats.items()} - for _, season_major_habitats_lvl1 in major_habitats_lvl_1.items(): - if 7 in season_major_habitats_lvl1: - raise ValueError("Habitat 7 in major importance habitat list") - report.not_major_caves = True - for _, season_major_habitats in major_habitats.items(): - if not season_major_habitats - set([5.1, 5.5, 5.6, 5.14, 5.16]): - raise ValueError("Freshwater lakes are major habitat") - report.not_major_freshwater_lakes = True + major_caves = any([7 in x for x in major_habitats_lvl_1.values()]) + if major_caves and not report.overriden: + raise ValueError("Habitat 7 in major importance habitat list") + report.not_major_caves = not major_caves + + major_freshwater_lakes = any([ + not (x - {5.1, 5.5, 5.6, 5.14, 5.16}) for x in major_habitats.values() + ]) + if major_freshwater_lakes and not report.overriden: + raise ValueError("Freshwater lakes are major habitat") + report.not_major_freshwater_lakes = not major_freshwater_lakes return habitats diff --git a/tests/test_species_filter.py b/tests/test_species_filter.py index 8631147..5d73def 100644 --- a/tests/test_species_filter.py +++ b/tests/test_species_filter.py @@ -85,6 +85,19 @@ def test_reject_if_caves_in_major_habitat(): assert report.keeps_habitats assert not report.not_major_caves +def test_reject_if_caves_in_major_habitat_overriden(): + habitat_data = [ + ("resident", "Yes", "4.1|7.2"), + ("resident", "No", "4.3"), + ] + report = SpeciesReport(1, 2, "name") + report.overriden = True + _ = process_habitats(habitat_data, report) + # No exception from above + assert report.has_habitats + assert report.keeps_habitats + assert not report.not_major_caves + def test_do_not_reject_if_caves_in_minor_habitat(): habitat_data = [ ("resident", "Yes", "4.1|4.2"), @@ -196,6 +209,19 @@ def test_13394_habitat_filter(): assert report.keeps_habitats assert not report.not_major_freshwater_lakes +def test_13394_habitat_filter_with_overrides(): + habitat_data = [ + ("resident", "Yes", "5.1"), + ("resident", "No", "1.6|1.9"), + ] + report = SpeciesReport(1, 2, "name") + report.overriden = True + _ = process_habitats(habitat_data, report) + # No exception from above + assert report.has_habitats + assert report.keeps_habitats + assert not report.not_major_freshwater_lakes + def test_similar_13394_habitat_filter(): habitat_data = [ ("resident", "Yes", "5.1|1.7"), From 239371c747497b1a851cea46c471d9c9dd7d364c Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Tue, 23 Sep 2025 08:48:59 +0100 Subject: [PATCH 7/9] Only build habitat maps in run.sh for scenarios we actually run --- scripts/run.sh | 225 ++++++++++++++++++++++++++++--------------------- 1 file changed, 128 insertions(+), 97 deletions(-) diff --git a/scripts/run.sh b/scripts/run.sh index 78e1bae..d2d8115 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -8,6 +8,7 @@ # https://github.com/quantifyearth/littlejohn - used to run batch jobs in parallel set -e +set -x if [ -z "${DATADIR}" ]; then echo "Please specify $DATADIR" @@ -25,6 +26,16 @@ declare -a TAXAS=("AMPHIBIA" "AVES" "MAMMALIA" "REPTILIA") export CURVE=0.25 export PIXEL_SCALE=0.016666666666667 +check_scenario() { + local target="$1" + for scenario in "${SCENARIOS[@]}"; do + if [[ "$scenario" == "$target" ]]; then + return 0 + fi + done + return 1 +} + if [ ! -f "${DATADIR}"/crosswalk.csv ]; then python3 ./prepare_layers/generate_crosswalk.py --output "${DATADIR}"/crosswalk.csv fi @@ -52,7 +63,6 @@ if [ ! -f "${DATADIR}"/habitat/current_raw.tif ]; then -j 16 fi - if [ ! -d "${DATADIR}"/habitat_maps/current ]; then aoh-habitat-process --habitat "${DATADIR}"/habitat/current_raw.tif \ --scale "${PIXEL_SCALE}" \ @@ -79,126 +89,138 @@ if [ ! -f "${DATADIR}"/area-per-pixel.tif ]; then fi # Generate the arable scenario map -if [ ! -f "${DATADIR}"/habitat/arable.tif ]; then - python3 ./prepare_layers/make_arable_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --output "${DATADIR}"/habitat/arable.tif -fi +if check_scenario "arable"; then + if [ ! -f "${DATADIR}"/habitat/arable.tif ]; then + python3 ./prepare_layers/make_arable_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --output "${DATADIR}"/habitat/arable.tif + fi -if [ ! -d "${DATADIR}"/habitat_maps/arable ]; then - aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/arable/ -fi + if [ ! -d "${DATADIR}"/habitat_maps/arable ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/arable.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/arable/ + fi -if [ ! -f "${DATADIR}"/habitat/arable_diff_area.tif ]; then - python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/arable.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/arable_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/arable_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/arable.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/arable_diff_area.tif + fi fi # Generate the pasture scenario map -if [ ! -f "${DATADIR}"/habitat/pasture.tif ]; then - python3 ./prepare_layers/make_pasture_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --output "${DATADIR}"/habitat/pasture.tif -fi +if check_scenario "pasture"; then + if [ ! -f "${DATADIR}"/habitat/pasture.tif ]; then + python3 ./prepare_layers/make_pasture_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --output "${DATADIR}"/habitat/pasture.tif + fi -if [ ! -d "${DATADIR}"/habitat_maps/pasture ]; then - aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/pasture/ -fi + if [ ! -d "${DATADIR}"/habitat_maps/pasture ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/pasture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/pasture/ + fi -if [ ! -f "${DATADIR}"/habitat/pasture_diff_area.tif ]; then - python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/pasture.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/pasture_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/pasture_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/pasture.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/pasture_diff_area.tif + fi fi # Generate the restore map -if [ ! -f "${DATADIR}"/habitat/restore.tif ]; then - python3 ./prepare_layers/make_restore_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore.tif -fi +if check_scenario "restore"; then + if [ ! -f "${DATADIR}"/habitat/restore.tif ]; then + python3 ./prepare_layers/make_restore_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore.tif + fi -if [ ! -d "${DATADIR}"/habitat_maps/restore ]; then - aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore/ -fi + if [ ! -d "${DATADIR}"/habitat_maps/restore ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore/ + fi -if [ ! -f "${DATADIR}"/habitat/restore_diff_area.tif ]; then - python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/restore_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/restore_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_diff_area.tif + fi fi -# Generate the restore map -if [ ! -f "${DATADIR}"/habitat/restore_agriculture.tif ]; then - python3 ./prepare_layers/make_restore_agriculture_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore_agriculture.tif -fi +# Generate the restore_agriculture map +if check_scenario "restore_agriculture"; then + if [ ! -f "${DATADIR}"/habitat/restore_agriculture.tif ]; then + python3 ./prepare_layers/make_restore_agriculture_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore_agriculture.tif + fi -if [ ! -d "${DATADIR}"/habitat_maps/restore_agriculture ]; then - aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore_agriculture/ -fi + if [ ! -d "${DATADIR}"/habitat_maps/restore_agriculture ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_agriculture.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_agriculture/ + fi -if [ ! -f "${DATADIR}"/habitat/restore_agriculture_diff_area.tif ]; then - python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore_agriculture.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/restore_agriculture_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/restore_agriculture_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore_agriculture.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_agriculture_diff_area.tif + fi fi # Generate the restore all map -if [ ! -f "${DATADIR}"/habitat/restore_all.tif ]; then - python3 ./prepare_layers/make_restore_all_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ - --current "${DATADIR}"/habitat/current_raw.tif \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat/restore_all.tif -fi +if check_scenario "restore_all"; then + if [ ! -f "${DATADIR}"/habitat/restore_all.tif ]; then + python3 ./prepare_layers/make_restore_all_map.py --pnv "${DATADIR}"/habitat/pnv_raw.tif \ + --current "${DATADIR}"/habitat/current_raw.tif \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat/restore_all.tif + fi -if [ ! -d "${DATADIR}"/habitat_maps/restore_all ]; then - aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat_maps/restore_all/ -fi + if [ ! -d "${DATADIR}"/habitat_maps/restore_all ]; then + aoh-habitat-process --habitat "${DATADIR}"/habitat/restore_all.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat_maps/restore_all/ + fi -if [ ! -f "${DATADIR}"/habitat/restore_all_diff_area.tif ]; then - python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --scenario "${DATADIR}"/habitat/restore_all.tif \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/restore_all_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/restore_all_diff_area.tif ]; then + python3 ./prepare_layers/make_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --scenario "${DATADIR}"/habitat/restore_all.tif \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/restore_all_diff_area.tif + fi fi # Generate urban all map -if [ ! -d "${DATADIR}"/habitat_maps/urban ]; then - python3 ./prepare_layers/make_constant_habitat.py --examplar "${DATADIR}"/habitat_maps/arable/lcc_1401.tif \ - --habitat_code 14.5 \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --output "${DATADIR}"/habitat_maps/urban -fi +if check_scenario "urban"; then + if [ ! -d "${DATADIR}"/habitat_maps/urban ]; then + python3 ./prepare_layers/make_constant_habitat.py --examplar "${DATADIR}"/habitat_maps/arable/lcc_1401.tif \ + --habitat_code 14.5 \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --output "${DATADIR}"/habitat_maps/urban + fi -if [ ! -f "${DATADIR}"/habitat/urban_diff_area.tif ]; then - python3 ./prepare_layers/make_constant_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ - --habitat_code 14.5 \ - --crosswalk "${DATADIR}"/crosswalk.csv \ - --area "${DATADIR}"/area-per-pixel.tif \ - --scale "${PIXEL_SCALE}" \ - --output "${DATADIR}"/habitat/urban_diff_area.tif + if [ ! -f "${DATADIR}"/habitat/urban_diff_area.tif ]; then + python3 ./prepare_layers/make_constant_diff_map.py --current "${DATADIR}"/habitat/current_raw.tif \ + --habitat_code 14.5 \ + --crosswalk "${DATADIR}"/crosswalk.csv \ + --area "${DATADIR}"/area-per-pixel.tif \ + --scale "${PIXEL_SCALE}" \ + --output "${DATADIR}"/habitat/urban_diff_area.tif + fi fi # Fetch and prepare the elevation layers @@ -215,7 +237,16 @@ fi # Get species data per taxa from IUCN data for TAXA in "${TAXAS[@]}" do - python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" --output "${DATADIR}"/species-info/"${TAXA}"/ --projection "EPSG:4326" + if [ ! -f "${DATADIR}"/overrides.csv ]; then + python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" \ + --output "${DATADIR}"/species-info/"${TAXA}"/ \ + --projection "EPSG:4326" \ + --overrides "${DATADIR}"/overrides.csv + else + python3 ./prepare_species/extract_species_psql.py --class "${TAXA}" \ + --output "${DATADIR}"/species-info/"${TAXA}"/ \ + --projection "EPSG:4326" + fi done # Generate the batch job input CSVs From e0ccc660157aeb297478c0b153e1b201796b7f49 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Tue, 23 Sep 2025 10:26:16 +0100 Subject: [PATCH 8/9] Fix pylint concerns --- prepare_species/common.py | 7 +++---- prepare_species/extract_species_psql.py | 16 +++++++++++++--- utils/persistencegenerator.py | 1 + utils/speciesgenerator.py | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/prepare_species/common.py b/prepare_species/common.py index 4a14f77..6a60a56 100644 --- a/prepare_species/common.py +++ b/prepare_species/common.py @@ -1,4 +1,3 @@ -import importlib import logging from pathlib import Path from typing import Any, Dict, List, Set, Tuple @@ -155,14 +154,14 @@ def process_habitats( major_habitats_lvl_1 = {k: {int(v) for v in x} for k, x in major_habitats.items()} - major_caves = any([7 in x for x in major_habitats_lvl_1.values()]) + major_caves = any(7 in x for x in major_habitats_lvl_1.values()) if major_caves and not report.overriden: raise ValueError("Habitat 7 in major importance habitat list") report.not_major_caves = not major_caves - major_freshwater_lakes = any([ + major_freshwater_lakes = any( not (x - {5.1, 5.5, 5.6, 5.14, 5.16}) for x in major_habitats.values() - ]) + ) if major_freshwater_lakes and not report.overriden: raise ValueError("Freshwater lakes are major habitat") report.not_major_freshwater_lakes = not major_freshwater_lakes diff --git a/prepare_species/extract_species_psql.py b/prepare_species/extract_species_psql.py index 264bd1a..d8fd55f 100644 --- a/prepare_species/extract_species_psql.py +++ b/prepare_species/extract_species_psql.py @@ -1,6 +1,7 @@ import argparse import logging import os +import sys from functools import partial from multiprocessing import Pool from pathlib import Path @@ -171,12 +172,12 @@ def extract_data_per_species( if overrides_path is not None: try: overrides_df = pd.read_csv(overrides_path) - overrides = set(list(overrides_df.id_no)) + overrides: Set[int] = set(list(overrides_df.id_no)) except FileNotFoundError: sys.exit(f"Failed to find overrides at {overrides_path}") logger.info("Found %d species overrides", len(overrides)) else: - overrides={} + overrides=set() for era, presence in [("current", (1, 2)), ("historic", (1, 2, 4, 5))]: era_output_directory_path = output_directory_path / era @@ -184,7 +185,16 @@ def extract_data_per_species( # The limiting amount here is how many concurrent connections the database can take with Pool(processes=20) as pool: - reports = pool.map(partial(process_row, class_name, overrides, era_output_directory_path, presence), results) + reports = pool.map( + partial( + process_row, + class_name, + overrides, + era_output_directory_path, + presence + ), + results, + ) reports_df = pd.DataFrame( [x.as_row() for x in reports], diff --git a/utils/persistencegenerator.py b/utils/persistencegenerator.py index 0b910b5..d8464c1 100644 --- a/utils/persistencegenerator.py +++ b/utils/persistencegenerator.py @@ -1,6 +1,7 @@ import argparse import sys from pathlib import Path +from typing import List import pandas as pd diff --git a/utils/speciesgenerator.py b/utils/speciesgenerator.py index ad0db02..a6cec90 100644 --- a/utils/speciesgenerator.py +++ b/utils/speciesgenerator.py @@ -1,6 +1,7 @@ import argparse import sys from pathlib import Path +from typing import List import pandas as pd From 96424100bf937e20ceee4e1333db699f34dd0db0 Mon Sep 17 00:00:00 2001 From: Michael Dales Date: Tue, 23 Sep 2025 12:02:18 +0100 Subject: [PATCH 9/9] Fix typing concerns --- deltap/delta_p_scaled.py | 1 + deltap/global_code_residents_pixel.py | 7 ++++--- prepare_layers/make_area_map.py | 3 +-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deltap/delta_p_scaled.py b/deltap/delta_p_scaled.py index 3c72977..aed4923 100644 --- a/deltap/delta_p_scaled.py +++ b/deltap/delta_p_scaled.py @@ -52,6 +52,7 @@ def delta_p_scaled_area( scaled_filtered_layer.parallel_save(result, band=1) for idx, inlayer in enumerate(per_taxa): + assert inlayer.name _, name = os.path.split(inlayer.name) taxa = name[:-4] species_count = int(total_counts[total_counts.taxa==taxa]["count"].values[0]) diff --git a/deltap/global_code_residents_pixel.py b/deltap/global_code_residents_pixel.py index 00657a2..6bb310d 100644 --- a/deltap/global_code_residents_pixel.py +++ b/deltap/global_code_residents_pixel.py @@ -5,6 +5,7 @@ import sys from enum import Enum from tempfile import TemporaryDirectory +from typing import Union import geopandas as gpd import numpy as np @@ -26,7 +27,7 @@ def gen_gompertz(x: float) -> float: def numpy_gompertz(x: float) -> float: return np.exp(-np.exp(GOMPERTZ_A + (GOMPERTZ_B * (x ** GOMPERTZ_ALPHA)))) -def open_layer_as_float64(filename: str) -> RasterLayer: +def open_layer_as_float64(filename: str) -> Union[ConstantLayer,RasterLayer]: if filename == "nan": return ConstantLayer(0.0) layer = RasterLayer.layer_from_file(filename) @@ -42,8 +43,8 @@ def calc_persistence_value(current_aoh: float, historic_aoh: float, exponent_fun return sp_p_fix def process_delta_p( - current: RasterLayer, - scenario: RasterLayer, + current: Union[ConstantLayer,RasterLayer], + scenario: Union[ConstantLayer,RasterLayer], current_aoh: float, historic_aoh: float, exponent_func_raster diff --git a/prepare_layers/make_area_map.py b/prepare_layers/make_area_map.py index cf047db..3a2ca29 100644 --- a/prepare_layers/make_area_map.py +++ b/prepare_layers/make_area_map.py @@ -44,8 +44,7 @@ def make_area_map( pixels = [0.0,] * math.floor(90.0 / pixel_scale) for i in range(len(pixels)): # pylint: disable=C0200 y = (i + 0.5) * pixel_scale - area = area_of_pixel(pixel_scale, y) - pixels[i] = area + pixels[i] = area_of_pixel(pixel_scale, y) allpixels = np.rot90(np.array([list(reversed(pixels)) + pixels]))