Updates and overrides

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "aoh-calculator"]
-	path = aoh-calculator
-	url = git@github.com:quantifyearth/aoh-calculator.git

Dockerfile

@@ -3,7 +3,7 @@ RUN git clone https://github.com/carboncredits/littlejohn.git
 WORKDIR littlejohn
 RUN go build
 
-from  ghcr.io/osgeo/gdal:ubuntu-small-3.10.1
+from  ghcr.io/osgeo/gdal:ubuntu-small-3.11.4
 
 COPY --from=littlejohn /go/littlejohn/littlejohn /bin/littlejohn
 
@@ -12,14 +12,15 @@ RUN apt-get update -qqy && \
 		git \
 		libpq-dev \
 		python3-pip \
+		R-base \
 	&& rm -rf /var/lib/apt/lists/* \
 	&& rm -rf /var/cache/apt/*
 
 # You must install numpy before anything else otherwise
 # gdal's python bindings are sad. Pandas we full out as its slow
 # to build, and this means it'll be cached
 RUN pip install --break-system-packages numpy
-RUN pip install --break-system-packages gdal[numpy]==3.10.1
+RUN pip install --break-system-packages gdal[numpy]==3.11.4
 RUN pip install --break-system-packages pandas
 
 COPY requirements.txt /tmp/

deltap/delta_p_scaled.py

@@ -52,6 +52,7 @@ def delta_p_scaled_area(
         scaled_filtered_layer.parallel_save(result, band=1)
 
         for idx, inlayer in enumerate(per_taxa):
+            assert inlayer.name
             _, name = os.path.split(inlayer.name)
             taxa = name[:-4]
             species_count = int(total_counts[total_counts.taxa==taxa]["count"].values[0])

deltap/global_code_residents_pixel.py

@@ -5,6 +5,7 @@
 import sys
 from enum import Enum
 from tempfile import TemporaryDirectory
+from typing import Union
 
 import geopandas as gpd
 import numpy as np
@@ -26,7 +27,7 @@ def gen_gompertz(x: float) -> float:
 def numpy_gompertz(x: float) -> float:
     return np.exp(-np.exp(GOMPERTZ_A + (GOMPERTZ_B * (x ** GOMPERTZ_ALPHA))))
 
-def open_layer_as_float64(filename: str) -> RasterLayer:
+def open_layer_as_float64(filename: str) -> Union[ConstantLayer,RasterLayer]:
     if filename == "nan":
         return ConstantLayer(0.0)
     layer = RasterLayer.layer_from_file(filename)
@@ -42,8 +43,8 @@ def calc_persistence_value(current_aoh: float, historic_aoh: float, exponent_fun
     return sp_p_fix
 
 def process_delta_p(
-    current: RasterLayer,
-    scenario: RasterLayer,
+    current: Union[ConstantLayer,RasterLayer],
+    scenario: Union[ConstantLayer,RasterLayer],
     current_aoh: float,
     historic_aoh: float,
     exponent_func_raster

prepare_layers/make_area_map.py

@@ -44,8 +44,7 @@ def make_area_map(
     pixels = [0.0,] * math.floor(90.0 / pixel_scale)
     for i in range(len(pixels)):  # pylint: disable=C0200
         y = (i + 0.5) * pixel_scale
-        area = area_of_pixel(pixel_scale, y)
-        pixels[i] = area
+        pixels[i] = area_of_pixel(pixel_scale, y)
 
     allpixels = np.rot90(np.array([list(reversed(pixels)) + pixels]))
 

prepare_species/common.py

@@ -1,8 +1,8 @@
-import importlib
 import logging
 from pathlib import Path
 from typing import Any, Dict, List, Set, Tuple
 
+import aoh
 import geopandas as gpd
 import pyproj
 import shapely
@@ -11,8 +11,6 @@
 logging.basicConfig()
 logger.setLevel(logging.DEBUG)
 
-aoh_cleaning = importlib.import_module("aoh-calculator.cleaning")
-
 SEASON_NAME = {
     1: "RESIDENT",
     2: "BREEDING",
@@ -40,6 +38,7 @@ class SpeciesReport:
         "id_no",
         "assessment_id",
         "scientific_name",
+        "overriden",
         "has_systems",
         "not_marine",
         "has_habitats",
@@ -76,23 +75,26 @@ def __getattr__(self, name: str) -> Any:
     def as_row(self) -> List:
         return [self.info[k] for k in self.REPORT_COLUMNS]
 
-
 def process_systems(
     systems_data: List[Tuple],
     report: SpeciesReport,
 ) -> None:
     if len(systems_data) == 0:
-        raise ValueError("No systems found")
-    if len(systems_data) > 1:
-        raise ValueError("More than one systems aggregation found")
-    systems = systems_data[0][0]
-    if systems is None:
-        raise ValueError("no systems info")
-    report.has_systems = True
-
-    if "Marine" in systems:
-        raise ValueError("Marine in systems")
-    report.not_marine = True
+        if not report.overriden:
+            raise ValueError("No systems found")
+    else:
+        if len(systems_data) > 1:
+            # We don't allow override on this as it's a programmer/database error
+            raise ValueError("More than one systems aggregation found")
+        systems = systems_data[0][0]
+        if systems is None and not report.overriden:
+            raise ValueError("no systems info")
+        report.has_systems = (len(systems_data) == 1) and (systems is not None)
+
+        has_marine_in_systems = "Marine" in systems
+        if has_marine_in_systems and not report.overriden:
+            raise ValueError("Marine in systems")
+        report.not_marine = not has_marine_in_systems
 
 def process_habitats(
     habitats_data: List[Tuple],
@@ -152,14 +154,17 @@ def process_habitats(
 
     major_habitats_lvl_1 = {k: {int(v) for v in x} for k, x in major_habitats.items()}
 
-    for _, season_major_habitats_lvl1 in major_habitats_lvl_1.items():
-        if 7 in season_major_habitats_lvl1:
-            raise ValueError("Habitat 7 in major importance habitat list")
-    report.not_major_caves = True
-    for _, season_major_habitats in major_habitats.items():
-        if not season_major_habitats - set([5.1, 5.5, 5.6, 5.14, 5.16]):
-            raise ValueError("Freshwater lakes are major habitat")
-    report.not_major_freshwater_lakes = True
+    major_caves = any(7 in x for x in major_habitats_lvl_1.values())
+    if major_caves and not report.overriden:
+        raise ValueError("Habitat 7 in major importance habitat list")
+    report.not_major_caves = not major_caves
+
+    major_freshwater_lakes = any(
+        not (x - {5.1, 5.5, 5.6, 5.14, 5.16}) for x in major_habitats.values()
+    )
+    if major_freshwater_lakes and not report.overriden:
+        raise ValueError("Freshwater lakes are major habitat")
+    report.not_major_freshwater_lakes = not major_freshwater_lakes
 
     return habitats
 
@@ -205,7 +210,7 @@ def tidy_reproject_save(
     target_crs = src_crs #pyproj.CRS.from_string(target_projection)
 
     graw = gdf.loc[0].copy()
-    grow = aoh_cleaning.tidy_data(graw)
+    grow = aoh.tidy_data(graw)
     output_path = output_directory_path / f"{grow.id_no}_{grow.season}.geojson"
     res = gpd.GeoDataFrame(grow.to_frame().transpose(), crs=src_crs, geometry="geometry")
     res_projected = res.to_crs(target_crs)

prepare_species/extract_species_psql.py

@@ -1,10 +1,11 @@
 import argparse
 import logging
 import os
+import sys
 from functools import partial
 from multiprocessing import Pool
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import Optional, Set, Tuple
 
 # import pyshark # pylint: disable=W0611
 import pandas as pd
@@ -100,6 +101,7 @@
 
 def process_row(
     class_name: str,
+    overrides: Set[int],
     output_directory_path: Path,
     presence: Tuple[int,...],
     row: Tuple,
@@ -110,6 +112,8 @@ def process_row(
 
     (id_no, assessment_id, _elevation_lower, _elevation_upper, scientific_name, _family_name, _threat_code) = row
     report = SpeciesReport(id_no, assessment_id, scientific_name)
+    if id_no in overrides:
+        report.overriden = True
 
     cursor.execute(SYSTEMS_STATEMENT, (assessment_id,))
     systems_data = cursor.fetchall()
@@ -152,6 +156,7 @@ def extract_data_per_species(
     class_name: str,
     output_directory_path: Path,
     _target_projection: Optional[str],
+    overrides_path: Optional[Path],
 ) -> None:
 
     connection = psycopg2.connect(DB_CONFIG)
@@ -164,13 +169,32 @@ def extract_data_per_species(
 
     logger.info("Found %d species in class %s", len(results), class_name)
 
+    if overrides_path is not None:
+        try:
+            overrides_df = pd.read_csv(overrides_path)
+            overrides: Set[int] = set(list(overrides_df.id_no))
+        except FileNotFoundError:
+            sys.exit(f"Failed to find overrides at {overrides_path}")
+        logger.info("Found %d species overrides", len(overrides))
+    else:
+        overrides=set()
+
     for era, presence in [("current", (1, 2)), ("historic", (1, 2, 4, 5))]:
         era_output_directory_path = output_directory_path / era
         os.makedirs(era_output_directory_path, exist_ok=True)
 
         # The limiting amount here is how many concurrent connections the database can take
         with Pool(processes=20) as pool:
-            reports = pool.map(partial(process_row, class_name, era_output_directory_path, presence), results)
+            reports = pool.map(
+                partial(
+                    process_row,
+                    class_name,
+                    overrides,
+                    era_output_directory_path,
+                    presence
+                ),
+                results,
+            )
 
         reports_df = pd.DataFrame(
             [x.as_row() for x in reports],
@@ -202,12 +226,20 @@ def main() -> None:
         dest="target_projection",
         default="ESRI:54017"
     )
+    parser.add_argument(
+        '--overrides',
+        type=Path,
+        help="CSV of species which should be included despite failing other checks.",
+        required=False,
+        dest="overrides_path",
+    )
     args = parser.parse_args()
 
     extract_data_per_species(
         args.classname,
         args.output_directory_path,
-        args.target_projection
+        args.target_projection,
+        args.overrides_path,
     )
 
 if __name__ == "__main__":

requirements.txt

@@ -1,3 +1,4 @@
+# general requirements
 geopandas
 numpy
 duckdb
@@ -13,15 +14,17 @@ rasterio
 requests
 alive-progress
 matplotlib
-
+yirgacheffe
+aoh
 gdal[numpy]
-
 git+https://github.com/quantifyearth/iucn_modlib.git
-git+https://github.com/quantifyearth/yirgacheffe
 git+https://github.com/quantifyearth/seasonality
 
+# developer requirements
 pytest
 pylint
 mypy
 pandas-stubs
 types-requests
+yirgacheffe[dev]
+aoh[dev]

scripts/generate_food_map.sh

@@ -61,9 +61,9 @@ fi
 
 # We need rescaled versions of the current data
 if [ ! -d "${DATADIR}"/food/current_layers ]; then
-    python3 ./aoh-calculator/habitat_process.py --habitat "${DATADIR}"/habitat/current_raw.tif \
-                                                --scale 0.08333333333333333 \
-                                                --output "${DATADIR}"/food/current_layers/
+    aoh-habitat-process --habitat "${DATADIR}"/habitat/current_raw.tif \
+                        --scale 0.08333333333333333 \
+                        --output "${DATADIR}"/food/current_layers/
 fi
 
 # Combine GAEZ and HYDE data