From 882b8688a9c099c9cef5eedc94d799d335a8094b Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 16:33:46 -0700 Subject: [PATCH 1/2] Adding an LRUCache to convolve_2d. RE:#373 --- src/pygeoprocessing/geoprocessing.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/pygeoprocessing/geoprocessing.py b/src/pygeoprocessing/geoprocessing.py index 3935f4c3..639e5445 100644 --- a/src/pygeoprocessing/geoprocessing.py +++ b/src/pygeoprocessing/geoprocessing.py @@ -36,8 +36,9 @@ from .geoprocessing_core import DEFAULT_CREATION_OPTIONS from .geoprocessing_core import DEFAULT_GTIFF_CREATION_TUPLE_OPTIONS from .geoprocessing_core import DEFAULT_OSR_AXIS_MAPPING_STRATEGY +from .geoprocessing_core import gdal_use_exceptions +from .geoprocessing_core import GDALUseExceptions from .geoprocessing_core import INT8_CREATION_OPTIONS -from .geoprocessing_core import gdal_use_exceptions, GDALUseExceptions # This is used to efficiently pass data to the raster stats worker if available if sys.version_info >= (3, 8): @@ -3825,7 +3826,9 @@ def get_gis_type(path): ``pygeoprocessing.RASTER_TYPE``, or ``pygeoprocessing.VECTOR_TYPE``. """ - from pygeoprocessing import UNKNOWN_TYPE, RASTER_TYPE, VECTOR_TYPE + from pygeoprocessing import RASTER_TYPE + from pygeoprocessing import UNKNOWN_TYPE + from pygeoprocessing import VECTOR_TYPE gis_type = UNKNOWN_TYPE try: gis_raster = gdal.OpenEx(path, gdal.OF_RASTER) @@ -3956,6 +3959,14 @@ def _convolve_2d_worker( kernel_block[numpy.isclose(kernel_block, kernel_nodata)] = 0.0 kernel_sum += numpy.sum(kernel_block) + @functools.lru_cache + def _read_kernel_array(**offset): + return kernel_band.ReadAsArray(**offset).astype(numpy.float64) + + @functools.lru_cache + def _read_signal_array(**offset): + return signal_band.ReadAsArray(**offset).astype(numpy.float64) + while True: payload = work_queue.get() if payload is None: @@ -3965,10 +3976,8 @@ def _convolve_2d_worker( # ensure signal and kernel are internally float64 precision # irrespective of their base type - signal_block = signal_band.ReadAsArray(**signal_offset).astype( - numpy.float64) - kernel_block = kernel_band.ReadAsArray(**kernel_offset).astype( - numpy.float64) + signal_block = _read_signal_array(**signal_offset) + kernel_block = _read_kernel_array(**kernel_offset) # don't ever convolve the nodata value if signal_nodata is not None: From 93993bbc07ce7990b6077c5d19312ae83482db9f Mon Sep 17 00:00:00 2001 From: James Douglass Date: Fri, 11 Oct 2024 16:34:58 -0700 Subject: [PATCH 2/2] Noting change in HISTORY. RE:#373 --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index c5307969..eda70196 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,8 @@ Unreleased Changes ------------------ * Removing the ``numpy<2`` constraint for requirements.txt that should have been included in the 2.4.5 release. https://github.com/natcap/pygeoprocessing/issues/396 +* An LRU Cache has been added to ``convolve_2d`` which should reduce runtimes + on large rasters by approximately 10%. https://github.com/natcap/pygeoprocessing/issues/373 2.4.5 (2024-10-08) ------------------