From 2cb1d928b9812621a7e83aca9966ab03bba79f4c Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Tue, 8 Sep 2020 14:57:50 -0700 Subject: [PATCH 1/6] Updated NLCD download URLs and supported years --- tools/download/download_nlcd.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tools/download/download_nlcd.py b/tools/download/download_nlcd.py index 643fb20..295437a 100644 --- a/tools/download/download_nlcd.py +++ b/tools/download/download_nlcd.py @@ -13,15 +13,15 @@ import _utils -def main(output_folder, year='2011', overwrite_flag=False): +def main(output_folder, year='2016', overwrite_flag=False): """Download NLCD raster Parameters ---------- output_folder : str Folder path where files will be saved. - year : {2001, 2006, 2011}; optional - NLCD year (the default is 2011). + year : {2001, 2004, 2006, 2008, 2011, 2013, 2016}; optional + NLCD year (the default is 2016). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). @@ -29,19 +29,16 @@ def main(output_folder, year='2011', overwrite_flag=False): ------- None + Notes + ----- + https://s3-us-west-2.amazonaws.com/mrlc/NLCD_2016_Land_Cover_L48_20190424.zip + """ - download_url = ( - 'https://prd-tnm.s3.amazonaws.com/StagedProducts/NLCD2011/Land_Cover/' - 'CONUS/nlcd_{}_landcover_2011_edition_2014_10_10.zip').format(year) - # download_url = ( - # 'http://www.landfire.gov/bulk/downloadfile.php?' - # 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year) - # download_url = ( - # 'http://gisdata.usgs.gov/TDDS/DownloadFile.php?' - # 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year) - - zip_name = 'nlcd_{}_landcover_2011_edition_2014_10_10.zip'.format(year) + download_url = f'https://s3-us-west-2.amazonaws.com/mrlc/' \ + f'NLCD_{year}_Land_Cover_L48_20190424.zip' + + zip_name = download_url.split('/')[-1] zip_path = os.path.join(output_folder, zip_name) output_name = zip_name.replace('.zip', '.img') @@ -89,9 +86,9 @@ def arg_parse(): '--output', default=output_folder, metavar='FOLDER', help='Output folder') parser.add_argument( - '-y', '--year', metavar='YEAR', default='2011', - choices=['2001', '2006', '2011'], - help='NLCD Year (2001, 2006, or 2011)') + '-y', '--year', metavar='YEAR', default='2016', + choices=['2001', '2004', '2006', '2008', '2011', '2013', '2016'], + help='NLCD Year (2001, 2004, 2006, 2008, 2011, 2013, or 2016)') parser.add_argument( '-o', '--overwrite', default=None, action="store_true", help='Force overwrite of existing files') From fc6f6f77daed8593cd51f5f3f2df59e8b4ded460 Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Tue, 8 Sep 2020 15:01:37 -0700 Subject: [PATCH 2/6] Started removing Python 2 stuff Added an environment yml. We should probably update the install document to point to this. Removed future module. --- .../interpolate_rasters_func.py | 1 - code/interp_functions/interpolate_support.py | 1 - .../interpolate_tables_func.py | 1 - docs/INSTALL.md | 14 ++++---------- environment.yml | 19 +++++++++++++++++++ requirements.txt | 6 ++---- 6 files changed, 25 insertions(+), 17 deletions(-) create mode 100644 environment.yml diff --git a/code/interp_functions/interpolate_rasters_func.py b/code/interp_functions/interpolate_rasters_func.py index 3b7e934..be6ea93 100644 --- a/code/interp_functions/interpolate_rasters_func.py +++ b/code/interp_functions/interpolate_rasters_func.py @@ -4,7 +4,6 @@ # Purpose: Interpolate ETrF rasters between Landsat scenes based on DOY #-------------------------------- -from __future__ import division import argparse from builtins import input from collections import defaultdict diff --git a/code/interp_functions/interpolate_support.py b/code/interp_functions/interpolate_support.py index c67081e..c30ee78 100644 --- a/code/interp_functions/interpolate_support.py +++ b/code/interp_functions/interpolate_support.py @@ -3,7 +3,6 @@ # Purpose: Interpolator support functions #-------------------------------- -from __future__ import division import datetime as dt # import gc import logging diff --git a/code/interp_functions/interpolate_tables_func.py b/code/interp_functions/interpolate_tables_func.py index 47f5675..0d6b3a9 100644 --- a/code/interp_functions/interpolate_tables_func.py +++ b/code/interp_functions/interpolate_tables_func.py @@ -4,7 +4,6 @@ # Purpose: Interpolate ETrF rasters between Landsat scenes based on DOY #-------------------------------- -from __future__ import division import argparse from builtins import input from collections import defaultdict diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 4aeb1dd..1d01d47 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -6,14 +6,13 @@ Most of the documentation and examples are written assuming you are running pyME ## Python -pyMETRIC has only been tested using Python 2.7 and 3.6, but may work with other versions. +pyMETRIC will only officially work with Python 3.6+, but may work with older versions. ## Dependencies The following external Python modules must be present to run pyMETRIC: * [fiona](http://toblerity.org/fiona/) (used to read and write multi-layered GIS file formats) -* [future](https://pypi.python.org/pypi/future) (adds features from Python 3 to Python 2 installations) -* [gdal](http://www.gdal.org/) (version >2.2 and <3.0.0) (the Geospatial Data Abstraction Library is used to interact with raster and vector geospatial data) +* [gdal](http://www.gdal.org/) (version >=3.0) (the Geospatial Data Abstraction Library is used to interact with raster and vector geospatial data) * [matplotlib](https://matplotlib.org/) (necessary for creating plots of ET related data) * [netcdf4](https://www.unidata.ucar.edu/software/netcdf/) (for interacting with multi-dimensional scientific datasets, such as GRIDMET/DAYMET) * [numpy](http://www.numpy.org/) @@ -25,10 +24,6 @@ The following external Python modules must be present to run pyMETRIC: Please see the [requirements](../requirements.txt) file for details on the versioning requirements. The module version numbers listed in the file were tested and are known to work. Other combinations of versions may work but have not been tested. -### Python 2 -The following external Python modules must be present to run pyMETRIC on Python 2 -* [configparser]()(Python 2 implementation of the Python 3 configparser module) - ## Anaconda/Miniconda The easiest way of obtaining Python and all of the necessary external modules, is to install [Miniconda](https://conda.io/miniconda.html) or [Anaconda](https://www.anaconda.com/download/). @@ -44,7 +39,7 @@ After installing Miniconda, make sure to add the [conda-forge](https://conda-for The user is strongly encouraged to setup a dedicated conda environment for pyMETRIC: ``` -> conda create -n pymetric python=3.6 +> conda create -n pymetric python=3.7 ``` The environment must be "activated" before use: @@ -54,8 +49,7 @@ The environment must be "activated" before use: Most of the external modules can then be installed by calling: ``` -> conda install future matplotlib netcdf4 numpy pandas refet requests scipy -> conda install gdal<=3.0.0 +> conda install gdal>=3.0.0 matplotlib netcdf4 numpy pandas refet requests scipy ``` The drigo modules must be installed separately with pip: diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..cc399f4 --- /dev/null +++ b/environment.yml @@ -0,0 +1,19 @@ +name: pymetric +channels: + - defaults + - conda-forge +dependencies: + - gdal>=3.0 + - matplotlib + - netCDF4>=1.3.1 + - numpy>=1.14.0 + - pandas>=1.0.0 + - pip + - python=3.7 + - refet>=0.3.7 + - requests + - scipy>=1.0.0 + - pip: + - drigo>=0.2.0 + - requests-html>=0.10.0 + diff --git a/requirements.txt b/requirements.txt index fbbfddc..6fbeb27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -configparser -drigo>=0.1.4 -future==0.16.0 -gdal>=2.2.3 +drigo>=0.2.0 +gdal>=3.0.0 matplotlib>=2.1.1 netCDF4>=1.3.1 numpy>=1.14.0 From c9d5462e7d119db562e31d4af6a16d29470d9da6 Mon Sep 17 00:00:00 2001 From: Charles Morton Date: Tue, 8 Sep 2020 16:27:32 -0700 Subject: [PATCH 3/6] NLDAS mask source changed to a netcdf file The elevation ascii file is also gone. Will fix that next. Need to double check the NLDAS geotransform. The values in earth engine are slightly different than in the NetCDF. --- tools/download/download_ned.py | 2 +- tools/gridmet/gridmet_ancillary.py | 7 +- tools/nldas/nldas_ancillary.py | 227 ++++++++++++++--------------- 3 files changed, 115 insertions(+), 121 deletions(-) diff --git a/tools/download/download_ned.py b/tools/download/download_ned.py index e0fe03c..6d9ca12 100644 --- a/tools/download/download_ned.py +++ b/tools/download/download_ned.py @@ -94,7 +94,7 @@ def main(extent_path, output_folder, overwrite_flag=False): 'EXPAND', tile_x, tile_y, tile_cs) logging.debug('Tile Extent: {}'.format(tile_extent)) - # Get list of avaiable tiles that intersect the extent + # Get list of available tiles that intersect the extent lat_lon_list.extend([ (lat, -lon) for lon in range(int(tile_extent.xmin), int(tile_extent.xmax)) diff --git a/tools/gridmet/gridmet_ancillary.py b/tools/gridmet/gridmet_ancillary.py index 115b660..8a3af06 100644 --- a/tools/gridmet/gridmet_ancillary.py +++ b/tools/gridmet/gridmet_ancillary.py @@ -12,7 +12,7 @@ import drigo import netCDF4 import numpy as np -from osgeo import osr +from osgeo import gdal, osr import _utils @@ -43,9 +43,10 @@ def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, # Manually define the spatial reference and extent of the GRIDMET data # This could be read in from a raster gridmet_osr = osr.SpatialReference() - # Assume GRIDMET data is in WGS84 not NAD83 (need to check with John) gridmet_osr.ImportFromEPSG(4326) - # gridmet_osr.ImportFromEPSG(4326) + if int(gdal.__version__[0]) >= 3: + # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546 + gridmet_osr.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) gridmet_proj = drigo.osr_proj(gridmet_osr) gridmet_cs = 1. / 24 # 0.041666666666666666 gridmet_x = -125 + gridmet_cs * 5 diff --git a/tools/nldas/nldas_ancillary.py b/tools/nldas/nldas_ancillary.py index a919db7..830c528 100644 --- a/tools/nldas/nldas_ancillary.py +++ b/tools/nldas/nldas_ancillary.py @@ -7,12 +7,14 @@ import datetime as dt import logging import os -import subprocess +# import subprocess import sys import drigo +import netCDF4 import numpy as np -import pandas as pd +from osgeo import gdal, osr +# import pandas as pd import _utils @@ -38,134 +40,125 @@ def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, logging.info('\nProcess NLDAS ancillary data') # Site URLs - mask_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/NLDASmask_UMDunified.asc' - elev_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/gtopomean15k.asc' - - nldas_epsg = 'EPSG:4269' - # nldas_epsg = 'EPSG:4326' - - nldas_nodata = -9999.0 - - # Site URLs - # file_re = re.compile( - # 'NLDAS_FORA0125_H.A(?P\d{4})(?P\d{2})(?P\d{2}).' + - # '(?P