diff --git a/code/ini_templates/landsat_project.ini b/code/ini_templates/landsat_project.ini index 04ba359..dbfc3f4 100644 --- a/code/ini_templates/landsat_project.ini +++ b/code/ini_templates/landsat_project.ini @@ -138,7 +138,7 @@ dem_input_folder = C:\pymetric\dem\tiles dem_tile_fmt = {}{}.img ## NLCD -nlcd_input_path = C:\pymetric\nlcd\nlcd_2011_landcover_2011_edition_2014_10_10\nlcd_2011_landcover_2011_edition_2014_10_10.img +nlcd_input_path = C:\pymetric\nlcd\NLCD_2016_Land_Cover_L48_20190424.img ## Buffer path/row extents by N degrees for selecting DEM tiles and clipping NLCD tile_buffer = 0.25 diff --git a/code/interp_functions/interpolate_rasters_func.py b/code/interp_functions/interpolate_rasters_func.py index 3b7e934..be6ea93 100644 --- a/code/interp_functions/interpolate_rasters_func.py +++ b/code/interp_functions/interpolate_rasters_func.py @@ -4,7 +4,6 @@ # Purpose: Interpolate ETrF rasters between Landsat scenes based on DOY #-------------------------------- -from __future__ import division import argparse from builtins import input from collections import defaultdict diff --git a/code/interp_functions/interpolate_support.py b/code/interp_functions/interpolate_support.py index c67081e..c30ee78 100644 --- a/code/interp_functions/interpolate_support.py +++ b/code/interp_functions/interpolate_support.py @@ -3,7 +3,6 @@ # Purpose: Interpolator support functions #-------------------------------- -from __future__ import division import datetime as dt # import gc import logging diff --git a/code/interp_functions/interpolate_tables_func.py b/code/interp_functions/interpolate_tables_func.py index 47f5675..0d6b3a9 100644 --- a/code/interp_functions/interpolate_tables_func.py +++ b/code/interp_functions/interpolate_tables_func.py @@ -4,7 +4,6 @@ # Purpose: Interpolate ETrF rasters between Landsat scenes based on DOY #-------------------------------- -from __future__ import division import argparse from builtins import input from collections import defaultdict diff --git a/docs/INSTALL.md b/docs/INSTALL.md index b1d7835..1ddc092 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -6,14 +6,13 @@ Most of the documentation and examples are written assuming you are running pyME ## Python -pyMETRIC has only been tested using Python 2.7 and 3.6, but may work with other versions. +pyMETRIC will only officially work with Python 3.6+, but may work with older versions. ## Dependencies The following external Python modules must be present to run pyMETRIC: * [fiona](http://toblerity.org/fiona/) (used to read and write multi-layered GIS file formats) -* [future](https://pypi.python.org/pypi/future) (adds features from Python 3 to Python 2 installations) -* [gdal](http://www.gdal.org/) (version >2.2 and <3.0.0) (the Geospatial Data Abstraction Library is used to interact with raster and vector geospatial data) +* [gdal](http://www.gdal.org/) (version >=3.0) (the Geospatial Data Abstraction Library is used to interact with raster and vector geospatial data) * [matplotlib](https://matplotlib.org/) (necessary for creating plots of ET related data) * [netcdf4](https://www.unidata.ucar.edu/software/netcdf/) (for interacting with multi-dimensional scientific datasets, such as GRIDMET/DAYMET) * [numpy](http://www.numpy.org/) @@ -25,10 +24,6 @@ The following external Python modules must be present to run pyMETRIC: Please see the [requirements](../requirements.txt) file for details on the versioning requirements. The module version numbers listed in the file were tested and are known to work. Other combinations of versions may work but have not been tested. -### Python 2 -The following external Python modules must be present to run pyMETRIC on Python 2 -* [configparser]()(Python 2 implementation of the Python 3 configparser module) - ## Anaconda/Miniconda The easiest way of obtaining Python and all of the necessary external modules, is to install [Miniconda](https://conda.io/miniconda.html) or [Anaconda](https://www.anaconda.com/download/). diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..cc399f4 --- /dev/null +++ b/environment.yml @@ -0,0 +1,19 @@ +name: pymetric +channels: + - defaults + - conda-forge +dependencies: + - gdal>=3.0 + - matplotlib + - netCDF4>=1.3.1 + - numpy>=1.14.0 + - pandas>=1.0.0 + - pip + - python=3.7 + - refet>=0.3.7 + - requests + - scipy>=1.0.0 + - pip: + - drigo>=0.2.0 + - requests-html>=0.10.0 + diff --git a/requirements.txt b/requirements.txt index fbbfddc..6fbeb27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -configparser -drigo>=0.1.4 -future==0.16.0 -gdal>=2.2.3 +drigo>=0.2.0 +gdal>=3.0.0 matplotlib>=2.1.1 netCDF4>=1.3.1 numpy>=1.14.0 diff --git a/tools/download/download_landfire.py b/tools/download/download_landfire.py index 106d4dd..459c769 100644 --- a/tools/download/download_landfire.py +++ b/tools/download/download_landfire.py @@ -21,7 +21,7 @@ def main(output_folder, version='140', overwrite_flag=False): ---------- output_folder : str Folder path where files will be saved. - version : {'105', '110', '120', '130', '140'} + version : {'105', '110', '120', '130', '140', '200'} LANDFIRE version string (the default is '140'). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). @@ -35,6 +35,7 @@ def main(output_folder, version='140', overwrite_flag=False): base_url = 'http://www.landfire.gov/bulk/downloadfile.php?FNAME=' zip_dict = { + '200': 'US_{0}_mosaic-US_{0}EVT.zip&TYPE=landfire'.format(version), '140': 'US_{0}_mosaic-US_{0}EVT_20180618.zip&TYPE=landfire'.format(version), '130': 'US_{0}_Mosaic-US_{0}_EVT_04232015.zip&TYPE=landfire'.format(version), '120': 'US_{0}_Mosaic-US_{0}_EVT_06142017.zip&TYPE=landfire'.format(version), @@ -117,8 +118,8 @@ def arg_parse(): formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '-v', '--version', metavar='VERSION', default='140', - choices=['105', '110', '120', '130', '140'], - help='Version (105, 110, 120, 130, or 140)') + choices=['105', '110', '120', '130', '140', '200'], + help='Version (105, 110, 120, 130, 140, or 200)') parser.add_argument( '--output', help='Output folder', metavar='FOLDER', default=output_folder) diff --git a/tools/download/download_ned.py b/tools/download/download_ned.py index e0fe03c..6d9ca12 100644 --- a/tools/download/download_ned.py +++ b/tools/download/download_ned.py @@ -94,7 +94,7 @@ def main(extent_path, output_folder, overwrite_flag=False): 'EXPAND', tile_x, tile_y, tile_cs) logging.debug('Tile Extent: {}'.format(tile_extent)) - # Get list of avaiable tiles that intersect the extent + # Get list of available tiles that intersect the extent lat_lon_list.extend([ (lat, -lon) for lon in range(int(tile_extent.xmin), int(tile_extent.xmax)) diff --git a/tools/download/download_nlcd.py b/tools/download/download_nlcd.py index 643fb20..295437a 100644 --- a/tools/download/download_nlcd.py +++ b/tools/download/download_nlcd.py @@ -13,15 +13,15 @@ import _utils -def main(output_folder, year='2011', overwrite_flag=False): +def main(output_folder, year='2016', overwrite_flag=False): """Download NLCD raster Parameters ---------- output_folder : str Folder path where files will be saved. - year : {2001, 2006, 2011}; optional - NLCD year (the default is 2011). + year : {2001, 2004, 2006, 2008, 2011, 2013, 2016}; optional + NLCD year (the default is 2016). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). @@ -29,19 +29,16 @@ def main(output_folder, year='2011', overwrite_flag=False): ------- None + Notes + ----- + https://s3-us-west-2.amazonaws.com/mrlc/NLCD_2016_Land_Cover_L48_20190424.zip + """ - download_url = ( - 'https://prd-tnm.s3.amazonaws.com/StagedProducts/NLCD2011/Land_Cover/' - 'CONUS/nlcd_{}_landcover_2011_edition_2014_10_10.zip').format(year) - # download_url = ( - # 'http://www.landfire.gov/bulk/downloadfile.php?' - # 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year) - # download_url = ( - # 'http://gisdata.usgs.gov/TDDS/DownloadFile.php?' - # 'TYPE=nlcd{0}&FNAME=nlcd_{0}_landcover_2011_edition_2014_10_10.zip').format(year) - - zip_name = 'nlcd_{}_landcover_2011_edition_2014_10_10.zip'.format(year) + download_url = f'https://s3-us-west-2.amazonaws.com/mrlc/' \ + f'NLCD_{year}_Land_Cover_L48_20190424.zip' + + zip_name = download_url.split('/')[-1] zip_path = os.path.join(output_folder, zip_name) output_name = zip_name.replace('.zip', '.img') @@ -89,9 +86,9 @@ def arg_parse(): '--output', default=output_folder, metavar='FOLDER', help='Output folder') parser.add_argument( - '-y', '--year', metavar='YEAR', default='2011', - choices=['2001', '2006', '2011'], - help='NLCD Year (2001, 2006, or 2011)') + '-y', '--year', metavar='YEAR', default='2016', + choices=['2001', '2004', '2006', '2008', '2011', '2013', '2016'], + help='NLCD Year (2001, 2004, 2006, 2008, 2011, 2013, or 2016)') parser.add_argument( '-o', '--overwrite', default=None, action="store_true", help='Force overwrite of existing files') diff --git a/tools/gridmet/gridmet_ancillary.py b/tools/gridmet/gridmet_ancillary.py index 115b660..8a3af06 100644 --- a/tools/gridmet/gridmet_ancillary.py +++ b/tools/gridmet/gridmet_ancillary.py @@ -12,7 +12,7 @@ import drigo import netCDF4 import numpy as np -from osgeo import osr +from osgeo import gdal, osr import _utils @@ -43,9 +43,10 @@ def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, # Manually define the spatial reference and extent of the GRIDMET data # This could be read in from a raster gridmet_osr = osr.SpatialReference() - # Assume GRIDMET data is in WGS84 not NAD83 (need to check with John) gridmet_osr.ImportFromEPSG(4326) - # gridmet_osr.ImportFromEPSG(4326) + if int(gdal.__version__[0]) >= 3: + # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546 + gridmet_osr.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) gridmet_proj = drigo.osr_proj(gridmet_osr) gridmet_cs = 1. / 24 # 0.041666666666666666 gridmet_x = -125 + gridmet_cs * 5 diff --git a/tools/nldas/nldas_ancillary.py b/tools/nldas/nldas_ancillary.py index a919db7..512a561 100644 --- a/tools/nldas/nldas_ancillary.py +++ b/tools/nldas/nldas_ancillary.py @@ -7,12 +7,14 @@ import datetime as dt import logging import os -import subprocess +# import subprocess import sys import drigo +import netCDF4 import numpy as np -import pandas as pd +from osgeo import gdal, osr +# import pandas as pd import _utils @@ -38,134 +40,155 @@ def main(ancillary_ws=os.getcwd(), zero_elev_nodata_flag=False, logging.info('\nProcess NLDAS ancillary data') # Site URLs - mask_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/NLDASmask_UMDunified.asc' - elev_url = 'http://ldas.gsfc.nasa.gov/nldas/asc/gtopomean15k.asc' - - nldas_epsg = 'EPSG:4269' - # nldas_epsg = 'EPSG:4326' - + mask_url = 'https://ldas.gsfc.nasa.gov/sites/default/files/ldas/nldas/NLDAS_masks-veg-soil.nc4' + elev_url = 'https://ldas.gsfc.nasa.gov/sites/default/files/ldas/nldas/NLDAS_elevation.nc4' + + # Manually define the spatial reference and extent of the NLDAS data + # This could be read in from a raster + nldas_osr = osr.SpatialReference() + nldas_osr.ImportFromEPSG(4326) + if int(gdal.__version__[0]) >= 3: + # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546 + nldas_osr.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) + nldas_proj = drigo.osr_proj(nldas_osr) + nldas_geo = (-125.0005, 0.125, 0., 53.0005, 0., -0.125) + # nldas_geo = (-124.9375, 0.125, 0., 25.0625 + 224 * 0.125, 0., -0.125) + logging.debug(' Geo: {}'.format(nldas_geo)) nldas_nodata = -9999.0 - - # Site URLs - # file_re = re.compile( - # 'NLDAS_FORA0125_H.A(?P\d{4})(?P\d{2})(?P\d{2}).' + - # '(?P