tweak env/reqs to utilize latest version(s) of PCMDI/cmor

CONTRIBUTING.md

@@ -3,6 +3,7 @@
 * Developers are free to use this repository's `README.md` to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless
 * Gain access to the repository with `git clone --recursive git@github.com:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key
     - Once inside the repository, developers can test local changes by running a `pip install .` inside of the root directory to install the fre-cli package locally with the newest local changes on top of the installed Conda fre-cli dependencies
+    - Optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines (e.g., `module load fre-nctools` on GFDL/Gaea systems)
     - Test as a normal user would use the CLI
 * Create a GitHub issue to reflect your contribution's background and reference it with Git commits
 

README.md

@@ -67,6 +67,11 @@ conda config --append channels noaa-gfdl
 conda config --append channels conda-forge
 conda create --name fre-2025.04 --channel noaa-gfdl --channel conda-forge fre-cli::2025.04
 conda activate fre-2025.04
+# optional: install or load fre-nctools to gain access to regridding and certain time-averaging routines
+# add to your path like: export PATH=/path/to/your/fre-nctools/build/bin:$PATH
+# or if you have lmod/modules: module load fre-nctools/<version>
+# or compile/install from source: see github.comNOAA-GDFL/FRE-NCTools documentation on compilation/installation
+# DO NOT USE noaa-gfdl::fre-nctools==2022.01 at this time, it is being deprecated
 ```
 
 ## GFDL/RDHPCS deployment notes

docs/contributing_to_doc.rst

@@ -52,7 +52,7 @@
 lightweight approach (recommended for docs-only)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 If you only need to build documentation and don't need the full ``fre-cli`` environment,
 you can use a minimal setup. This approach uses ``autodoc_mock_imports`` in ``docs/conf.py``
 to mock heavy dependencies like ``netCDF4``, ``cmor``, ``xarray``, etc.
 
 From the root-directory of your local repository copy:
@@ -66,8 +66,8 @@
  # Install minimal dependencies
  pip install sphinx renku-sphinx-theme sphinx-rtd-theme click pyyaml jsonschema
 
  # Set PYTHONPATH so Sphinx can find the fre modules
  export PYTHONPATH="${PWD}:${PYTHONPATH}"
 
  # Generate API docs and build
  sphinx-apidoc --output-dir docs fre/ --separate
@@ -90,6 +90,9 @@
  # Activate your fre-cli environment
  conda activate fre-cli
 
+ # optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines
+ # module load fre-nctools
+
  # Install documentation dependencies
  pip install .[docs]
 

docs/setup.rst

@@ -42,3 +42,6 @@ Assuming one has ``conda`` in their path, then do the following::
   # now we pip install the local code under the `fre/` directory
   # the -e flag makes re-installing the code after editing not necessary
   pip install -e .
+
+  # optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines
+  # module load fre-nctools

environment.yml

@@ -3,25 +3,25 @@ channels:
   - conda-forge
   - noaa-gfdl
 dependencies:
-  - python=3.11.*
+  - python>=3.11
   - noaa-gfdl::analysis_scripts==0.0.1
   - noaa-gfdl::catalogbuilder==2025.01.01
-  - noaa-gfdl::fre-nctools==2022.02.01
+#  - noaa-gfdl::fre-nctools==2022.02.01
   - conda-forge::cdo>=2
   - conda-forge::cftime
   - conda-forge::click>=8.2
-  - conda-forge::cmor>=3.11.*
+  - conda-forge::cmor>=3.14
   - conda-forge::cylc-flow>=8.2
   - conda-forge::cylc-rose
   - conda-forge::jinja2>=3
   - conda-forge::jsonschema
   - conda-forge::metomi-rose
   - conda-forge::nccmp
-  - conda-forge::numpy=1.26.*
+  - conda-forge::numpy==1.26.4
   - conda-forge::pytest
   - conda-forge::pytest-cov
   - conda-forge::pylint
   - conda-forge::python-cdo
   - conda-forge::pyyaml
   - conda-forge::xarray>=2024.*
-  - conda-forge::netcdf4=1.7.*
+  - conda-forge::netcdf4>=1.7.*

fre/app/generate_time_averages/tests/test_generate_time_averages.py

@@ -1,4 +1,6 @@
-''' For testing fre app generate-time-averages '''
+'''
+For testing fre app generate-time-averages
+'''
 from pathlib import Path
 import subprocess
 import os
@@ -44,7 +46,9 @@
 
 # setup, TODO convert to fixture ?
 def setup_test_files():
-    """Generate test files needed for the tests. Called by test functions that need these files."""
+    '''
+    Generate test files needed for the tests. Called by test functions that need these files.
+    '''
     # Generate first test file
     if Path(NCGEN_OUTPUT).exists():
         Path(NCGEN_OUTPUT).unlink()
@@ -61,7 +65,9 @@
 
 # setup, TODO convert to fixture?
 def setup_two_test_files():
-    """Generate the two ocean test files needed for multi-file tests."""
+    '''
+    Generate the two ocean test files needed for multi-file tests.
+    '''
     for ocean_base_file_name in OCEAN_BASE_FILE_NAMES:
         ncgen_input = TIME_AVG_FILE_DIR + ocean_base_file_name + ".cdl"
         ncgen_output = TIME_AVG_FILE_DIR + ocean_base_file_name + ".nc"
@@ -73,13 +79,17 @@
         subprocess.run(ex, check = True)
 
 def test_setups():
-    """setup by generating netcdf files from cdl inputs"""
+    '''
+    setup by generating netcdf files from cdl inputs
+    '''
     setup_test_files()
     setup_two_test_files()
 
 # sanity check
 def test_time_avg_file_dir_exists():
-    ''' look for input test file directory '''
+    '''
+    look for input test file directory
+    '''
     assert Path(TIME_AVG_FILE_DIR).exists()
 
 FULL_TEST_FILE_PATH = TIME_AVG_FILE_DIR + TEST_FILE_NAME
@@ -101,26 +111,26 @@
     #cdo cases, all, one/multiple files, weighted/unweighted
     pytest.param( 'cdo', 'all', True ,
                   FULL_TEST_FILE_PATH,
-                  TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TEST_FILE_NAME),
+                  STR_UNWGT_CDO_INF),
     pytest.param( 'cdo', 'all', True ,
                   TWO_TEST_FILE_NAMES,
                   TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TWO_OUT_FILE_NAME),
     pytest.param( 'cdo', 'all', False ,
                   FULL_TEST_FILE_PATH,
-                  TIME_AVG_FILE_DIR + 'timmean_' + TEST_FILE_NAME),
+                  STR_CDO_INF),
     pytest.param( 'cdo', 'all', False ,
                   TWO_TEST_FILE_NAMES,
                   TIME_AVG_FILE_DIR + 'timmean_' + TWO_OUT_FILE_NAME),
     #fre-python-tools cases, all, one/multiple files, weighted/unweighted flag
     pytest.param( 'fre-python-tools', 'all',  False ,
                   FULL_TEST_FILE_PATH,
-                  TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TEST_FILE_NAME),
+                  STR_FRE_PYTOOLS_INF),
     pytest.param( 'fre-python-tools', 'all',  False ,
                   TWO_TEST_FILE_NAMES,
                   TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TWO_OUT_FILE_NAME),
     pytest.param( 'fre-python-tools', 'all',  True ,
                   FULL_TEST_FILE_PATH,
-                  TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TEST_FILE_NAME),
+                  STR_UNWGT_FRE_PYTOOLS_INF),
     pytest.param( 'fre-python-tools', 'all',  True ,
                   TWO_TEST_FILE_NAMES,
                   TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TWO_OUT_FILE_NAME),
@@ -146,8 +156,9 @@
                                        unwgt    ,
                                        infile   ,
                                        outfile  ):
-
-    ''' test-harness function, called by other test functions. '''
+    '''
+    test-harness function, called by other test functions.
+    '''
 
     # because the conda package for fre-nctools is a bit... special
     if pkg=='fre-nctools':
@@ -202,7 +213,9 @@
 
 @pytest.mark.xfail(reason = 'fre-cli seems to not bitwise reproduce frenctools at this time') #TODO
 def test_compare_fre_cli_to_fre_nctools():
-    ''' compares fre_cli pkg answer to fre_nctools pkg answer '''
+    '''
+    compares fre_cli pkg answer to fre_nctools pkg answer
+    '''
     assert Path(STR_FRE_PYTOOLS_INF).exists(), f'DNE: STR_FRE_PYTOOLS_INF = {STR_FRE_PYTOOLS_INF}'
     fre_pytools_inf = Dataset(STR_FRE_PYTOOLS_INF,'r')
     fre_pytools_timavg = fre_pytools_inf[VAR][:].copy()
@@ -227,12 +240,14 @@
 
     non_zero_count = np.count_nonzero( diff_pytools_nctools_timavg[:] )
     #assert (non_zero_count == 0.) # bad way to check for zero.
-    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between frepy / frenctools were found"
 
 
 @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer')
 def test_compare_fre_cli_to_cdo():
-    ''' compares fre_cli pkg answer to cdo pkg answer '''
+    '''
+    compares fre_cli pkg answer to cdo pkg answer
+    '''
     assert Path(STR_FRE_PYTOOLS_INF).exists(), f'DNE: STR_FRE_PYTOOLS_INF = {STR_FRE_PYTOOLS_INF}'
     fre_pytools_inf = Dataset(STR_FRE_PYTOOLS_INF, 'r')
     fre_pytools_timavg = fre_pytools_inf[VAR][:].copy()
@@ -255,11 +270,13 @@
         break
 
     non_zero_count = np.count_nonzero( diff_pytools_cdo_timavg[:] )
-    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frepytools were found"
 
 
 def test_compare_unwgt_fre_cli_to_unwgt_cdo():
-    ''' compares fre_cli pkg answer to cdo pkg answer '''
+    '''
+    compares fre_cli pkg answer to cdo pkg answer
+    '''
     assert Path(STR_UNWGT_FRE_PYTOOLS_INF).exists(), f'DNE: STR_UNWGT_FRE_PYTOOLS_INF = {STR_UNWGT_FRE_PYTOOLS_INF}'
     fre_pytools_inf = Dataset(STR_UNWGT_FRE_PYTOOLS_INF, 'r')
     fre_pytools_timavg = fre_pytools_inf[VAR][:].copy()
@@ -282,11 +299,13 @@
         break
 
     non_zero_count = np.count_nonzero(diff_pytools_cdo_timavg[:])
-    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frepytools were found"
 
 @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer')
 def test_compare_cdo_to_fre_nctools():
-    ''' compares cdo pkg answer to fre_nctools pkg answer '''
+    '''
+    compares cdo pkg answer to fre_nctools pkg answer
+    '''
 
     assert Path(STR_FRENCTOOLS_INF).exists(), f'DNE: STR_FRENCTOOLS_INF = {STR_FRENCTOOLS_INF}'
     fre_nctools_inf = Dataset(STR_FRENCTOOLS_INF, 'r')
@@ -310,11 +329,13 @@
         break
 
     non_zero_count = np.count_nonzero(diff_cdo_nctools_timavg[:])
-    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) )
+    assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frenctools were found"
 
 # if we use fixtures and tmpdirs, can omit this error prone thing
 def test_fre_app_gen_time_avg_test_data_cleanup():
-    ''' Removes all .nc files in fre/app/generate_time_averages/tests/test_data/ '''
+    '''
+    Removes all .nc files in fre/app/generate_time_averages/tests/test_data/
+    '''
     nc_files = [ Path(os.path.join(TIME_AVG_FILE_DIR, el)) for el in os.listdir(TIME_AVG_FILE_DIR)
                  if el.endswith('.nc')]
     for nc in nc_files:
@@ -329,8 +350,10 @@
     pytest.param( 'foo_input_file', 'foo_output_file', 'DNE' ),]
 @pytest.mark.parametrize( "infile,outfile,pkg", value_err_args_cases )
 def test_no_req_arg_inputfile( infile, outfile, pkg):
+    '''
+    test failures/exceptions regarding input args
+    '''
     with pytest.raises(ValueError):
         gtas.generate_time_average( infile = infile,
                                     outfile = outfile,
                                     pkg = pkg )
-

fre/app/regrid_xy/README.md

@@ -2,7 +2,7 @@
 
 `regrid_xy.py` remaps scalar and/or vector fields from one kind of lat/lon grid to another.  It can remap between different grids of the same type (e.g. spherical), and between grids of different types (e.g. spherical to tripolar). By default, it uses an O(1) conservative interpolation scheme to accomplish the regridding, except under certain conditions [defined within `fregrid`](https://github.com/NOAA-GFDL/FRE-NCtools/blob/master/tools/fregrid/fregrid.c#L915-L920) the underlying CLI tool which does the heavy lifting.
 
-requires `fre-nctools` and `fregrid` to be in one's `PATH` variable, and `python3` (tested/developed with python 3.9.16). there should be `netCDF4` and `metomi` python modules in one's python environment for imports. `pytest` and `nccmp` is required for tests. `pylint` recommended for future developers working on this tool. 
+requires `fregrid` to be in one's `PATH` variable (`fre-nctools` no longer bundled; load via `module load fre-nctools`), and `python3` >= 3.11 (tested/developed with python 3.12). there should be `netCDF4` and `xarray` python modules in one's python environment for imports. `pytest` and `nccmp` is required for tests. `pylint` recommended for future developers working on this tool. 
 
 
 # INPUT PARAMETERS (mandatory, env vars)