From e55f40dc89c7f167d939677ba9f1df37e062ccdd Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 29 Jan 2026 14:33:11 -0500 Subject: [PATCH 1/8] tweak versions, lets see what happens to env resolution --- environment.yml | 8 ++++---- meta.yaml | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/environment.yml b/environment.yml index 512253f9a..6f90d3e41 100644 --- a/environment.yml +++ b/environment.yml @@ -3,25 +3,25 @@ channels: - conda-forge - noaa-gfdl dependencies: - - python=3.11.* + - python>=3.12.* - noaa-gfdl::analysis_scripts==0.0.1 - noaa-gfdl::catalogbuilder==2025.01.01 - noaa-gfdl::fre-nctools==2022.02.01 - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor>=3.11.* + - conda-forge::cmor>=3.12.* - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy=1.26.* + - conda-forge::numpy>=1.26.* - conda-forge::pytest - conda-forge::pytest-cov - conda-forge::pylint - conda-forge::python-cdo - conda-forge::pyyaml - conda-forge::xarray>=2024.* - - conda-forge::netcdf4=1.7.* + - conda-forge::netcdf4>=1.7.* diff --git a/meta.yaml b/meta.yaml index 936ab41e1..791f6472e 100644 --- a/meta.yaml +++ b/meta.yaml @@ -19,27 +19,27 @@ channels: requirements: host: - - python=3.11.* + - python>=3.12.* run: - - python=3.11.* + - python>=3.12.* - noaa-gfdl::analysis_scripts==0.0.1 - noaa-gfdl::catalogbuilder==2025.01.01 - noaa-gfdl::fre-nctools==2022.02.01 - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor=3.11.* + - conda-forge::cmor>=3.12.* - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy=1.26.* + - conda-forge::numpy>=1.26.* - conda-forge::python-cdo - conda-forge::pyyaml - conda-forge::xarray>=2024.* - - conda-forge::netcdf4=1.7.* + - conda-forge::netcdf4>=1.7.* test: requires: From 4b82a3c79da8ccc82859d54323c2a781794fe4b0 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 29 Jan 2026 16:59:58 -0500 Subject: [PATCH 2/8] loosed python version req to be >=3.11, no more fre-nctools which pins the libnetcdf req too hard --- environment.yml | 10 +++++----- meta.yaml | 10 +++++----- pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/environment.yml b/environment.yml index 6f90d3e41..16b494836 100644 --- a/environment.yml +++ b/environment.yml @@ -1,23 +1,23 @@ -name: fre-cli +name: fre-cli-nofrenctools channels: - conda-forge - noaa-gfdl dependencies: - - python>=3.12.* + - python>=3.11 - noaa-gfdl::analysis_scripts==0.0.1 - noaa-gfdl::catalogbuilder==2025.01.01 - - noaa-gfdl::fre-nctools==2022.02.01 +# - noaa-gfdl::fre-nctools==2022.02.01 - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor>=3.12.* + - conda-forge::cmor>=3.13.* - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy>=1.26.* + - conda-forge::numpy=2.* - conda-forge::pytest - conda-forge::pytest-cov - conda-forge::pylint diff --git a/meta.yaml b/meta.yaml index 791f6472e..fbce86070 100644 --- a/meta.yaml +++ b/meta.yaml @@ -19,23 +19,23 @@ channels: requirements: host: - - python>=3.12.* + - python>=3.11 run: - - python>=3.12.* + - python>=3.11 - noaa-gfdl::analysis_scripts==0.0.1 - noaa-gfdl::catalogbuilder==2025.01.01 - - noaa-gfdl::fre-nctools==2022.02.01 +# - noaa-gfdl::fre-nctools==2022.02.01 - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor>=3.12.* + - conda-forge::cmor>=3.13.* - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy>=1.26.* + - conda-forge::numpy>=2.* - conda-forge::python-cdo - conda-forge::pyyaml - conda-forge::xarray>=2024.* diff --git a/pyproject.toml b/pyproject.toml index 19bc423d0..83d7b44de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "fre-cli" description = "Command Line Interface for FRE commands" readme = "README.md" -requires-python = "==3.11.*" # loosen this TODO +requires-python = ">=3.11" license = "Apache-2.0" authors = [ {name = "MSD Workflow Team", email="oar.gfdl.workflow@noaa.gov"} From 8088cb7d2e9e56d33594518aa1ddb26fd9a0749a Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 30 Jan 2026 12:21:47 -0500 Subject: [PATCH 3/8] tweak time averager tests a little bit. back off numpy2.0 but specify cmor3.14 specifically (the latest) --- environment.yml | 6 +- .../tests/test_generate_time_averages.py | 60 ++++++++++++++----- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/environment.yml b/environment.yml index 16b494836..fb38c5ee1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: fre-cli-nofrenctools +name: fre-cli-nofrenctools2 channels: - conda-forge - noaa-gfdl @@ -10,14 +10,14 @@ dependencies: - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor>=3.13.* + - conda-forge::cmor>=3.14 - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy=2.* + - conda-forge::numpy==1.26.4 - conda-forge::pytest - conda-forge::pytest-cov - conda-forge::pylint diff --git a/fre/app/generate_time_averages/tests/test_generate_time_averages.py b/fre/app/generate_time_averages/tests/test_generate_time_averages.py index 934f6398e..f03ab5adb 100644 --- a/fre/app/generate_time_averages/tests/test_generate_time_averages.py +++ b/fre/app/generate_time_averages/tests/test_generate_time_averages.py @@ -1,4 +1,6 @@ -''' For testing fre app generate-time-averages ''' +''' +For testing fre app generate-time-averages +''' from pathlib import Path import subprocess import os @@ -44,7 +46,9 @@ # setup, TODO convert to fixture ? def setup_test_files(): - """Generate test files needed for the tests. Called by test functions that need these files.""" + ''' + Generate test files needed for the tests. Called by test functions that need these files. + ''' # Generate first test file if Path(NCGEN_OUTPUT).exists(): Path(NCGEN_OUTPUT).unlink() @@ -61,7 +65,9 @@ def setup_test_files(): # setup, TODO convert to fixture? def setup_two_test_files(): - """Generate the two ocean test files needed for multi-file tests.""" + ''' + Generate the two ocean test files needed for multi-file tests. + ''' for ocean_base_file_name in OCEAN_BASE_FILE_NAMES: ncgen_input = TIME_AVG_FILE_DIR + ocean_base_file_name + ".cdl" ncgen_output = TIME_AVG_FILE_DIR + ocean_base_file_name + ".nc" @@ -73,13 +79,17 @@ def setup_two_test_files(): subprocess.run(ex, check = True) def test_setups(): - """setup by generating netcdf files from cdl inputs""" + ''' + setup by generating netcdf files from cdl inputs + ''' setup_test_files() setup_two_test_files() # sanity check def test_time_avg_file_dir_exists(): - ''' look for input test file directory ''' + ''' + look for input test file directory + ''' assert Path(TIME_AVG_FILE_DIR).exists() FULL_TEST_FILE_PATH = TIME_AVG_FILE_DIR + TEST_FILE_NAME @@ -101,26 +111,30 @@ def test_time_avg_file_dir_exists(): #cdo cases, all, one/multiple files, weighted/unweighted pytest.param( 'cdo', 'all', True , FULL_TEST_FILE_PATH, - TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TEST_FILE_NAME), + STR_UNWGT_CDO_INF), +# TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TEST_FILE_NAME), pytest.param( 'cdo', 'all', True , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TWO_OUT_FILE_NAME), pytest.param( 'cdo', 'all', False , FULL_TEST_FILE_PATH, - TIME_AVG_FILE_DIR + 'timmean_' + TEST_FILE_NAME), + STR_CDO_INF), +# TIME_AVG_FILE_DIR + 'timmean_' + TEST_FILE_NAME), pytest.param( 'cdo', 'all', False , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'timmean_' + TWO_OUT_FILE_NAME), #fre-python-tools cases, all, one/multiple files, weighted/unweighted flag pytest.param( 'fre-python-tools', 'all', False , FULL_TEST_FILE_PATH, - TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TEST_FILE_NAME), + STR_FRE_PYTOOLS_INF), +# TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TEST_FILE_NAME), pytest.param( 'fre-python-tools', 'all', False , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TWO_OUT_FILE_NAME), pytest.param( 'fre-python-tools', 'all', True , FULL_TEST_FILE_PATH, - TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TEST_FILE_NAME), + STR_UNWGT_FRE_PYTOOLS_INF), +# TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TEST_FILE_NAME), pytest.param( 'fre-python-tools', 'all', True , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TWO_OUT_FILE_NAME), @@ -147,7 +161,9 @@ def test_run_avgtype_pkg_calculations( pkg , infile , outfile ): - ''' test-harness function, called by other test functions. ''' + ''' + test-harness function, called by other test functions. + ''' # because the conda package for fre-nctools is a bit... special if pkg=='fre-nctools': @@ -202,7 +218,9 @@ def test_run_avgtype_pkg_calculations( pkg , @pytest.mark.xfail(reason = 'fre-cli seems to not bitwise reproduce frenctools at this time') #TODO def test_compare_fre_cli_to_fre_nctools(): - ''' compares fre_cli pkg answer to fre_nctools pkg answer ''' + ''' + compares fre_cli pkg answer to fre_nctools pkg answer + ''' assert Path(STR_FRE_PYTOOLS_INF).exists(), f'DNE: STR_FRE_PYTOOLS_INF = {STR_FRE_PYTOOLS_INF}' fre_pytools_inf = Dataset(STR_FRE_PYTOOLS_INF,'r') fre_pytools_timavg = fre_pytools_inf[VAR][:].copy() @@ -232,7 +250,9 @@ def test_compare_fre_cli_to_fre_nctools(): @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') def test_compare_fre_cli_to_cdo(): - ''' compares fre_cli pkg answer to cdo pkg answer ''' + ''' + compares fre_cli pkg answer to cdo pkg answer + ''' assert Path(STR_FRE_PYTOOLS_INF).exists(), f'DNE: STR_FRE_PYTOOLS_INF = {STR_FRE_PYTOOLS_INF}' fre_pytools_inf = Dataset(STR_FRE_PYTOOLS_INF, 'r') fre_pytools_timavg = fre_pytools_inf[VAR][:].copy() @@ -259,7 +279,9 @@ def test_compare_fre_cli_to_cdo(): def test_compare_unwgt_fre_cli_to_unwgt_cdo(): - ''' compares fre_cli pkg answer to cdo pkg answer ''' + ''' + compares fre_cli pkg answer to cdo pkg answer + ''' assert Path(STR_UNWGT_FRE_PYTOOLS_INF).exists(), f'DNE: STR_UNWGT_FRE_PYTOOLS_INF = {STR_UNWGT_FRE_PYTOOLS_INF}' fre_pytools_inf = Dataset(STR_UNWGT_FRE_PYTOOLS_INF, 'r') fre_pytools_timavg = fre_pytools_inf[VAR][:].copy() @@ -286,7 +308,9 @@ def test_compare_unwgt_fre_cli_to_unwgt_cdo(): @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') def test_compare_cdo_to_fre_nctools(): - ''' compares cdo pkg answer to fre_nctools pkg answer ''' + ''' + compares cdo pkg answer to fre_nctools pkg answer + ''' assert Path(STR_FRENCTOOLS_INF).exists(), f'DNE: STR_FRENCTOOLS_INF = {STR_FRENCTOOLS_INF}' fre_nctools_inf = Dataset(STR_FRENCTOOLS_INF, 'r') @@ -314,7 +338,9 @@ def test_compare_cdo_to_fre_nctools(): # if we use fixtures and tmpdirs, can omit this error prone thing def test_fre_app_gen_time_avg_test_data_cleanup(): - ''' Removes all .nc files in fre/app/generate_time_averages/tests/test_data/ ''' + ''' + Removes all .nc files in fre/app/generate_time_averages/tests/test_data/ + ''' nc_files = [ Path(os.path.join(TIME_AVG_FILE_DIR, el)) for el in os.listdir(TIME_AVG_FILE_DIR) if el.endswith('.nc')] for nc in nc_files: @@ -329,8 +355,10 @@ def test_fre_app_gen_time_avg_test_data_cleanup(): pytest.param( 'foo_input_file', 'foo_output_file', 'DNE' ),] @pytest.mark.parametrize( "infile,outfile,pkg", value_err_args_cases ) def test_no_req_arg_inputfile( infile, outfile, pkg): + ''' + test failures/exceptions regarding input args + ''' with pytest.raises(ValueError): gtas.generate_time_average( infile = infile, outfile = outfile, pkg = pkg ) - From 2cbe750295e8dc7d0834bae1ac5371850613113d Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 30 Jan 2026 12:50:16 -0500 Subject: [PATCH 4/8] put env name back to fre-cli, bring the meta.yaml specs up to snuff with the environment yaml --- environment.yml | 2 +- meta.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index fb38c5ee1..b7650dfdf 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: fre-cli-nofrenctools2 +name: fre-cli channels: - conda-forge - noaa-gfdl diff --git a/meta.yaml b/meta.yaml index fbce86070..4d9e5ada0 100644 --- a/meta.yaml +++ b/meta.yaml @@ -28,14 +28,14 @@ requirements: - conda-forge::cdo>=2 - conda-forge::cftime - conda-forge::click>=8.2 - - conda-forge::cmor>=3.13.* + - conda-forge::cmor>=3.14 - conda-forge::cylc-flow>=8.2 - conda-forge::cylc-rose - conda-forge::jinja2>=3 - conda-forge::jsonschema - conda-forge::metomi-rose - conda-forge::nccmp - - conda-forge::numpy>=2.* + - conda-forge::numpy==1.26.4 - conda-forge::python-cdo - conda-forge::pyyaml - conda-forge::xarray>=2024.* From 56a8f2936a301644a5c595f01846baf4c7a3ea31 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 30 Jan 2026 13:20:22 -0500 Subject: [PATCH 5/8] add conditional skips to fregrid dependent tests. specify build meta and backend tools to try and address conda package build error --- fre/app/regrid_xy/tests/test_regrid_xy.py | 7 +++++-- fre/tests/test_fre_app_cli.py | 7 ++++++- meta.yaml | 6 ++++++ pyproject.toml | 10 +++++----- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/fre/app/regrid_xy/tests/test_regrid_xy.py b/fre/app/regrid_xy/tests/test_regrid_xy.py index af5831d3e..f9de2d9a4 100644 --- a/fre/app/regrid_xy/tests/test_regrid_xy.py +++ b/fre/app/regrid_xy/tests/test_regrid_xy.py @@ -1,7 +1,9 @@ -import numpy as np import os from pathlib import Path import shutil +from shutil import which + +import numpy as np import xarray as xr import fre.app.regrid_xy.regrid_xy as regrid_xy @@ -70,7 +72,8 @@ def cleanup_test(): if work_dir.exists(): shutil.rmtree(work_dir) generate_files.cleanup() - +@pytest.mark.skipif(which('fregrid') is None, + reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_regrid_xy(): """ diff --git a/fre/tests/test_fre_app_cli.py b/fre/tests/test_fre_app_cli.py index 9394aff52..7cc1223ca 100644 --- a/fre/tests/test_fre_app_cli.py +++ b/fre/tests/test_fre_app_cli.py @@ -12,11 +12,14 @@ import os import subprocess +import shutil +from shutil import which from pathlib import Path -import pytest import click from click.testing import CliRunner +import pytest + from fre import fre @@ -191,6 +194,8 @@ def test_cli_fre_app_regrid_opt_dne(capfd): assert result.exit_code == 2 _out, _err = capfd.readouterr() +@pytest.mark.skipif(which('fregrid') is None, + reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_cli_fre_app_regrid_test_case_1(capfd): """ fre app regrid_xy --help """ diff --git a/meta.yaml b/meta.yaml index 4d9e5ada0..3860b7508 100644 --- a/meta.yaml +++ b/meta.yaml @@ -18,8 +18,14 @@ channels: - noaa-gfdl requirements: + build: + - python>=3.11 + - pip + - setuptools + - wheel host: - python>=3.11 + - setuptools run: - python>=3.11 - noaa-gfdl::analysis_scripts==0.0.1 diff --git a/pyproject.toml b/pyproject.toml index 83d7b44de..cfd4d7fce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,17 +11,14 @@ license = "Apache-2.0" authors = [ {name = "MSD Workflow Team", email="oar.gfdl.workflow@noaa.gov"} ] + keywords = [ "gfdl", "workflow", "fre", "fms", ] -# Some of these are conda dependencies (e.g., cdo, cmor) -# We should find out a better way to indicate those instead -# of listing them here. -# This package also requires some conda packages in the -# noaa-gfdl channel (e.g., analysis_scripts, catalogbuilder) + dependencies = [ 'analysis_scripts', 'catalogbuilder', @@ -50,12 +47,15 @@ dynamic = [ "version", ] + [tool.setuptools.dynamic] version = {attr = "fre.__version__"} + [project.scripts] fre = 'fre.fre:fre' + [project.optional-dependencies] docs = [ "sphinx", From 111f0f81dea8c1b3c7fc31a3dc086ea663f2bdea Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 30 Jan 2026 13:27:22 -0500 Subject: [PATCH 6/8] import pytest for mark.skipif --- fre/app/regrid_xy/tests/test_regrid_xy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fre/app/regrid_xy/tests/test_regrid_xy.py b/fre/app/regrid_xy/tests/test_regrid_xy.py index f9de2d9a4..4b209a30c 100644 --- a/fre/app/regrid_xy/tests/test_regrid_xy.py +++ b/fre/app/regrid_xy/tests/test_regrid_xy.py @@ -3,6 +3,7 @@ import shutil from shutil import which +import pytest import numpy as np import xarray as xr From 327edf15d57710bdbe686cfcda4bb38aeff30444 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 30 Jan 2026 13:51:46 -0500 Subject: [PATCH 7/8] light whitespace cleanup allfiles, adjust spacing of test_regrid_xy, add assert where it was missing and add a doc string here or there, clean up some commented out code, add some more helpful assert-failure statements --- .../tests/test_generate_time_averages.py | 13 +- fre/app/regrid_xy/tests/test_regrid_xy.py | 246 +++++++++--------- fre/tests/test_fre_app_cli.py | 2 +- meta.yaml | 2 +- 4 files changed, 132 insertions(+), 131 deletions(-) diff --git a/fre/app/generate_time_averages/tests/test_generate_time_averages.py b/fre/app/generate_time_averages/tests/test_generate_time_averages.py index f03ab5adb..2f7209b94 100644 --- a/fre/app/generate_time_averages/tests/test_generate_time_averages.py +++ b/fre/app/generate_time_averages/tests/test_generate_time_averages.py @@ -112,14 +112,12 @@ def test_time_avg_file_dir_exists(): pytest.param( 'cdo', 'all', True , FULL_TEST_FILE_PATH, STR_UNWGT_CDO_INF), -# TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TEST_FILE_NAME), pytest.param( 'cdo', 'all', True , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'timmean_unwgt_' + TWO_OUT_FILE_NAME), pytest.param( 'cdo', 'all', False , FULL_TEST_FILE_PATH, STR_CDO_INF), -# TIME_AVG_FILE_DIR + 'timmean_' + TEST_FILE_NAME), pytest.param( 'cdo', 'all', False , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'timmean_' + TWO_OUT_FILE_NAME), @@ -127,14 +125,12 @@ def test_time_avg_file_dir_exists(): pytest.param( 'fre-python-tools', 'all', False , FULL_TEST_FILE_PATH, STR_FRE_PYTOOLS_INF), -# TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TEST_FILE_NAME), pytest.param( 'fre-python-tools', 'all', False , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'frepytools_timavg_' + TWO_OUT_FILE_NAME), pytest.param( 'fre-python-tools', 'all', True , FULL_TEST_FILE_PATH, STR_UNWGT_FRE_PYTOOLS_INF), -# TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TEST_FILE_NAME), pytest.param( 'fre-python-tools', 'all', True , TWO_TEST_FILE_NAMES, TIME_AVG_FILE_DIR + 'frepytools_unwgt_timavg_' + TWO_OUT_FILE_NAME), @@ -160,7 +156,6 @@ def test_run_avgtype_pkg_calculations( pkg , unwgt , infile , outfile ): - ''' test-harness function, called by other test functions. ''' @@ -245,7 +240,7 @@ def test_compare_fre_cli_to_fre_nctools(): non_zero_count = np.count_nonzero( diff_pytools_nctools_timavg[:] ) #assert (non_zero_count == 0.) # bad way to check for zero. - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between frepy / frenctools were found" @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') @@ -275,7 +270,7 @@ def test_compare_fre_cli_to_cdo(): break non_zero_count = np.count_nonzero( diff_pytools_cdo_timavg[:] ) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frepytools were found" def test_compare_unwgt_fre_cli_to_unwgt_cdo(): @@ -304,7 +299,7 @@ def test_compare_unwgt_fre_cli_to_unwgt_cdo(): break non_zero_count = np.count_nonzero(diff_pytools_cdo_timavg[:]) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frepytools were found" @pytest.mark.xfail(reason = 'test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') def test_compare_cdo_to_fre_nctools(): @@ -334,7 +329,7 @@ def test_compare_cdo_to_fre_nctools(): break non_zero_count = np.count_nonzero(diff_cdo_nctools_timavg[:]) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ), "non-zero diffs between cdo / frenctools were found" # if we use fixtures and tmpdirs, can omit this error prone thing def test_fre_app_gen_time_avg_test_data_cleanup(): diff --git a/fre/app/regrid_xy/tests/test_regrid_xy.py b/fre/app/regrid_xy/tests/test_regrid_xy.py index 4b209a30c..6eeb2809b 100644 --- a/fre/app/regrid_xy/tests/test_regrid_xy.py +++ b/fre/app/regrid_xy/tests/test_regrid_xy.py @@ -1,3 +1,7 @@ +''' +tests for fre.app.regrid_xy submodule +''' + import os from pathlib import Path import shutil @@ -10,6 +14,8 @@ import fre.app.regrid_xy.regrid_xy as regrid_xy import fre.app.regrid_xy.tests.generate_files as generate_files +WHICH_FREGRID = which('fregrid') +HAVE_FREGRID = WHICH_FREGRID is not None and WHICH_FREGRID.split('/')[-1] == 'fregrid' nxy = 20 date = "20250729" @@ -50,141 +56,141 @@ def setup_test(): + ''' setup test directories and generate test files ''' + input_dir.mkdir(exist_ok=True) + output_dir.mkdir(exist_ok=True) + remap_dir.mkdir(exist_ok=True) + work_dir.mkdir(exist_ok=True) - input_dir.mkdir(exist_ok=True) - output_dir.mkdir(exist_ok=True) - remap_dir.mkdir(exist_ok=True) - work_dir.mkdir(exist_ok=True) - - #generate test files - generate_files.set_test(components_in=components, - date_in=date, - grid_spec_tar_in=str(grid_spec_tar), - yamlfile_in=str(yamlfile), - input_dir_in=str(input_dir)) - generate_files.make_all() + #generate test files + generate_files.set_test(components_in=components, + date_in=date, + grid_spec_tar_in=str(grid_spec_tar), + yamlfile_in=str(yamlfile), + input_dir_in=str(input_dir)) + generate_files.make_all() def cleanup_test(): - - #remove test directories - if output_dir.exists(): shutil.rmtree(output_dir) - if remap_dir.exists(): shutil.rmtree(remap_dir) - if work_dir.exists(): shutil.rmtree(work_dir) - generate_files.cleanup() - -@pytest.mark.skipif(which('fregrid') is None, + ''' remove test directories ''' + if output_dir.exists(): + shutil.rmtree(output_dir) + if remap_dir.exists(): + shutil.rmtree(remap_dir) + if work_dir.exists(): + shutil.rmtree(work_dir) + generate_files.cleanup() + +@pytest.mark.skipif(not HAVE_FREGRID, reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_regrid_xy(): - - """ - Tests the main function regrid_xy and ensures - data is regridded correctly - """ - - setup_test() - - #modify generate_files to change sources - for source_dict in pp_input_files + emma_input_files + here_input_files: - source = source_dict["history_file"] - regrid_xy.regrid_xy(yamlfile=str(yamlfile), - input_dir=str(input_dir), - output_dir=str(output_dir), - work_dir=str(work_dir), - remap_dir=str(remap_dir), - source=source, - input_date=date+"TTTT") - - #check answers - for source_dict in pp_input_files + emma_input_files: - # Files are now output to a subdirectory based on grid size and interpolation method - output_subdir = output_dir/f"{nxy}_{nxy}.conserve_order2" - outfile = output_subdir/f"{date}.{source_dict['history_file']}.nc" - - test = xr.load_dataset(outfile) - - assert "wet_c" not in test - assert "mister" in test - assert "darcy" in test - assert "wins" in test - - assert np.all(test["mister"].values==np.float64(1.0)) - assert np.all(test["darcy"].values==np.float64(2.0)) - assert np.all(test["wins"].values==np.float64(3.0)) - - #check answers, these shouldn't have been regridded - for source_dict in here_input_files: - ifile = source_dict["history_file"] - assert not (output_dir/f"{date}.{ifile}.nc").exists() - - #check remap_file exists and is not empty - remap_file = remap_dir/f"C{nxy}_mosaicX{nxy}by{nxy}_conserve_order2.nc" - assert remap_file.exists() - - cleanup_test() - - + """ + Tests the main function regrid_xy and ensures + data is regridded correctly + """ + + setup_test() + + #modify generate_files to change sources + for source_dict in pp_input_files + emma_input_files + here_input_files: + source = source_dict["history_file"] + regrid_xy.regrid_xy(yamlfile=str(yamlfile), + input_dir=str(input_dir), + output_dir=str(output_dir), + work_dir=str(work_dir), + remap_dir=str(remap_dir), + source=source, + input_date=date+"TTTT") + + #check answers + for source_dict in pp_input_files + emma_input_files: + # Files are now output to a subdirectory based on grid size and interpolation method + output_subdir = output_dir/f"{nxy}_{nxy}.conserve_order2" + outfile = output_subdir/f"{date}.{source_dict['history_file']}.nc" + + test = xr.load_dataset(outfile) + + assert "wet_c" not in test + assert "mister" in test + assert "darcy" in test + assert "wins" in test + + assert np.all(test["mister"].values==np.float64(1.0)) + assert np.all(test["darcy"].values==np.float64(2.0)) + assert np.all(test["wins"].values==np.float64(3.0)) + + #check answers, these shouldn't have been regridded + for source_dict in here_input_files: + ifile = source_dict["history_file"] + assert not (output_dir/f"{date}.{ifile}.nc").exists() + + #check remap_file exists and is not empty + remap_file = remap_dir/f"C{nxy}_mosaicX{nxy}by{nxy}_conserve_order2.nc" + assert remap_file.exists() + + cleanup_test() + +@pytest.mark.skipif(not HAVE_FREGRID, + reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_get_input_mosaic(): + """ + Tests get_input_mosaic correctly copies the mosaic file to the input directory + """ - """ - Tests get_input_mosaic correctly copies the mosaic file to the input directory - """ - - grid_spec = Path("grid_spec.nc") - mosaic_file = Path("ocean_mosaic.nc") - - generate_files.make_grid_spec() - mosaic_file.touch() + grid_spec = Path("grid_spec.nc") + mosaic_file = Path("ocean_mosaic.nc") - datadict=dict(grid_spec=grid_spec, inputRealm="ocean") + generate_files.make_grid_spec() + mosaic_file.touch() - assert regrid_xy.get_input_mosaic(datadict) == str(mosaic_file) + datadict=dict(grid_spec=grid_spec, inputRealm="ocean") - mosaic_file.unlink() #clean up - grid_spec.unlink() #clean up + assert regrid_xy.get_input_mosaic(datadict) == str(mosaic_file) + mosaic_file.unlink() #clean up + grid_spec.unlink() #clean up +@pytest.mark.skipif(not HAVE_FREGRID, + reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_get_input_file(): + """ + Tests get_input_file + """ - """ - Tests get_input_file - """ - - input_date = "20250807" - source = "pemberley" - datadict = {"input_date": input_date} - assert regrid_xy.get_input_file(datadict, source) == input_date + "." + source - - datadict["input_date"] = None - assert regrid_xy.get_input_file(datadict, source) == source + input_date = "20250807" + source = "pemberley" + datadict = {"input_date": input_date} + assert regrid_xy.get_input_file(datadict, source) == input_date + "." + source + datadict["input_date"] = None + assert regrid_xy.get_input_file(datadict, source) == source +@pytest.mark.skipif(not HAVE_FREGRID, + reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_get_remap_file(): - - """ - Tests get_remap_file - """ - - remap_dir = Path("remap_dir") - input_mosaic = "C20_mosaic" - nlon = 40 - nlat = 10 - interp_method = "conserve_order1" - - datadict = {"remap_dir": remap_dir.name, - "input_mosaic": input_mosaic+".nc", - "output_nlon": nlon, - "output_nlat": nlat, - "interp_method": interp_method} - - #check remap file from current directory is copied to input directory - remap_file = Path(f"remap_dir/{input_mosaic}X{nlon}by{nlat}_{interp_method}.nc") - - regrid_xy.get_remap_file(datadict) == str(remap_dir/remap_file) - - remap_dir.mkdir(exist_ok=True) - remap_file.touch() - regrid_xy.get_remap_file(datadict) == str(remap_dir/remap_file) - - Path(remap_file).unlink() - shutil.rmtree(remap_dir) + """ + Tests get_remap_file + """ + + remap_dir = Path("remap_dir") + input_mosaic = "C20_mosaic" + nlon = 40 + nlat = 10 + interp_method = "conserve_order1" + + datadict = {"remap_dir": remap_dir.name, + "input_mosaic": input_mosaic+".nc", + "output_nlon": nlon, + "output_nlat": nlat, + "interp_method": interp_method} + + #check remap file from current directory is copied to input directory + remap_file = Path(f"remap_dir/{input_mosaic}X{nlon}by{nlat}_{interp_method}.nc") + assert regrid_xy.get_remap_file(datadict) == str(remap_dir/remap_file) + + remap_dir.mkdir(exist_ok=True) + remap_file.touch() + assert regrid_xy.get_remap_file(datadict) == str(remap_dir/remap_file) + + Path(remap_file).unlink() + shutil.rmtree(remap_dir) diff --git a/fre/tests/test_fre_app_cli.py b/fre/tests/test_fre_app_cli.py index 7cc1223ca..823ee2de7 100644 --- a/fre/tests/test_fre_app_cli.py +++ b/fre/tests/test_fre_app_cli.py @@ -194,7 +194,7 @@ def test_cli_fre_app_regrid_opt_dne(capfd): assert result.exit_code == 2 _out, _err = capfd.readouterr() -@pytest.mark.skipif(which('fregrid') is None, +@pytest.mark.skipif(which('fregrid') is None, reason='fregrid not in env. it was removed from package reqs. you must load it externally') def test_cli_fre_app_regrid_test_case_1(capfd): """ fre app regrid_xy --help """ diff --git a/meta.yaml b/meta.yaml index 3860b7508..e4d33acea 100644 --- a/meta.yaml +++ b/meta.yaml @@ -80,7 +80,7 @@ test: - fre run --help - fre yamltools --help - pylint --rcfile pylintrc fre/ - - pytest --durations=20 --log-level INFO --ignore=fre/make/tests/compilation fre/ + - pytest --durations=20 --log-level INFO --ignore=fre/make/tests/compilation fre/ about: From 31a3b108cdc834d3358b86162b90d654f82ee5ab Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 30 Jan 2026 15:00:26 -0500 Subject: [PATCH 8/8] Document PR #707 dependency changes: Python >=3.11, fregrid unbundled (#712) * Initial plan * Add comprehensive documentation for PR #707 dependency changes Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> * Update regrid_xy README: Python >=3.11, fregrid no longer bundled Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> * Add optional fre-nctools loading instructions to setup sections Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> * Revise fre-nctools loading instructions in README Updated instructions for loading fre-nctools and added notes on installation options. * Add optional fre-nctools loading instructions to docs/ setup sections Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- CONTRIBUTING.md | 1 + README.md | 5 +++++ docs/contributing_to_doc.rst | 3 +++ docs/setup.rst | 3 +++ fre/app/regrid_xy/README.md | 2 +- 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7cccb8f1e..8cc56bc31 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,6 +3,7 @@ * Developers are free to use this repository's `README.md` to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless * Gain access to the repository with `git clone --recursive git@github.com:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key - Once inside the repository, developers can test local changes by running a `pip install .` inside of the root directory to install the fre-cli package locally with the newest local changes on top of the installed Conda fre-cli dependencies + - Optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines (e.g., `module load fre-nctools` on GFDL/Gaea systems) - Test as a normal user would use the CLI * Create a GitHub issue to reflect your contribution's background and reference it with Git commits diff --git a/README.md b/README.md index cd95effc8..c0cc7c4f6 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,11 @@ conda config --append channels noaa-gfdl conda config --append channels conda-forge conda create --name fre-2025.04 --channel noaa-gfdl --channel conda-forge fre-cli::2025.04 conda activate fre-2025.04 +# optional: install or load fre-nctools to gain access to regridding and certain time-averaging routines +# add to your path like: export PATH=/path/to/your/fre-nctools/build/bin:$PATH +# or if you have lmod/modules: module load fre-nctools/ +# or compile/install from source: see github.comNOAA-GDFL/FRE-NCTools documentation on compilation/installation +# DO NOT USE noaa-gfdl::fre-nctools==2022.01 at this time, it is being deprecated ``` ## GFDL/RDHPCS deployment notes diff --git a/docs/contributing_to_doc.rst b/docs/contributing_to_doc.rst index ea1924979..f343413e4 100644 --- a/docs/contributing_to_doc.rst +++ b/docs/contributing_to_doc.rst @@ -90,6 +90,9 @@ From the root-directory of your local repository copy: # Activate your fre-cli environment conda activate fre-cli + # optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines + # module load fre-nctools + # Install documentation dependencies pip install .[docs] diff --git a/docs/setup.rst b/docs/setup.rst index cc5721d41..55746b776 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -42,3 +42,6 @@ Assuming one has ``conda`` in their path, then do the following:: # now we pip install the local code under the `fre/` directory # the -e flag makes re-installing the code after editing not necessary pip install -e . + + # optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines + # module load fre-nctools diff --git a/fre/app/regrid_xy/README.md b/fre/app/regrid_xy/README.md index 80964521e..2d04660d9 100644 --- a/fre/app/regrid_xy/README.md +++ b/fre/app/regrid_xy/README.md @@ -2,7 +2,7 @@ `regrid_xy.py` remaps scalar and/or vector fields from one kind of lat/lon grid to another. It can remap between different grids of the same type (e.g. spherical), and between grids of different types (e.g. spherical to tripolar). By default, it uses an O(1) conservative interpolation scheme to accomplish the regridding, except under certain conditions [defined within `fregrid`](https://github.com/NOAA-GFDL/FRE-NCtools/blob/master/tools/fregrid/fregrid.c#L915-L920) the underlying CLI tool which does the heavy lifting. -requires `fre-nctools` and `fregrid` to be in one's `PATH` variable, and `python3` (tested/developed with python 3.9.16). there should be `netCDF4` and `metomi` python modules in one's python environment for imports. `pytest` and `nccmp` is required for tests. `pylint` recommended for future developers working on this tool. +requires `fregrid` to be in one's `PATH` variable (`fre-nctools` no longer bundled; load via `module load fre-nctools`), and `python3` >= 3.11 (tested/developed with python 3.12). there should be `netCDF4` and `xarray` python modules in one's python environment for imports. `pytest` and `nccmp` is required for tests. `pylint` recommended for future developers working on this tool. # INPUT PARAMETERS (mandatory, env vars)