From ee0795b1b0f11786aa1d17c69f6e70e3c5b0c915 Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Fri, 5 Dec 2025 17:31:47 +0100 Subject: [PATCH 1/3] update fetch strategy --- src/jade/app/app.py | 10 +++++-- src/jade/app/fetch.py | 63 ++++++++++++++++++++++++++++++++--------- src/jade/utilities.py | 1 - tests/app/test_fetch.py | 25 ++++++++++------ 4 files changed, 75 insertions(+), 24 deletions(-) diff --git a/src/jade/app/app.py b/src/jade/app/app.py index 11db7e0c..4cd46aa0 100644 --- a/src/jade/app/app.py +++ b/src/jade/app/app.py @@ -12,7 +12,7 @@ import jade.resources as res from jade import resources -from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs +from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_f4e_exp_data from jade.config.paths_tree import PathsTree from jade.config.pp_config import PostProcessConfig from jade.config.raw_config import ConfigRawProcessor @@ -91,18 +91,24 @@ def update_inputs(self): This will re-fetch inputs from the various repositories that feed JADE. """ + # install IAEA inputs and experimental data success = fetch_iaea_inputs( self.tree.benchmark_input_templates, self.tree.exp_data ) if not success: logging.error("Failed to update the IAEA benchmark inputs.") + # Install F4E exp data + success = fetch_f4e_exp_data(self.tree.exp_data) + if not success: + logging.error("Failed to update the F4E experimental data.") + + # Install F4E inputs f4e_gitlab_token = os.getenv("F4E_GITLAB_TOKEN") if f4e_gitlab_token is not None: success = fetch_f4e_inputs( self.tree.benchmark_input_templates, - self.tree.exp_data, f4e_gitlab_token, ) if not success: diff --git a/src/jade/app/fetch.py b/src/jade/app/fetch.py index 73be4933..13173edc 100644 --- a/src/jade/app/fetch.py +++ b/src/jade/app/fetch.py @@ -12,8 +12,8 @@ from jade.helper.aux_functions import PathLike -BRANCH = "main" # TODO change in main once merged the PR -IAEA_URL = f"https://github.com/IAEA-NDS/open-benchmarks/archive/{BRANCH}.zip" +IAEA_URL = r"https://github.com/IAEA-NDS/open-benchmarks/archive/main.zip" +RAW_DATA_GITHUB_URL = r"https://github.com/JADE-V-V/JADE-RAW-RESULTS/archive/main.zip" def _fetch_from_git( @@ -128,14 +128,23 @@ def _install_standard_folder_structure( exp_data_root: PathLike, path_to_inputs: str | os.PathLike, path_to_exp_data: str | os.PathLike, + only_exp_data: bool = False, + only_inputs: bool = False, ) -> bool: if isinstance(extracted_folder, bool): return False - for fetched_folder, install_folder in [ - (path_to_inputs, inputs_root), - (path_to_exp_data, exp_data_root), - ]: + if only_exp_data: + to_install = [(path_to_exp_data, exp_data_root)] + elif only_inputs: + to_install = [(path_to_inputs, inputs_root)] + else: + to_install = [ + (path_to_inputs, inputs_root), + (path_to_exp_data, exp_data_root), + ] + + for fetched_folder, install_folder in to_install: _install_data(fetched_folder, install_folder) # Once done, delete the src folder @@ -175,9 +184,7 @@ def fetch_iaea_inputs(inputs_root: PathLike, exp_data_root: PathLike) -> bool: return success -def fetch_f4e_inputs( - inputs_root: PathLike, exp_data_root: PathLike, access_token: str -) -> bool: +def fetch_f4e_inputs(inputs_root: PathLike, access_token: str) -> bool: """Fetch F4E benchmark inputs and experimental data and copy them to the correct folder in jade structure. This will always override the available data. @@ -186,8 +193,6 @@ def fetch_f4e_inputs( ---------- inputs_root : PathLike path to the root folder where the inputs will be stored. - exp_data_root : PathLike - path to the root folder where the experimental data will be stored. access_token : str Authorization token to access the F4E GitLab. @@ -207,11 +212,43 @@ def fetch_f4e_inputs( if not isinstance(extracted_folder, PathLike): # anything else that went wrong return False path_to_inputs = Path(extracted_folder, "inputs") + + success = _install_standard_folder_structure( + extracted_folder, + inputs_root, + "", + path_to_inputs, + "", + only_inputs=True, + ) + return success + + +def fetch_f4e_exp_data(exp_data_root: PathLike) -> bool: + """Fetch F4E benchmark experimental data and copy them to + the correct folder in jade structure. This will always override the available + data. + + Parameters + ---------- + exp_data_root : PathLike + path to the root folder where the experimental data will be stored. + + Returns + ------- + bool + True if the experimental data were successfully fetched, False otherwise. + """ + extracted_folder = str( + _fetch_from_git(RAW_DATA_GITHUB_URL) + ) # no token required anymore + path_to_exp_data = Path( extracted_folder, - "exp_results", + "ROOT", + "_exp_-_exp_", ) success = _install_standard_folder_structure( - extracted_folder, inputs_root, exp_data_root, path_to_inputs, path_to_exp_data + extracted_folder, "", exp_data_root, "", path_to_exp_data, only_exp_data=True ) return success diff --git a/src/jade/utilities.py b/src/jade/utilities.py index 8e58f9a4..978e2eec 100644 --- a/src/jade/utilities.py +++ b/src/jade/utilities.py @@ -1,7 +1,6 @@ import argparse from jade.app.app import JadeApp -from jade.helper.aux_functions import add_rmode0 def main(): diff --git a/tests/app/test_fetch.py b/tests/app/test_fetch.py index e05dfd65..ce0c4e65 100644 --- a/tests/app/test_fetch.py +++ b/tests/app/test_fetch.py @@ -4,7 +4,7 @@ import pytest -from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs +from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_f4e_exp_data # By default this should set the token to None if not found F4E_GITLAB_TOKEN = os.getenv("F4E_GITLAB_TOKEN") @@ -46,27 +46,36 @@ def test_wrong_fetch_f4e_inputs(tmpdir): test also the overwriting""" # test correct fetching in an empty folder inp_path = tmpdir.mkdir("inputs") - exp_path = tmpdir.mkdir("exp") - success = fetch_f4e_inputs(inp_path, exp_path, access_token="wrongtoken") + success = fetch_f4e_inputs(inp_path, access_token="wrongtoken") assert not success @pytest.mark.skipif( F4E_GITLAB_TOKEN is None or os.environ.get("GITHUB_ACTIONS") == "true", - reason="No token found or running on GitHub CI" + reason="No token found or running on GitHub CI", ) def test_fetch_f4e_inputs(tmpdir): assert F4E_GITLAB_TOKEN is not None # test correct fetching in an empty folder inp_path = tmpdir.mkdir("inputs") - exp_path = tmpdir.mkdir("exp") - success = fetch_f4e_inputs(inp_path, exp_path, F4E_GITLAB_TOKEN) + success = fetch_f4e_inputs(inp_path, F4E_GITLAB_TOKEN) assert success assert len(os.listdir(inp_path)) > 0 - assert len(os.listdir(exp_path)) > 0 # test that there no problems when the folder is not empty - success = fetch_f4e_inputs(inp_path, exp_path, F4E_GITLAB_TOKEN) + success = fetch_f4e_inputs(inp_path, F4E_GITLAB_TOKEN) assert success assert len(os.listdir(inp_path)) > 0 + + +def test_fetch_f4e_exp_data(tmpdir): + """ " Test that experimental data can be correctly fetched from the F4E GitLab.""" + exp_path = tmpdir.mkdir("exp") + success = fetch_f4e_exp_data(exp_path) + assert success + assert len(os.listdir(exp_path)) > 0 + + # test that there no problems when the folder is not empty + success = fetch_f4e_exp_data(exp_path) + assert success assert len(os.listdir(exp_path)) > 0 From 293d4dd3409f0a869f25b95d99a66b2364755dbd Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Fri, 5 Dec 2025 17:47:02 +0100 Subject: [PATCH 2/3] correct docstring --- src/jade/app/fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jade/app/fetch.py b/src/jade/app/fetch.py index 13173edc..076c5eca 100644 --- a/src/jade/app/fetch.py +++ b/src/jade/app/fetch.py @@ -185,7 +185,7 @@ def fetch_iaea_inputs(inputs_root: PathLike, exp_data_root: PathLike) -> bool: def fetch_f4e_inputs(inputs_root: PathLike, access_token: str) -> bool: - """Fetch F4E benchmark inputs and experimental data and copy them to + """Fetch F4E benchmark inputs and copy them to the correct folder in jade structure. This will always override the available data. From 3ba402bee05efd3f4110ce6871b941405664c4b3 Mon Sep 17 00:00:00 2001 From: Davide Laghi Date: Mon, 15 Dec 2025 09:19:55 +0100 Subject: [PATCH 3/3] rename fetch function --- src/jade/app/app.py | 4 ++-- src/jade/app/fetch.py | 4 ++-- tests/app/test_fetch.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jade/app/app.py b/src/jade/app/app.py index 4cd46aa0..0035cfe6 100644 --- a/src/jade/app/app.py +++ b/src/jade/app/app.py @@ -12,7 +12,7 @@ import jade.resources as res from jade import resources -from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_f4e_exp_data +from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_nonIAEA_exp_data from jade.config.paths_tree import PathsTree from jade.config.pp_config import PostProcessConfig from jade.config.raw_config import ConfigRawProcessor @@ -99,7 +99,7 @@ def update_inputs(self): logging.error("Failed to update the IAEA benchmark inputs.") # Install F4E exp data - success = fetch_f4e_exp_data(self.tree.exp_data) + success = fetch_nonIAEA_exp_data(self.tree.exp_data) if not success: logging.error("Failed to update the F4E experimental data.") diff --git a/src/jade/app/fetch.py b/src/jade/app/fetch.py index 076c5eca..48b2ad66 100644 --- a/src/jade/app/fetch.py +++ b/src/jade/app/fetch.py @@ -224,8 +224,8 @@ def fetch_f4e_inputs(inputs_root: PathLike, access_token: str) -> bool: return success -def fetch_f4e_exp_data(exp_data_root: PathLike) -> bool: - """Fetch F4E benchmark experimental data and copy them to +def fetch_nonIAEA_exp_data(exp_data_root: PathLike) -> bool: + """Fetch non IAEA benchmark experimental data and copy them to the correct folder in jade structure. This will always override the available data. diff --git a/tests/app/test_fetch.py b/tests/app/test_fetch.py index ce0c4e65..c0f28d93 100644 --- a/tests/app/test_fetch.py +++ b/tests/app/test_fetch.py @@ -4,7 +4,7 @@ import pytest -from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_f4e_exp_data +from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_nonIAEA_exp_data # By default this should set the token to None if not found F4E_GITLAB_TOKEN = os.getenv("F4E_GITLAB_TOKEN") @@ -71,11 +71,11 @@ def test_fetch_f4e_inputs(tmpdir): def test_fetch_f4e_exp_data(tmpdir): """ " Test that experimental data can be correctly fetched from the F4E GitLab.""" exp_path = tmpdir.mkdir("exp") - success = fetch_f4e_exp_data(exp_path) + success = fetch_nonIAEA_exp_data(exp_path) assert success assert len(os.listdir(exp_path)) > 0 # test that there no problems when the folder is not empty - success = fetch_f4e_exp_data(exp_path) + success = fetch_nonIAEA_exp_data(exp_path) assert success assert len(os.listdir(exp_path)) > 0