Update fetch strategy

src/jade/app/app.py

@@ -12,7 +12,7 @@
 
 import jade.resources as res
 from jade import resources
-from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs
+from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_nonIAEA_exp_data
 from jade.config.paths_tree import PathsTree
 from jade.config.pp_config import PostProcessConfig
 from jade.config.raw_config import ConfigRawProcessor
@@ -91,18 +91,24 @@ def update_inputs(self):
 
         This will re-fetch inputs from the various repositories that feed JADE.
         """
+        # install IAEA inputs and experimental data
         success = fetch_iaea_inputs(
             self.tree.benchmark_input_templates, self.tree.exp_data
         )
         if not success:
             logging.error("Failed to update the IAEA benchmark inputs.")
 
+        # Install F4E exp data
+        success = fetch_nonIAEA_exp_data(self.tree.exp_data)
+        if not success:
+            logging.error("Failed to update the F4E experimental data.")
+
+        # Install F4E inputs
         f4e_gitlab_token = os.getenv("F4E_GITLAB_TOKEN")
 
         if f4e_gitlab_token is not None:
             success = fetch_f4e_inputs(
                 self.tree.benchmark_input_templates,
-                self.tree.exp_data,
                 f4e_gitlab_token,
             )
             if not success:

src/jade/app/fetch.py

@@ -12,8 +12,8 @@
 
 from jade.helper.aux_functions import PathLike
 
-BRANCH = "main"  # TODO change in main once merged the PR
-IAEA_URL = f"https://github.com/IAEA-NDS/open-benchmarks/archive/{BRANCH}.zip"
+IAEA_URL = r"https://github.com/IAEA-NDS/open-benchmarks/archive/main.zip"
+RAW_DATA_GITHUB_URL = r"https://github.com/JADE-V-V/JADE-RAW-RESULTS/archive/main.zip"
 
 
 def _fetch_from_git(
@@ -128,14 +128,23 @@ def _install_standard_folder_structure(
     exp_data_root: PathLike,
     path_to_inputs: str | os.PathLike,
     path_to_exp_data: str | os.PathLike,
+    only_exp_data: bool = False,
+    only_inputs: bool = False,
 ) -> bool:
     if isinstance(extracted_folder, bool):
         return False
 
-    for fetched_folder, install_folder in [
-        (path_to_inputs, inputs_root),
-        (path_to_exp_data, exp_data_root),
-    ]:
+    if only_exp_data:
+        to_install = [(path_to_exp_data, exp_data_root)]
+    elif only_inputs:
+        to_install = [(path_to_inputs, inputs_root)]
+    else:
+        to_install = [
+            (path_to_inputs, inputs_root),
+            (path_to_exp_data, exp_data_root),
+        ]
+
+    for fetched_folder, install_folder in to_install:
         _install_data(fetched_folder, install_folder)
 
     # Once done, delete the src folder
@@ -175,19 +184,15 @@ def fetch_iaea_inputs(inputs_root: PathLike, exp_data_root: PathLike) -> bool:
     return success
 
 
-def fetch_f4e_inputs(
-    inputs_root: PathLike, exp_data_root: PathLike, access_token: str
-) -> bool:
-    """Fetch F4E benchmark inputs and experimental data and copy them to
+def fetch_f4e_inputs(inputs_root: PathLike, access_token: str) -> bool:
+    """Fetch F4E benchmark inputs and copy them to
     the correct folder in jade structure. This will always override the available
     data.
 
     Parameters
     ----------
     inputs_root : PathLike
         path to the root folder where the inputs will be stored.
-    exp_data_root : PathLike
-        path to the root folder where the experimental data will be stored.
     access_token : str
         Authorization token to access the F4E GitLab.
 
@@ -207,11 +212,43 @@ def fetch_f4e_inputs(
     if not isinstance(extracted_folder, PathLike):  # anything else that went wrong
         return False
     path_to_inputs = Path(extracted_folder, "inputs")
+
+    success = _install_standard_folder_structure(
+        extracted_folder,
+        inputs_root,
+        "",
+        path_to_inputs,
+        "",
+        only_inputs=True,
+    )
+    return success
+
+
+def fetch_nonIAEA_exp_data(exp_data_root: PathLike) -> bool:
+    """Fetch non IAEA benchmark experimental data and copy them to
+    the correct folder in jade structure. This will always override the available
+    data.
+
+    Parameters
+    ----------
+    exp_data_root : PathLike
+        path to the root folder where the experimental data will be stored.
+
+    Returns
+    -------
+    bool
+        True if the experimental data were successfully fetched, False otherwise.
+    """
+    extracted_folder = str(
+        _fetch_from_git(RAW_DATA_GITHUB_URL)
+    )  # no token required anymore
+
     path_to_exp_data = Path(
         extracted_folder,
-        "exp_results",
+        "ROOT",
+        "_exp_-_exp_",
     )
     success = _install_standard_folder_structure(
-        extracted_folder, inputs_root, exp_data_root, path_to_inputs, path_to_exp_data
+        extracted_folder, "", exp_data_root, "", path_to_exp_data, only_exp_data=True
     )
     return success

src/jade/utilities.py

@@ -1,7 +1,6 @@
 import argparse
 
 from jade.app.app import JadeApp
-from jade.helper.aux_functions import add_rmode0
 
 
 def main():

tests/app/test_fetch.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs
+from jade.app.fetch import fetch_f4e_inputs, fetch_iaea_inputs, fetch_nonIAEA_exp_data
 
 # By default this should set the token to None if not found
 F4E_GITLAB_TOKEN = os.getenv("F4E_GITLAB_TOKEN")
@@ -46,27 +46,36 @@ def test_wrong_fetch_f4e_inputs(tmpdir):
     test also the overwriting"""
     # test correct fetching in an empty folder
     inp_path = tmpdir.mkdir("inputs")
-    exp_path = tmpdir.mkdir("exp")
-    success = fetch_f4e_inputs(inp_path, exp_path, access_token="wrongtoken")
+    success = fetch_f4e_inputs(inp_path, access_token="wrongtoken")
     assert not success
 
 
 @pytest.mark.skipif(
     F4E_GITLAB_TOKEN is None or os.environ.get("GITHUB_ACTIONS") == "true",
-    reason="No token found or running on GitHub CI"
+    reason="No token found or running on GitHub CI",
 )
 def test_fetch_f4e_inputs(tmpdir):
     assert F4E_GITLAB_TOKEN is not None
     # test correct fetching in an empty folder
     inp_path = tmpdir.mkdir("inputs")
-    exp_path = tmpdir.mkdir("exp")
-    success = fetch_f4e_inputs(inp_path, exp_path, F4E_GITLAB_TOKEN)
+    success = fetch_f4e_inputs(inp_path, F4E_GITLAB_TOKEN)
     assert success
     assert len(os.listdir(inp_path)) > 0
-    assert len(os.listdir(exp_path)) > 0
 
     # test that there no problems when the folder is not empty
-    success = fetch_f4e_inputs(inp_path, exp_path, F4E_GITLAB_TOKEN)
+    success = fetch_f4e_inputs(inp_path, F4E_GITLAB_TOKEN)
     assert success
     assert len(os.listdir(inp_path)) > 0
+
+
+def test_fetch_f4e_exp_data(tmpdir):
+    """ " Test that experimental data can be correctly fetched from the F4E GitLab."""
+    exp_path = tmpdir.mkdir("exp")
+    success = fetch_nonIAEA_exp_data(exp_path)
+    assert success
+    assert len(os.listdir(exp_path)) > 0
+
+    # test that there no problems when the folder is not empty
+    success = fetch_nonIAEA_exp_data(exp_path)
+    assert success
     assert len(os.listdir(exp_path)) > 0