From e6d7e6510bfd14200031451b05bd071c6175d166 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 16 May 2025 09:21:58 +1000 Subject: [PATCH 01/16] added build script --- build.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 build.sh diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..3fcbaaf --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ + +#!/bin/bash +python -m venv .venv +source venv/bin/activate +pip install -r requirements.txt +pip install -e . \ No newline at end of file From 6aaab802d5911826f4437fc8d073c29bc70a1114 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 16 May 2025 09:34:22 +1000 Subject: [PATCH 02/16] adding ipykernal to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b084c45..0a7f705 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ requests pandas setuptools pytest -PyJWT \ No newline at end of file +PyJWT +ipykernal \ No newline at end of file From ba8545d23ea7846b46943db363c14f52ddbad942 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 16 May 2025 09:35:51 +1000 Subject: [PATCH 03/16] fixed typo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0a7f705..0f3daad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ pandas setuptools pytest PyJWT -ipykernal \ No newline at end of file +ipykernel \ No newline at end of file From 03eeeed830262aa97c67135d5125c67b61c008a6 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 16 May 2025 09:36:29 +1000 Subject: [PATCH 04/16] added upgrade command --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 3fcbaaf..f0169bb 100644 --- a/build.sh +++ b/build.sh @@ -2,5 +2,6 @@ #!/bin/bash python -m venv .venv source venv/bin/activate +pip install --upgrade pip pip install -r requirements.txt pip install -e . \ No newline at end of file From 2c73200ec0ec0172025c4fb7d3748596ed7c18bd Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Mon, 19 May 2025 15:36:11 +1000 Subject: [PATCH 05/16] added load dotenv --- README.md | 3 +++ protyping.ipynb | 14 ++++++++++++-- requirements.txt | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2fc6584..ad1e3a6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,10 @@ pip install -e . ```python import os from gen3_metadata.parser import Gen3MetadataParser +from dotenv import load_dotenv +load_dotenv() +key_file = os.getenv('credentials_path') # Set up credentials path key_file = os.getenv('credentials_path') diff --git a/protyping.ipynb b/protyping.ipynb index 9725256..9ea4df2 100644 --- a/protyping.ipynb +++ b/protyping.ipynb @@ -19,14 +19,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", + "from dotenv import load_dotenv\n", "from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser\n", + "load_dotenv()\n", + "key_file = os.getenv('credentials_path')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "\n", - "key_file = os.getenv('credentials_path')\n", "gen3metadata = Gen3MetadataParser(key_file)\n", "gen3metadata.authenticate()" ] diff --git a/requirements.txt b/requirements.txt index 0f3daad..522f559 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ pandas setuptools pytest PyJWT -ipykernel \ No newline at end of file +ipykernel +dotenv \ No newline at end of file From ef1e4ff5ab7945c57bab9aeb2015e29258fa9a93 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Mon, 19 May 2025 15:36:38 +1000 Subject: [PATCH 06/16] added extra method to convert credential json with no quotes --- src/gen3_metadata/gen3_metadata_parser.py | 38 ++++++++++- tests/test_gen3_metadata_parser.py | 78 ++++++++++++++++++++--- 2 files changed, 106 insertions(+), 10 deletions(-) diff --git a/src/gen3_metadata/gen3_metadata_parser.py b/src/gen3_metadata/gen3_metadata_parser.py index 4499b96..f9b42b0 100644 --- a/src/gen3_metadata/gen3_metadata_parser.py +++ b/src/gen3_metadata/gen3_metadata_parser.py @@ -2,6 +2,7 @@ import requests import pandas as pd import jwt +import re class Gen3MetadataParser: @@ -20,6 +21,20 @@ def __init__(self, key_file_path): self.headers = {} self.data_store = {} self.data_store_pd = {} + + def _add_quotes_to_json(self, input_str): + try: + # Try parsing as-is + return json.loads(input_str) + except json.JSONDecodeError: + # Add quotes around keys + fixed = re.sub(r'([{,]\s*)(\w+)\s*:', r'\1"\2":', input_str) + # Add quotes around simple string values (skip existing quoted values) + fixed = re.sub(r':\s*([A-Za-z0-9._:@/-]+)(?=\s*[},])', r': "\1"', fixed) + try: + return json.loads(fixed) + except json.JSONDecodeError as e: + raise ValueError(f"Could not fix JSON: {e}") def _load_api_key(self) -> dict: """ @@ -28,8 +43,27 @@ def _load_api_key(self) -> dict: Returns: dict: The API key loaded from the JSON file. """ - with open(self.key_file_path) as json_file: - return json.load(json_file) + try: + # Read the file as plain text + with open(self.key_file_path, "r") as f: + content = f.read() + # If the content does not contain any double or single quotes, try to fix it + if '"' not in content and "'" not in content: + return self._add_quotes_to_json(content) + + # Read the file as JSON + with open(self.key_file_path) as json_file: + return json.load(json_file) + except FileNotFoundError as fnf_err: + print(f"File not found: {fnf_err}") + raise + except json.JSONDecodeError as json_err: + print(f"JSON decode error: {json_err}") + print("Please make sure the file contains valid JSON with quotes and proper formatting.") + raise + except Exception as err: + print(f"An unexpected error occurred while loading API key: {err}") + raise def _url_from_jwt(self, cred: dict) -> str: """ diff --git a/tests/test_gen3_metadata_parser.py b/tests/test_gen3_metadata_parser.py index dbc553e..400a6a4 100644 --- a/tests/test_gen3_metadata_parser.py +++ b/tests/test_gen3_metadata_parser.py @@ -9,27 +9,89 @@ @pytest.fixture def fake_api_key(): - """Fixture to provide a fake API key. Note: these credentials have been inactivated""" + """Fixture to provide a fake API key. Note: these credentials have been inactivated.""" + # This is a valid JWT and UUID, but is not active. return { - "api_key": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eXAiOiJKV1QifQ.eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczovL2RhdGEudGVzdC5iaW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZGF0YS50ZXN0LmJpb2NvbW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLCJleHAiOjE3NDQ4NDU0ODAsImp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2JlNGVkMTIyOSIsImF6cCI6IiIsInNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9wZW5pZCIsImdvb2dsZV9zZXJ2aWNlX2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nbGVfbGluayIsImFkbWluIiwidXNlciIsImdhNGdoX3Bhc3Nwb3J0X3YxIl19.SGPjs6ljCJbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cnjduhWRKnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0vQJzrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqpMU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXLU3vSdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA", + "api_key": ( + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eXAiOiJKV1QifQ." + "eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczovL2RhdGEudGVzdC5i" + "aW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZGF0YS50ZXN0LmJpb2Nv" + "bW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLCJleHAiOjE3NDQ4NDU0ODAs" + "Imp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2JlNGVkMTIyOSIsImF6cCI6IiIs" + "InNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9wZW5pZCIsImdvb2dsZV9zZXJ2aWNl" + "X2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nbGVfbGluayIsImFkbWluIiwidXNlciIs" + "ImdhNGdoX3Bhc3Nwb3J0X3YxIl19." + "SGPjs6ljCJbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cn" + "jduhWRKnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0" + "vQJzrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqp" + "MU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXLU3v" + "SdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA" + ), "key_id": "b9042701-b08f-40da-89a3-753be4ed1229" } @pytest.fixture def gen3_metadata_parser(): """Fixture to create a Gen3MetadataParser instance.""" - return Gen3MetadataParser( - key_file_path="fake_credentials.json" - ) + return Gen3MetadataParser(key_file_path="fake_credentials.json") +@pytest.fixture +def malformed_json_credentials(): + """Fixture for malformed JSON credentials (no quotes, not valid Python dict).""" + # This is a string, not a dict, to simulate malformed file content. + return '{api_key: abc.def.ghi, key_id: 18bdaa-b018}' + +def test_add_quotes_to_json_valid(gen3_metadata_parser): + """Test _add_quotes_to_json with valid JSON (should parse as-is).""" + valid_json = '{"api_key": "abc.def.ghi", "key_id": "18bdaa-b018"}' + result = gen3_metadata_parser._add_quotes_to_json(valid_json) + assert result == {"api_key": "abc.def.ghi", "key_id": "18bdaa-b018"} + +def test_add_quotes_to_json_malformed(gen3_metadata_parser): + """Test _add_quotes_to_json with malformed JSON (no quotes).""" + malformed = '{api_key: abc.def.ghi, key_id: 18bdaa-b018}' + result = gen3_metadata_parser._add_quotes_to_json(malformed) + assert result == {"api_key": "abc.def.ghi", "key_id": "18bdaa-b018"} + +def test_add_quotes_to_json_url_and_uuid(gen3_metadata_parser): + """Test _add_quotes_to_json with keys/values including url and uuid.""" + malformed = '{key1: value1, key2:123, url: https://example.com, uuid: 18bdaa-b018}' + result = gen3_metadata_parser._add_quotes_to_json(malformed) + assert result == { + "key1": "value1", + "key2": "123", + "url": "https://example.com", + "uuid": "18bdaa-b018" + } + +def test_add_quotes_to_json_invalid(gen3_metadata_parser): + """Test _add_quotes_to_json with unrecoverable malformed JSON.""" + bad = '{key1 value1, key2:}' + with pytest.raises(ValueError): + gen3_metadata_parser._add_quotes_to_json(bad) -def test_load_api_key(gen3_metadata_parser, fake_api_key): - """Test the _load_api_key method.""" - # Mock open() to simulate reading the fake API key from a file +def test_load_api_key_valid_json(gen3_metadata_parser, fake_api_key): + """Test the _load_api_key method with valid JSON file content.""" + # Simulate reading a valid JSON file with patch("builtins.open", mock_open(read_data=json.dumps(fake_api_key))): result = gen3_metadata_parser._load_api_key() assert result == fake_api_key +def test_load_api_key_malformed_json(gen3_metadata_parser, malformed_json_credentials): + """Test the _load_api_key method with malformed JSON (no quotes).""" + # Simulate reading a malformed JSON file (no quotes) + with patch("builtins.open", mock_open(read_data=malformed_json_credentials)): + result = gen3_metadata_parser._load_api_key() + assert result == {"api_key": "abc.def.ghi", "key_id": "18bdaa-b018"} + +def test_load_api_key_invalid_json(gen3_metadata_parser): + """Test the _load_api_key method with unrecoverable malformed JSON.""" + # Simulate reading a badly malformed JSON file + bad_content = '{key1 value1, key2:}' + with patch("builtins.open", mock_open(read_data=bad_content)): + with pytest.raises(ValueError): + gen3_metadata_parser._load_api_key() + def test_url_from_jwt(gen3_metadata_parser, fake_api_key): """Test if you can infer the data commons url from the JWT token""" url = gen3_metadata_parser._url_from_jwt(fake_api_key) From ef560eb018ebf3ab36b2a1c2669234ac9495698e Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Tue, 24 Jun 2025 13:59:43 +1000 Subject: [PATCH 07/16] added example notebook and updated readme --- README.md | 12 +++++ example_notebook.ipynb | 109 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 example_notebook.ipynb diff --git a/README.md b/README.md index ad1e3a6..422c64d 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,18 @@ pip install -e . ``` +## Alternatively you can build using: +```bash +bash build.sh +``` + +# Usage + +## 4. Run the notebook +- Notebook can be found in the `example_notebook.ipynb` file +- Make sure to select .venv as the kernel in the notebook + + ## 4. Usage Example ```python diff --git a/example_notebook.ipynb b/example_notebook.ipynb new file mode 100644 index 0000000..0be7896 --- /dev/null +++ b/example_notebook.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Example Workflow\n", + "- Make sure to run `bash build.sh` or follow the instructions in the README.md to build the package\n", + "- Make sure to select .venv as the kernel in the notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import gen3_metadata\n", + "from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Load credentiaals path from .env\n", + "import os\n", + "from dotenv import load_dotenv\n", + "from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser\n", + "load_dotenv()\n", + "key_file = os.getenv('credentials_path')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Or define credentials path in this notebook\n", + "key_file = \"path/to/credentials.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Autheticating\n", + "gen3metadata = Gen3MetadataParser(key_file)\n", + "gen3metadata.authenticate()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# fetching data\n", + "program_name= \"program1\"\n", + "project_code= \"AusDiab_Simulated\"\n", + "\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"subject\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"demographic\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"exposure\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"lab_result\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"medical_history\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"medication\")\n", + "gen3metadata.fetch_data(program_name, project_code, node_label= \"blood_pressure_test\")\n", + "\n", + "gen3metadata.data_to_pd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gen3metadata.data_store[\"program1/AusDiab_Simulated/demographic\"]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 7f1daa95fd8427492a5851b7633abfec2aad4e25 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Tue, 24 Jun 2025 14:02:14 +1000 Subject: [PATCH 08/16] updated example notebook to use dynamic data store paths and added an empty code cell --- example_notebook.ipynb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/example_notebook.ipynb b/example_notebook.ipynb index 0be7896..09cde1a 100644 --- a/example_notebook.ipynb +++ b/example_notebook.ipynb @@ -81,8 +81,16 @@ "metadata": {}, "outputs": [], "source": [ - "gen3metadata.data_store[\"program1/AusDiab_Simulated/demographic\"]" + "gen3metadata.data_store[f\"{program_name}/{project_code}/subject\"]\n", + "gen3metadata.data_store[f\"{program_name}/{project_code}/demographic\"]" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From b49295ad28360e189837e013fb83081e14dae786 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Tue, 24 Jun 2025 14:02:51 +1000 Subject: [PATCH 09/16] updated example notebook to use pandas data store for subject and demographic data --- example_notebook.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_notebook.ipynb b/example_notebook.ipynb index 09cde1a..dba64b1 100644 --- a/example_notebook.ipynb +++ b/example_notebook.ipynb @@ -81,8 +81,8 @@ "metadata": {}, "outputs": [], "source": [ - "gen3metadata.data_store[f\"{program_name}/{project_code}/subject\"]\n", - "gen3metadata.data_store[f\"{program_name}/{project_code}/demographic\"]" + "gen3metadata.data_store_pd[f\"{program_name}/{project_code}/subject\"]\n", + "gen3metadata.data_store_pd[f\"{program_name}/{project_code}/demographic\"]" ] }, { From 4490ba4c4f3451c7134b0940f30dc0e649c1fa43 Mon Sep 17 00:00:00 2001 From: CoreyGiles Date: Thu, 26 Jun 2025 11:22:21 +1000 Subject: [PATCH 10/16] Initial commit of R version of gen3-metadata --- .gitignore | 1 + gen3-metadata-R/DESCRIPTION | 20 +++ gen3-metadata-R/NAMESPACE | 17 +++ gen3-metadata-R/R/gen3_metadata.R | 40 ++++++ gen3-metadata-R/R/generics.R | 44 +++++++ gen3-metadata-R/R/get_base_url.R | 31 +++++ gen3-metadata-R/R/load_key_file.R | 27 ++++ gen3-metadata-R/R/methods-gen3_metadata.R | 146 ++++++++++++++++++++++ gen3-metadata-R/README.md | 62 +++++++++ gen3-metadata-R/gen3-metadata-R.Rproj | 16 +++ gen3-metadata-R/man/Gen3MetadataParser.Rd | 18 +++ gen3-metadata-R/man/authenticate.Rd | 26 ++++ gen3-metadata-R/man/fetch_data.Rd | 40 ++++++ gen3-metadata-R/man/get_base_url.Rd | 18 +++ gen3-metadata-R/man/load_key_file.Rd | 17 +++ gen3-metadata-R/man/print.Rd | 24 ++++ 16 files changed, 547 insertions(+) create mode 100644 gen3-metadata-R/DESCRIPTION create mode 100644 gen3-metadata-R/NAMESPACE create mode 100644 gen3-metadata-R/R/gen3_metadata.R create mode 100644 gen3-metadata-R/R/generics.R create mode 100644 gen3-metadata-R/R/get_base_url.R create mode 100644 gen3-metadata-R/R/load_key_file.R create mode 100644 gen3-metadata-R/R/methods-gen3_metadata.R create mode 100644 gen3-metadata-R/README.md create mode 100644 gen3-metadata-R/gen3-metadata-R.Rproj create mode 100644 gen3-metadata-R/man/Gen3MetadataParser.Rd create mode 100644 gen3-metadata-R/man/authenticate.Rd create mode 100644 gen3-metadata-R/man/fetch_data.Rd create mode 100644 gen3-metadata-R/man/get_base_url.Rd create mode 100644 gen3-metadata-R/man/load_key_file.Rd create mode 100644 gen3-metadata-R/man/print.Rd diff --git a/.gitignore b/.gitignore index 0a19790..64ba84b 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,4 @@ cython_debug/ # PyPI configuration file .pypirc +.Rproj.user diff --git a/gen3-metadata-R/DESCRIPTION b/gen3-metadata-R/DESCRIPTION new file mode 100644 index 0000000..4a4ada8 --- /dev/null +++ b/gen3-metadata-R/DESCRIPTION @@ -0,0 +1,20 @@ +Package: gen3metadata +Type: Package +Title: Gen3 metadata tools +Version: 0.1.0 +Author: Corey Giles [aut, cre] +Authors@R: c(person("Corey", "Giles", role = c("aut", "cre"), + email = "Corey.Giles@Baker.edu.au", + comment = c(ORCID = "0000-0002-6050-1259"))) +Maintainer: Corey Giles +Description: User friendly tools for downloading and manipulating gen3 metadata +License: GPL-3 +Encoding: UTF-8 +LazyData: true +Language: en-AU +RoxygenNote: 7.3.2 +Imports: + httr, + jsonlite, + jose, + glue diff --git a/gen3-metadata-R/NAMESPACE b/gen3-metadata-R/NAMESPACE new file mode 100644 index 0000000..cbf1a59 --- /dev/null +++ b/gen3-metadata-R/NAMESPACE @@ -0,0 +1,17 @@ +# Generated by roxygen2: do not edit by hand + +S3method(authenticate,gen3_metadata) +S3method(fetch_data,gen3_metadata) +S3method(print,gen3_metadata) +export(Gen3MetadataParser) +export(authenticate) +export(fetch_data) +export(print) +importFrom(glue,glue) +importFrom(httr,GET) +importFrom(httr,POST) +importFrom(httr,add_headers) +importFrom(httr,content) +importFrom(httr,http_error) +importFrom(jose,jwt_split) +importFrom(jsonlite,fromJSON) diff --git a/gen3-metadata-R/R/gen3_metadata.R b/gen3-metadata-R/R/gen3_metadata.R new file mode 100644 index 0000000..f46e044 --- /dev/null +++ b/gen3-metadata-R/R/gen3_metadata.R @@ -0,0 +1,40 @@ +#' Create a Gen3 metadata parser object +#' +#' This function creates a new Gen3 metadata parser object by loading +#' credentials from a key file. Use this object to interact with the Gen3 API. +#' +#' @param key_file Character string path to the JSON key file containing Gen3 credentials +#' +#' @return A gen3_metadata object with credentials and base URL configured +#' +#' @export +Gen3MetadataParser <- function(key_file) { + + # Create the object to store data + obj <- list( + key_file = key_file, + base_url = "", + credentials = list( + api_key = "", + key_id = "" + ), + header = NULL + ) + + # Load the key file + creds <- load_key_file(key_file) + + # Set the credentials in the object + obj$credentials$api_key <- creds$api_key + obj$credentials$key_id <- creds$key_id + + # Get the base URL from the API key + obj$base_url <- get_base_url(obj$credentials$api_key) + + # Set the class of the object + class(obj) <- "gen3_metadata" + + # Return the object + return(obj) + +} \ No newline at end of file diff --git a/gen3-metadata-R/R/generics.R b/gen3-metadata-R/R/generics.R new file mode 100644 index 0000000..a77322e --- /dev/null +++ b/gen3-metadata-R/R/generics.R @@ -0,0 +1,44 @@ +#' Authenticate with Gen3 API +#' +#' Generic function to authenticate a gen3_metadata object with the Gen3 API +#' and obtain an access token for subsequent requests. +#' +#' @param gen3_metadata A gen3_metadata object +#' +#' @return The authenticated gen3_metadata object (invisibly) +#' +#' @export +authenticate <- function(gen3_metadata) { + UseMethod("authenticate") +} + +#' Fetch data from Gen3 API +#' +#' Generic function to fetch data from a specific node in the Gen3 submission API +#' for a given program and project. +#' +#' @param gen3_metadata An authenticated gen3_metadata object +#' @param program_name Character string name of the program +#' @param project_code Character string code of the project +#' @param node_label Character string label of the node to fetch data from +#' @param api_version Character string API version (default: "v0") +#' +#' @return Data frame containing the fetched data +#' +#' @export +fetch_data <- function(gen3_metadata, program_name, project_code, node_label, api_version) { + UseMethod("fetch_data") +} + +#' Print gen3_metadata object +#' +#' Generic function to print a summary of a gen3_metadata object. +#' +#' @param gen3_metadata A gen3_metadata object +#' +#' @return The gen3_metadata object (invisibly) +#' +#' @export +print <- function(gen3_metadata) { + UseMethod("print") +} \ No newline at end of file diff --git a/gen3-metadata-R/R/get_base_url.R b/gen3-metadata-R/R/get_base_url.R new file mode 100644 index 0000000..2ad0b34 --- /dev/null +++ b/gen3-metadata-R/R/get_base_url.R @@ -0,0 +1,31 @@ +#' Extract base URL from Gen3 API key +#' +#' This function extracts the base URL from a Gen3 API key JWT token +#' by parsing the 'iss' (issuer) field from the payload. +#' +#' @param api_key Character string containing the Gen3 API key (JWT token) +#' +#' @return Character string containing the base URL +#' +#' @importFrom jose jwt_split +get_base_url <- function(api_key) { + + # Check if the API key is provided + if (is.null(api_key) || api_key == "") { + stop("API key must be provided.") + } + + # Extract the payload from the JWT + payload <- jose::jwt_split(api_key)$payload + + # Validate the payload + if (!"iss" %in% names(payload)) { + stop("The JWT payload must contain 'iss'.") + } + + # Extract the base URL from the payload + base_url <- sub("/user$", "", payload$iss) + + # Return the base url + return(base_url) +} \ No newline at end of file diff --git a/gen3-metadata-R/R/load_key_file.R b/gen3-metadata-R/R/load_key_file.R new file mode 100644 index 0000000..8b342fc --- /dev/null +++ b/gen3-metadata-R/R/load_key_file.R @@ -0,0 +1,27 @@ +#' Load Gen3 credentials from key file +#' +#' This function reads a JSON key file containing Gen3 API credentials. +#' +#' @param key_file Character string path to the JSON key file +#' +#' @return List containing 'api_key' and 'key_id' credentials +#' +#' @importFrom jsonlite fromJSON +load_key_file <- function(key_file) { + + # Check if the key file exists + if (!file.exists(key_file)) { + stop("Key file does not exist: ", key_file) + } + + # Read the JSON file + creds <- jsonlite::fromJSON(key_file) + + # Validate the contents of the key file + if (!"api_key" %in% names(creds) || !"key_id" %in% names(creds)) { + stop("Key file must contain 'api_key' and 'key_id'.") + } + + # Return the credentials as a list + return(creds) +} \ No newline at end of file diff --git a/gen3-metadata-R/R/methods-gen3_metadata.R b/gen3-metadata-R/R/methods-gen3_metadata.R new file mode 100644 index 0000000..79c84d7 --- /dev/null +++ b/gen3-metadata-R/R/methods-gen3_metadata.R @@ -0,0 +1,146 @@ +#' Authenticate gen3_metadata object with Gen3 API +#' +#' This method authenticates a gen3_metadata object by sending a POST request +#' to the Gen3 API with the provided credentials to obtain an access token. +#' +#' @param gen3_metadata A gen3_metadata object containing credentials +#' +#' @return The authenticated gen3_metadata object with token and headers set (invisibly) +#' +#' @method authenticate gen3_metadata +#' @rdname authenticate +#' @importFrom httr POST http_error content add_headers +#' @export +authenticate.gen3_metadata <- function(gen3_metadata) { + + # Check that the credentials are provided + if (is.null(gen3_metadata$credentials) || gen3_metadata$credentials$api_key == "") { + stop("Credentials must be provided to authenticate gen3_metadata.") + } + + # Send a POST request to the Gen3 API to get an access token + res <- httr::POST( + url = paste0(gen3_metadata$base_url, "/user/credentials/cdis/access_token"), + body = gen3_metadata$credentials, + encode = "json" + ) + + # Check for errors in the response + if (httr::http_error(res)) { + stop("Failed to authenticate with the Gen3 API. Please check your credentials.") + } + + # Get the content from the response + content <- httr::content(res, as = "parsed", type = "application/json") + + # Check if the access token is present in the response + if (is.null(content$access_token)) { + stop("Authentication failed: access token not found in the response.") + } + + # Extract the access token + gen3_metadata$token <- content$access_token + + # Set the headers for future requests + gen3_metadata$headers <- httr::add_headers( + Authorization = paste("Bearer", gen3_metadata$token) + ) + + # Return the updated gen3_metadata object + return(invisible(gen3_metadata)) +} + + +#' Fetch data from Gen3 submission API +#' +#' This method fetches data from a specific node in the Gen3 API +#' for a given program and project. The object must be authenticated first. +#' +#' @param gen3_metadata An authenticated gen3_metadata object +#' @param program_name Character string name of the program +#' @param project_code Character string code of the project +#' @param node_label Character string label of the node to fetch data from +#' @param api_version Character string API version (default: "v0") +#' +#' @return Data frame containing the fetched data from the specified node +#' +#' @method fetch_data gen3_metadata +#' @rdname fetch_data +#' @importFrom glue glue +#' @importFrom httr GET http_error content +#' @importFrom jsonlite fromJSON +#' @export +fetch_data.gen3_metadata <- function(gen3_metadata, + program_name, + project_code, + node_label, + api_version = "v0") { + + # Check that the gen3_metadata is authenticated + if (is.null(gen3_metadata$headers)) { + stop("Gen3 metadata object is not authenticated. Please authenticate first.") + } + + # Construct the URL for the API request + url <- glue::glue( + "{gen3_metadata$base_url}/api/{api_version}/submission", + "/{program_name}/{project_code}/export/" + ) + + # Make the GET request to the Gen3 API + res <- httr::GET( + url, + gen3_metadata$headers, + query = list(node_label = node_label, format = "json") + ) + + # Check for errors in the response + if (httr::http_error(res)) { + stop("Failed to fetch data from the Gen3 API. Please check your parameters and authentication.") + } + + # Extract the content from the response + content <- httr::content(res, as = "text", encoding = "UTF-8") + + # Parse the JSON content and return the data + data <- jsonlite::fromJSON(content, flatten = TRUE)$data + + # Return the fetched data + return(data) +} + + +#' Print gen3_metadata object summary +#' +#' This method prints a formatted summary of a gen3_metadata object. +#' +#' @param gen3_metadata A gen3_metadata object +#' +#' @return The gen3_metadata object (invisibly) +#' +#' @method print gen3_metadata +#' @rdname print +#' @export +print.gen3_metadata <- function(gen3_metadata) { + + # Print basic information about the gen3_metadata object + cat("Gen3 Metadata Parser\n") + cat("====================\n") + + # Display base URL + cat("Base URL:", gen3_metadata$base_url, "\n") + + # Display authentication status + if (!is.null(gen3_metadata$headers)) { + cat("Authentication: Authenticated\n") + } else { + cat("Authentication: Not authenticated\n") + } + + # Display key file path + cat("\n") + + # Return invisibly + return(invisible(gen3_metadata)) +} + diff --git a/gen3-metadata-R/README.md b/gen3-metadata-R/README.md new file mode 100644 index 0000000..16fea13 --- /dev/null +++ b/gen3-metadata-R/README.md @@ -0,0 +1,62 @@ +

+gen3metadata R tool +

+ +

+User friendly tools for downloading and manipulating gen3 metadata +

+ +
+ +R +Australian BioCommons +Baker Institute +
+ +

+Love our work? Visit our WebPortal. +

+ +
+ + +## Installation + +You can install the gen3metadata R tool from +[GitHub](https://github.com/) with: + +``` r +if (!require("devtools")) install.packages("devtools") +devtools::install_github("AustralianBioCommons/gen3-metadata", subdir = "gen3metadata-R", ref = "feature/R-package") +``` + +The package depends on several other packages, which should hopefully be installed automatically. +If case this doesn't happen, run: +``` r +install.packages(c("httr", "jsonlite", "jose", "glue")) +``` + +Then all you need to do is load the package. + +``` r +library("gen3metadata") +``` + +## Example + +This is a basic example to authenticate and load some data. +You will need a credential file (stored in `key_file_path` in this example). + +``` r +# Create the Gen3 Metadata Parser object +gen3 <- Gen3MetadataParser(key_file_path) + +# Authenticate the object +gen3 <- authenticate(gen3) + +# Load some data +dat <- fetch_data(gen3, + program_name = "program1", + project_code = "AusDiab", + node_label = "subject") +``` diff --git a/gen3-metadata-R/gen3-metadata-R.Rproj b/gen3-metadata-R/gen3-metadata-R.Rproj new file mode 100644 index 0000000..9241cc8 --- /dev/null +++ b/gen3-metadata-R/gen3-metadata-R.Rproj @@ -0,0 +1,16 @@ +Version: 1.0 + +RestoreWorkspace: Default +SaveWorkspace: Default +AlwaysSaveHistory: Default + +EnableCodeIndexing: Yes +UseSpacesForTab: Yes +NumSpacesForTab: 4 +Encoding: UTF-8 + +RnwWeave: Sweave +LaTeX: pdfLaTeX + +AutoAppendNewline: Yes +StripTrailingWhitespace: Yes diff --git a/gen3-metadata-R/man/Gen3MetadataParser.Rd b/gen3-metadata-R/man/Gen3MetadataParser.Rd new file mode 100644 index 0000000..0cd9f98 --- /dev/null +++ b/gen3-metadata-R/man/Gen3MetadataParser.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gen3_metadata.R +\name{Gen3MetadataParser} +\alias{Gen3MetadataParser} +\title{Create a Gen3 metadata parser object} +\usage{ +Gen3MetadataParser(key_file) +} +\arguments{ +\item{key_file}{Character string path to the JSON key file containing Gen3 credentials} +} +\value{ +A gen3_metadata object with credentials and base URL configured +} +\description{ +This function creates a new Gen3 metadata parser object by loading +credentials from a key file. Use this object to interact with the Gen3 API. +} diff --git a/gen3-metadata-R/man/authenticate.Rd b/gen3-metadata-R/man/authenticate.Rd new file mode 100644 index 0000000..5c01fdd --- /dev/null +++ b/gen3-metadata-R/man/authenticate.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/generics.R, R/methods-gen3_metadata.R +\name{authenticate} +\alias{authenticate} +\alias{authenticate.gen3_metadata} +\title{Authenticate with Gen3 API} +\usage{ +authenticate(gen3_metadata) + +\method{authenticate}{gen3_metadata}(gen3_metadata) +} +\arguments{ +\item{gen3_metadata}{A gen3_metadata object containing credentials} +} +\value{ +The authenticated gen3_metadata object (invisibly) + +The authenticated gen3_metadata object with token and headers set (invisibly) +} +\description{ +Generic function to authenticate a gen3_metadata object with the Gen3 API +and obtain an access token for subsequent requests. + +This method authenticates a gen3_metadata object by sending a POST request +to the Gen3 API with the provided credentials to obtain an access token. +} diff --git a/gen3-metadata-R/man/fetch_data.Rd b/gen3-metadata-R/man/fetch_data.Rd new file mode 100644 index 0000000..d4e37a6 --- /dev/null +++ b/gen3-metadata-R/man/fetch_data.Rd @@ -0,0 +1,40 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/generics.R, R/methods-gen3_metadata.R +\name{fetch_data} +\alias{fetch_data} +\alias{fetch_data.gen3_metadata} +\title{Fetch data from Gen3 API} +\usage{ +fetch_data(gen3_metadata, program_name, project_code, node_label, api_version) + +\method{fetch_data}{gen3_metadata}( + gen3_metadata, + program_name, + project_code, + node_label, + api_version = "v0" +) +} +\arguments{ +\item{gen3_metadata}{An authenticated gen3_metadata object} + +\item{program_name}{Character string name of the program} + +\item{project_code}{Character string code of the project} + +\item{node_label}{Character string label of the node to fetch data from} + +\item{api_version}{Character string API version (default: "v0")} +} +\value{ +Data frame containing the fetched data + +Data frame containing the fetched data from the specified node +} +\description{ +Generic function to fetch data from a specific node in the Gen3 submission API +for a given program and project. + +This method fetches data from a specific node in the Gen3 API +for a given program and project. The object must be authenticated first. +} diff --git a/gen3-metadata-R/man/get_base_url.Rd b/gen3-metadata-R/man/get_base_url.Rd new file mode 100644 index 0000000..e8a1550 --- /dev/null +++ b/gen3-metadata-R/man/get_base_url.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_base_url.R +\name{get_base_url} +\alias{get_base_url} +\title{Extract base URL from Gen3 API key} +\usage{ +get_base_url(api_key) +} +\arguments{ +\item{api_key}{Character string containing the Gen3 API key (JWT token)} +} +\value{ +Character string containing the base URL +} +\description{ +This function extracts the base URL from a Gen3 API key JWT token +by parsing the 'iss' (issuer) field from the payload. +} diff --git a/gen3-metadata-R/man/load_key_file.Rd b/gen3-metadata-R/man/load_key_file.Rd new file mode 100644 index 0000000..e6db863 --- /dev/null +++ b/gen3-metadata-R/man/load_key_file.Rd @@ -0,0 +1,17 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/load_key_file.R +\name{load_key_file} +\alias{load_key_file} +\title{Load Gen3 credentials from key file} +\usage{ +load_key_file(key_file) +} +\arguments{ +\item{key_file}{Character string path to the JSON key file} +} +\value{ +List containing 'api_key' and 'key_id' credentials +} +\description{ +This function reads a JSON key file containing Gen3 API credentials. +} diff --git a/gen3-metadata-R/man/print.Rd b/gen3-metadata-R/man/print.Rd new file mode 100644 index 0000000..59f89c7 --- /dev/null +++ b/gen3-metadata-R/man/print.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/generics.R, R/methods-gen3_metadata.R +\name{print} +\alias{print} +\alias{print.gen3_metadata} +\title{Print gen3_metadata object} +\usage{ +print(gen3_metadata) + +\method{print}{gen3_metadata}(gen3_metadata) +} +\arguments{ +\item{gen3_metadata}{A gen3_metadata object} +} +\value{ +The gen3_metadata object (invisibly) + +The gen3_metadata object (invisibly) +} +\description{ +Generic function to print a summary of a gen3_metadata object. + +This method prints a formatted summary of a gen3_metadata object. +} From 4a8d726fa2350518260166ddf49dc38937ce8dc6 Mon Sep 17 00:00:00 2001 From: CoreyGiles Date: Thu, 26 Jun 2025 11:26:38 +1000 Subject: [PATCH 11/16] Change directory name. Minor tweak to readme. Removed print generic --- {gen3-metadata-R => gen3metadata-R}/DESCRIPTION | 0 {gen3-metadata-R => gen3metadata-R}/NAMESPACE | 0 .../R/gen3_metadata.R | 0 {gen3-metadata-R => gen3metadata-R}/R/generics.R | 13 ------------- .../R/get_base_url.R | 0 .../R/load_key_file.R | 0 .../R/methods-gen3_metadata.R | 0 {gen3-metadata-R => gen3metadata-R}/README.md | 2 +- .../gen3-metadata-R.Rproj | 0 .../man/Gen3MetadataParser.Rd | 0 .../man/authenticate.Rd | 0 .../man/fetch_data.Rd | 0 .../man/get_base_url.Rd | 0 .../man/load_key_file.Rd | 0 {gen3-metadata-R => gen3metadata-R}/man/print.Rd | 0 15 files changed, 1 insertion(+), 14 deletions(-) rename {gen3-metadata-R => gen3metadata-R}/DESCRIPTION (100%) rename {gen3-metadata-R => gen3metadata-R}/NAMESPACE (100%) rename {gen3-metadata-R => gen3metadata-R}/R/gen3_metadata.R (100%) rename {gen3-metadata-R => gen3metadata-R}/R/generics.R (79%) rename {gen3-metadata-R => gen3metadata-R}/R/get_base_url.R (100%) rename {gen3-metadata-R => gen3metadata-R}/R/load_key_file.R (100%) rename {gen3-metadata-R => gen3metadata-R}/R/methods-gen3_metadata.R (100%) rename {gen3-metadata-R => gen3metadata-R}/README.md (93%) rename {gen3-metadata-R => gen3metadata-R}/gen3-metadata-R.Rproj (100%) rename {gen3-metadata-R => gen3metadata-R}/man/Gen3MetadataParser.Rd (100%) rename {gen3-metadata-R => gen3metadata-R}/man/authenticate.Rd (100%) rename {gen3-metadata-R => gen3metadata-R}/man/fetch_data.Rd (100%) rename {gen3-metadata-R => gen3metadata-R}/man/get_base_url.Rd (100%) rename {gen3-metadata-R => gen3metadata-R}/man/load_key_file.Rd (100%) rename {gen3-metadata-R => gen3metadata-R}/man/print.Rd (100%) diff --git a/gen3-metadata-R/DESCRIPTION b/gen3metadata-R/DESCRIPTION similarity index 100% rename from gen3-metadata-R/DESCRIPTION rename to gen3metadata-R/DESCRIPTION diff --git a/gen3-metadata-R/NAMESPACE b/gen3metadata-R/NAMESPACE similarity index 100% rename from gen3-metadata-R/NAMESPACE rename to gen3metadata-R/NAMESPACE diff --git a/gen3-metadata-R/R/gen3_metadata.R b/gen3metadata-R/R/gen3_metadata.R similarity index 100% rename from gen3-metadata-R/R/gen3_metadata.R rename to gen3metadata-R/R/gen3_metadata.R diff --git a/gen3-metadata-R/R/generics.R b/gen3metadata-R/R/generics.R similarity index 79% rename from gen3-metadata-R/R/generics.R rename to gen3metadata-R/R/generics.R index a77322e..8502803 100644 --- a/gen3-metadata-R/R/generics.R +++ b/gen3metadata-R/R/generics.R @@ -29,16 +29,3 @@ authenticate <- function(gen3_metadata) { fetch_data <- function(gen3_metadata, program_name, project_code, node_label, api_version) { UseMethod("fetch_data") } - -#' Print gen3_metadata object -#' -#' Generic function to print a summary of a gen3_metadata object. -#' -#' @param gen3_metadata A gen3_metadata object -#' -#' @return The gen3_metadata object (invisibly) -#' -#' @export -print <- function(gen3_metadata) { - UseMethod("print") -} \ No newline at end of file diff --git a/gen3-metadata-R/R/get_base_url.R b/gen3metadata-R/R/get_base_url.R similarity index 100% rename from gen3-metadata-R/R/get_base_url.R rename to gen3metadata-R/R/get_base_url.R diff --git a/gen3-metadata-R/R/load_key_file.R b/gen3metadata-R/R/load_key_file.R similarity index 100% rename from gen3-metadata-R/R/load_key_file.R rename to gen3metadata-R/R/load_key_file.R diff --git a/gen3-metadata-R/R/methods-gen3_metadata.R b/gen3metadata-R/R/methods-gen3_metadata.R similarity index 100% rename from gen3-metadata-R/R/methods-gen3_metadata.R rename to gen3metadata-R/R/methods-gen3_metadata.R diff --git a/gen3-metadata-R/README.md b/gen3metadata-R/README.md similarity index 93% rename from gen3-metadata-R/README.md rename to gen3metadata-R/README.md index 16fea13..6d7590b 100644 --- a/gen3-metadata-R/README.md +++ b/gen3metadata-R/README.md @@ -27,7 +27,7 @@ You can install the gen3metadata R tool from ``` r if (!require("devtools")) install.packages("devtools") -devtools::install_github("AustralianBioCommons/gen3-metadata", subdir = "gen3metadata-R", ref = "feature/R-package") +devtools::install_github("CoreyGiles/gen3-metadata", subdir = "gen3metadata-R", ref = "feature/R-package") ``` The package depends on several other packages, which should hopefully be installed automatically. diff --git a/gen3-metadata-R/gen3-metadata-R.Rproj b/gen3metadata-R/gen3-metadata-R.Rproj similarity index 100% rename from gen3-metadata-R/gen3-metadata-R.Rproj rename to gen3metadata-R/gen3-metadata-R.Rproj diff --git a/gen3-metadata-R/man/Gen3MetadataParser.Rd b/gen3metadata-R/man/Gen3MetadataParser.Rd similarity index 100% rename from gen3-metadata-R/man/Gen3MetadataParser.Rd rename to gen3metadata-R/man/Gen3MetadataParser.Rd diff --git a/gen3-metadata-R/man/authenticate.Rd b/gen3metadata-R/man/authenticate.Rd similarity index 100% rename from gen3-metadata-R/man/authenticate.Rd rename to gen3metadata-R/man/authenticate.Rd diff --git a/gen3-metadata-R/man/fetch_data.Rd b/gen3metadata-R/man/fetch_data.Rd similarity index 100% rename from gen3-metadata-R/man/fetch_data.Rd rename to gen3metadata-R/man/fetch_data.Rd diff --git a/gen3-metadata-R/man/get_base_url.Rd b/gen3metadata-R/man/get_base_url.Rd similarity index 100% rename from gen3-metadata-R/man/get_base_url.Rd rename to gen3metadata-R/man/get_base_url.Rd diff --git a/gen3-metadata-R/man/load_key_file.Rd b/gen3metadata-R/man/load_key_file.Rd similarity index 100% rename from gen3-metadata-R/man/load_key_file.Rd rename to gen3metadata-R/man/load_key_file.Rd diff --git a/gen3-metadata-R/man/print.Rd b/gen3metadata-R/man/print.Rd similarity index 100% rename from gen3-metadata-R/man/print.Rd rename to gen3metadata-R/man/print.Rd From 2659fc79177d25c10622e43d6cfc23caaba27715 Mon Sep 17 00:00:00 2001 From: CoreyGiles Date: Fri, 27 Jun 2025 09:43:38 +1000 Subject: [PATCH 12/16] Preparing for pull request --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++ gen3metadata-R/README.md | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 422c64d..473f378 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,47 @@ pytest tests/ + +## Installation of the R version of gen3-metadata + +You can install the gen3metadata R tool from +[GitHub](https://github.com/) with: + +``` r +if (!require("devtools")) install.packages("devtools") +devtools::install_github("AustralianBioCommons/gen3-metadata", subdir = "gen3metadata-R") +``` + +The package depends on several other packages, which should hopefully be installed automatically. +If case this doesn't happen, run: +``` r +install.packages(c("httr", "jsonlite", "jose", "glue")) +``` + +Then all you need to do is load the package. + +``` r +library("gen3metadata") +``` + +## Usage Example + +This is a basic example to authenticate and load some data. +You will need a credential file (stored in `key_file_path` in this example). + +``` r +# Load the library +library("gen3metadata") + +# Create the Gen3 Metadata Parser object +gen3 <- Gen3MetadataParser(key_file_path) + +# Authenticate the object +gen3 <- authenticate(gen3) + +# Load some data +dat <- fetch_data(gen3, + program_name = "program1", + project_code = "AusDiab", + node_label = "subject") +``` diff --git a/gen3metadata-R/README.md b/gen3metadata-R/README.md index 6d7590b..a1063c2 100644 --- a/gen3metadata-R/README.md +++ b/gen3metadata-R/README.md @@ -27,7 +27,7 @@ You can install the gen3metadata R tool from ``` r if (!require("devtools")) install.packages("devtools") -devtools::install_github("CoreyGiles/gen3-metadata", subdir = "gen3metadata-R", ref = "feature/R-package") +devtools::install_github("AustralianBioCommons/gen3-metadata", subdir = "gen3metadata-R") ``` The package depends on several other packages, which should hopefully be installed automatically. From 6703a682cca736aac61015fd6e042b63c5066d94 Mon Sep 17 00:00:00 2001 From: CoreyGiles Date: Fri, 27 Jun 2025 17:21:42 +1000 Subject: [PATCH 13/16] Unit tests for gen3metadata --- gen3metadata-R/DESCRIPTION | 4 + gen3metadata-R/NAMESPACE | 1 - gen3metadata-R/man/print.Rd | 13 +- gen3metadata-R/tests/testthat.R | 12 ++ .../tests/testthat/test-authenticate.R | 122 ++++++++++++++++++ .../tests/testthat/test-fetch_data.R | 115 +++++++++++++++++ .../tests/testthat/test-gen3_metadata.R | 56 ++++++++ .../tests/testthat/test-get_base_url.R | 43 ++++++ .../tests/testthat/test-load_key_file.R | 69 ++++++++++ 9 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 gen3metadata-R/tests/testthat.R create mode 100644 gen3metadata-R/tests/testthat/test-authenticate.R create mode 100644 gen3metadata-R/tests/testthat/test-fetch_data.R create mode 100644 gen3metadata-R/tests/testthat/test-gen3_metadata.R create mode 100644 gen3metadata-R/tests/testthat/test-get_base_url.R create mode 100644 gen3metadata-R/tests/testthat/test-load_key_file.R diff --git a/gen3metadata-R/DESCRIPTION b/gen3metadata-R/DESCRIPTION index 4a4ada8..22f3554 100644 --- a/gen3metadata-R/DESCRIPTION +++ b/gen3metadata-R/DESCRIPTION @@ -18,3 +18,7 @@ Imports: jsonlite, jose, glue +Suggests: + testthat (>= 3.0.0), + webmockr +Config/testthat/edition: 3 diff --git a/gen3metadata-R/NAMESPACE b/gen3metadata-R/NAMESPACE index cbf1a59..555b84b 100644 --- a/gen3metadata-R/NAMESPACE +++ b/gen3metadata-R/NAMESPACE @@ -6,7 +6,6 @@ S3method(print,gen3_metadata) export(Gen3MetadataParser) export(authenticate) export(fetch_data) -export(print) importFrom(glue,glue) importFrom(httr,GET) importFrom(httr,POST) diff --git a/gen3metadata-R/man/print.Rd b/gen3metadata-R/man/print.Rd index 59f89c7..bb057c1 100644 --- a/gen3metadata-R/man/print.Rd +++ b/gen3metadata-R/man/print.Rd @@ -1,24 +1,17 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/generics.R, R/methods-gen3_metadata.R -\name{print} -\alias{print} +% Please edit documentation in R/methods-gen3_metadata.R +\name{print.gen3_metadata} \alias{print.gen3_metadata} -\title{Print gen3_metadata object} +\title{Print gen3_metadata object summary} \usage{ -print(gen3_metadata) - \method{print}{gen3_metadata}(gen3_metadata) } \arguments{ \item{gen3_metadata}{A gen3_metadata object} } \value{ -The gen3_metadata object (invisibly) - The gen3_metadata object (invisibly) } \description{ -Generic function to print a summary of a gen3_metadata object. - This method prints a formatted summary of a gen3_metadata object. } diff --git a/gen3metadata-R/tests/testthat.R b/gen3metadata-R/tests/testthat.R new file mode 100644 index 0000000..7613d39 --- /dev/null +++ b/gen3metadata-R/tests/testthat.R @@ -0,0 +1,12 @@ +# This file is part of the standard setup for testthat. +# It is recommended that you do not modify it. +# +# Where should you do additional test configuration? +# Learn more about the roles of various files in: +# * https://r-pkgs.org/testing-design.html#sec-tests-files-overview +# * https://testthat.r-lib.org/articles/special-files.html + +library(testthat) +library(gen3metadata) + +test_check("gen3metadata") diff --git a/gen3metadata-R/tests/testthat/test-authenticate.R b/gen3metadata-R/tests/testthat/test-authenticate.R new file mode 100644 index 0000000..eaec34d --- /dev/null +++ b/gen3metadata-R/tests/testthat/test-authenticate.R @@ -0,0 +1,122 @@ +#' Unit tests for authenticate + +# Load required packages +library(testthat) +library(webmockr) +library(gen3metadata) + +# Enable webmockr to intercept HTTP requests +webmockr::enable() + +## Fixture to provide a fake API key. Note: these credentials have been inactivated. +## This is a valid JWT and UUID, but is not active. +fake_api_key <- paste0( + "{\"api_key\":\"eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eX", + "AiOiJKV1QifQ.eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczov", + "L2RhdGEudGVzdC5iaW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZ", + "GF0YS50ZXN0LmJpb2NvbW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLC", + "JleHAiOjE3NDQ4NDU0ODAsImp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2J", + "lNGVkMTIyOSIsImF6cCI6IiIsInNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9w", + "ZW5pZCIsImdvb2dsZV9zZXJ2aWNlX2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nb", + "GVfbGluayIsImFkbWluIiwidXNlciIsImdhNGdoX3Bhc3Nwb3J0X3YxIl19.SGPjs6ljC", + "JbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cnjduhWR", + "KnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0vQJ", + "zrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqp", + "MU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXL", + "U3vSdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA\",\"key_id\"", + ":\"b9042701-b08f-40da-89a3-753be4ed1229\"}" +) + + +test_that("authenticate method works correctly", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + gen3 <- Gen3MetadataParser(tmp_key_file) + + # Expect that headers and token are NULL initially + expect_null(gen3$headers) + expect_null(gen3$token) + + # Mock the POST request to the Gen3 API + mock_post <- stub_request("post", uri = "https://data.test.biocommons.org.au/user/credentials/cdis/access_token") + mock_post <- to_return(mock_post, + body = "{\"access_token\": \"fake_access_token\"}", + status = 200, + headers = list("Content-Type" = "application/json")) + + # Authenticate the Gen3 metadata object + gen3 <- authenticate(gen3) + + # Check that the token is set correctly + expect_equal(gen3$token, "fake_access_token") + + # Check that the headers are set correctly + expect_true(!is.null(gen3$headers)) + expect_equal(gen3$headers$headers, c("Authorization" = "Bearer fake_access_token")) + + # Clean up temporary file + unlink(tmp_key_file) + + # Clear the stub registry to remove the mock + webmockr::stub_registry_clear() +}) + + +test_that("authenticate method handles HTTP errors", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + gen3 <- Gen3MetadataParser(tmp_key_file) + + # Mock the POST request to return an error + mock_post <- stub_request("post", uri = "https://data.test.biocommons.org.au/user/credentials/cdis/access_token") + mock_post <- to_return(mock_post, + body = "{\"error\": \"invalid_grant\"}", + status = 400, + headers = list("Content-Type" = "application/json")) + + # Expect an error when trying to authenticate + expect_error(authenticate(gen3), "Failed to authenticate with the Gen3 API. Please check your credentials.") + + # Clean up temporary file + unlink(tmp_key_file) + + # Clear the stub registry to remove the mock + webmockr::stub_registry_clear() +}) + + +test_that("authenticate method handles missing access token", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + gen3 <- Gen3MetadataParser(tmp_key_file) + + # Mock the POST request to return an error + mock_post <- stub_request("post", uri = "https://data.test.biocommons.org.au/user/credentials/cdis/access_token") + mock_post <- to_return(mock_post, + body = "{}", + status = 200, + headers = list("Content-Type" = "application/json")) + + # Expect an error when trying to authenticate + expect_error(authenticate(gen3), "access token not found in the response.") + + # Clean up temporary file + unlink(tmp_key_file) + + # Clear the stub registry to remove the mock + webmockr::stub_registry_clear() + +}) + diff --git a/gen3metadata-R/tests/testthat/test-fetch_data.R b/gen3metadata-R/tests/testthat/test-fetch_data.R new file mode 100644 index 0000000..cfb505f --- /dev/null +++ b/gen3metadata-R/tests/testthat/test-fetch_data.R @@ -0,0 +1,115 @@ +#' Unit tests for fetch_data + +# Load required packages +library(testthat) +library(webmockr) +library(gen3metadata) + +# Enable webmockr to intercept HTTP requests +webmockr::enable() + +## Fixture to provide a fake API key. Note: these credentials have been inactivated. +## This is a valid JWT and UUID, but is not active. +fake_api_key <- paste0( + "{\"api_key\":\"eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eX", + "AiOiJKV1QifQ.eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczov", + "L2RhdGEudGVzdC5iaW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZ", + "GF0YS50ZXN0LmJpb2NvbW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLC", + "JleHAiOjE3NDQ4NDU0ODAsImp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2J", + "lNGVkMTIyOSIsImF6cCI6IiIsInNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9w", + "ZW5pZCIsImdvb2dsZV9zZXJ2aWNlX2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nb", + "GVfbGluayIsImFkbWluIiwidXNlciIsImdhNGdoX3Bhc3Nwb3J0X3YxIl19.SGPjs6ljC", + "JbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cnjduhWR", + "KnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0vQJ", + "zrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqp", + "MU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXL", + "U3vSdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA\",\"key_id\"", + ":\"b9042701-b08f-40da-89a3-753be4ed1229\"}" +) + + +test_that("fetch_data method works correctly", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + gen3 <- Gen3MetadataParser(tmp_key_file) + + # Mock the POST request to the Gen3 API + mock_post <- stub_request("post", uri = "https://data.test.biocommons.org.au/user/credentials/cdis/access_token") + mock_post <- to_return(mock_post, + body = "{\"access_token\": \"fake_access_token\"}", + status = 200, + headers = list("Content-Type" = "application/json")) + + # Authenticate the Gen3 metadata object + gen3 <- authenticate(gen3) + + # Mock the GET request to the Gen3 API + mock_get <- stub_request("get", uri = "https://data.test.biocommons.org.au/api/v0/submission/program1/AusDiab/export/?node_label=subject&format=json") + mock_get <- to_return(mock_get, + body = "{\"data\": [{\"id\": 1, \"name\": \"test\"}]}", + status = 200, + headers = list("Content-Type" = "application/json")) + + # Call fetch_data and check the result + result <- fetch_data(gen3, "program1", "AusDiab", "subject") + + # Check that the result is a data frame + expect_s3_class(result, "data.frame") + + # Check that the data frame has the expected columns + expect_true("id" %in% colnames(result)) + expect_true("name" %in% colnames(result)) + + # Check that the data frame contains the expected data + expect_equal(nrow(result), 1) + expect_equal(result$id, 1) + expect_equal(result$name, "test") + + # Clean up temporary file + unlink(tmp_key_file) + + # Clear the stub registry to remove the mock + webmockr::stub_registry_clear() +}) + + +test_that("fetch_data handles HTTP errors", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + gen3 <- Gen3MetadataParser(tmp_key_file) + + # Mock the POST request to the Gen3 API + mock_post <- stub_request("post", uri = "https://data.test.biocommons.org.au/user/credentials/cdis/access_token") + mock_post <- to_return(mock_post, + body = "{\"access_token\": \"fake_access_token\"}", + status = 200, + headers = list("Content-Type" = "application/json")) + + # Authenticate the Gen3 metadata object + gen3 <- authenticate(gen3) + + # Mock the GET request to the Gen3 API + mock_get <- stub_request("get", uri = "https://data.test.biocommons.org.au/api/v0/submission/program1/AusDiab/export/?node_label=subject&format=json") + mock_get <- to_return(mock_get, + body = "{\"data\": [{\"id\": 1, \"name\": \"test\"}]}", + status = 400, + headers = list("Content-Type" = "application/json")) + + # Call fetch_data and check the result + expect_error(fetch_data(gen3, "program1", "AusDiab", "subject"), "Failed to fetch data from the Gen3 API") + + # Clean up temporary file + unlink(tmp_key_file) + + # Clear the stub registry to remove the mock + webmockr::stub_registry_clear() +}) + diff --git a/gen3metadata-R/tests/testthat/test-gen3_metadata.R b/gen3metadata-R/tests/testthat/test-gen3_metadata.R new file mode 100644 index 0000000..3ab35f5 --- /dev/null +++ b/gen3metadata-R/tests/testthat/test-gen3_metadata.R @@ -0,0 +1,56 @@ +#' Unit tests for gen3metadata + +# Load required packages +library(testthat) +library(gen3metadata) + +## Fixture to provide a fake API key. Note: these credentials have been inactivated. +## This is a valid JWT and UUID, but is not active. +fake_api_key <- paste0( + "{\"api_key\":\"eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eX", + "AiOiJKV1QifQ.eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczov", + "L2RhdGEudGVzdC5iaW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZ", + "GF0YS50ZXN0LmJpb2NvbW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLC", + "JleHAiOjE3NDQ4NDU0ODAsImp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2J", + "lNGVkMTIyOSIsImF6cCI6IiIsInNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9w", + "ZW5pZCIsImdvb2dsZV9zZXJ2aWNlX2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nb", + "GVfbGluayIsImFkbWluIiwidXNlciIsImdhNGdoX3Bhc3Nwb3J0X3YxIl19.SGPjs6ljC", + "JbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cnjduhWR", + "KnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0vQJ", + "zrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqp", + "MU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXL", + "U3vSdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA\",\"key_id\"", + ":\"b9042701-b08f-40da-89a3-753be4ed1229\"}" +) + + +test_that("Gen3MetadataParser creates an object with correct class", { + + # Create a temporary key file + tmp_key_file <- tempfile(fileext = ".json") + writeLines(fake_api_key, tmp_key_file) + + # Create the Gen3 metadata parser object + obj <- Gen3MetadataParser(tmp_key_file) + + # Check that the object is of class 'gen3_metadata' + expect_s3_class(obj, "gen3_metadata") + + # Check that the key file path is set correctly + expect_equal(obj$key_file, tmp_key_file) + + # Check that the base URL is set correctly + expect_equal(obj$base_url, "https://data.test.biocommons.org.au") + + # Read the credentials from the key file + creds <- load_key_file(tmp_key_file) + + # Check that the credentials are set correctly + expect_equal(obj$credentials$api_key, creds$api_key) + expect_equal(obj$credentials$key_id, creds$key_id) + + # Clean up temporary file + unlink(tmp_key_file) +}) + + diff --git a/gen3metadata-R/tests/testthat/test-get_base_url.R b/gen3metadata-R/tests/testthat/test-get_base_url.R new file mode 100644 index 0000000..c637a89 --- /dev/null +++ b/gen3metadata-R/tests/testthat/test-get_base_url.R @@ -0,0 +1,43 @@ +#' Unit tests for get_base_url + +# Load required packages +library(testthat) +library(gen3metadata) + +## Fixture to provide a fake API key. Note: these credentials have been inactivated. +## This is a valid JWT and UUID, but is not active. +fake_api_key <- list( + "api_key" = paste0( + "eyJhbGciOiJSUzI1NiIsImtpZCI6ImZlbmNlX2tleV9rZXkiLCJ0eXAiOiJKV1QifQ.", + "eyJwdXIiOiJhcGlfa2V5Iiwic3ViIjoiMjEiLCJpc3MiOiJodHRwczovL2RhdGEudGVzdC5i", + "aW9jb21tb25zLm9yZy5hdS91c2VyIiwiYXVkIjpbImh0dHBzOi8vZGF0YS50ZXN0LmJpb2Nv", + "bW1vbnMub3JnLmF1L3VzZXIiXSwiaWF0IjoxNzQyMjUzNDgwLCJleHAiOjE3NDQ4NDU0ODAs", + "Imp0aSI6ImI5MDQyNzAxLWIwOGYtNDBkYS04OWEzLTc1M2JlNGVkMTIyOSIsImF6cCI6IiIs", + "InNjb3BlIjpbImdvb2dsZV9jcmVkZW50aWFscyIsIm9wZW5pZCIsImdvb2dsZV9zZXJ2aWNl", + "X2FjY291bnQiLCJkYXRhIiwiZmVuY2UiLCJnb29nbGVfbGluayIsImFkbWluIiwidXNlciIs", + "ImdhNGdoX3Bhc3Nwb3J0X3YxIl19.", + "SGPjs6ljCJbwDu-6WAnI5dN8o5467_ktcnsxRFrX_aCQNrOwSPgTCDvWEzamRmB5Oa0yB6cn", + "jduhWRKnPWIZDal86H0etm77wilCteHF_zFl1IV6LW23AfOVOG3zB9KL6o-ZYqpSRyo0FDj0", + "vQJzrHXPjqvQ15S6Js2sIwIa3ONTeHbR6fRecfPaLK1uGIY5tJFeigXzrLzlifKCEnt_2gqp", + "MU2_b2QgW1315FixNIUOl8A7FZJ2-ddSMJPO0IYQ0QMSWV9-bbxie4Zjsaa1HtQYOhfXLU3v", + "SdUOBO0btSfd6-NnWfx_-lDo5V9lkSH_aecEyew0IHBx-e7rSR5cxA"), + "key_id" = "b9042701-b08f-40da-89a3-753be4ed1229" +) + + +test_that("get_base_url can get the data commons url from JWT token", { + + # Call get_base_url with the fake API key + base_url <- gen3metadata:::get_base_url(fake_api_key$api_key) + + # Check that the base URL is as expected + expect_equal(base_url, "https://data.test.biocommons.org.au") +}) + + +test_that("get_base_url errors when passing empty string", { + + # Test that get_base_url raises an error when the API key is an empty string + expect_error(gen3metadata:::get_base_url(""), "API key must be provided.") +}) + diff --git a/gen3metadata-R/tests/testthat/test-load_key_file.R b/gen3metadata-R/tests/testthat/test-load_key_file.R new file mode 100644 index 0000000..5440432 --- /dev/null +++ b/gen3metadata-R/tests/testthat/test-load_key_file.R @@ -0,0 +1,69 @@ +#' Unit tests for load_key_file + +# Load required packages +library(testthat) +library(gen3metadata) + + +test_that("load_key_file reads valid JSON file", { + + # Create a temporary file + tmp <- tempfile(fileext = ".json") + + # Write valid JSON to the temporary file + writeLines('{"api_key":"abc.def.ghi","key_id":"18b018"}', tmp) + + # Call load_key_file and check the result + creds <- load_key_file(tmp) + + # Check that the credentials are as expected + expect_equal(creds, + list(api_key = "abc.def.ghi", + key_id = "18b018")) + + # Clean up temporary file + unlink(tmp) +}) + + +test_that("load_key_file errors when file missing or no required fields", { + + # Test that load_key_file raises an error when the file does not exist + expect_error(load_key_file("no/such/file.json"), "does not exist") + + # Create a temporary file + tmp <- tempfile(fileext = ".json") + + # Write JSON without required fields + writeLines('{"api_key":"abc.def.ghi"}', tmp) + + # Test that load_key_file raises an error when the file is missing required fields + expect_error(load_key_file(tmp), "must contain 'api_key' and 'key_id'") + + # Clean up temporary file + unlink(tmp) +}) + + +test_that("load_key_file handles non-existent file", { + + # Test that load_key_file raises an error when the file does not exist + expect_error(load_key_file("non_existent_file.json"), "does not exist") +}) + + +test_that("load_key_file handles malformed JSON", { + + # Create a temporary file + tmp <- tempfile(fileext = ".json") + + # Write malformed JSON to the temporary file + writeLines('{"api_key":"abc.def.ghi", "key_id":18b018}', tmp) + + # Test that load_key_file raises an error for malformed JSON + expect_error(gen3metadata:::load_key_file(tmp), "invalid char in json") + + # Clean up temporary file + unlink(tmp) +}) + From ebb214f7852ab9e422e08fc4901345a4ecb12c07 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 11 Jul 2025 15:00:52 +1000 Subject: [PATCH 14/16] added fetch data pd and fetch data json methods for more granularity --- src/gen3_metadata/gen3_metadata_parser.py | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/gen3_metadata/gen3_metadata_parser.py b/src/gen3_metadata/gen3_metadata_parser.py index f9b42b0..8c0ddcc 100644 --- a/src/gen3_metadata/gen3_metadata_parser.py +++ b/src/gen3_metadata/gen3_metadata_parser.py @@ -182,3 +182,31 @@ def data_to_pd(self) -> None: print(f"Converting {key} to pandas dataframe...") self.data_store_pd[key] = self.json_to_pd(value['data']) return + + def fetch_data_pd(self, program_name, project_code, node_label, api_version="v0"): + """ + Fetches data from the Gen3 API for a specific program, project, and node label, + and converts it to a pandas DataFrame. + + Args: + program_name (str): The name of the program. + project_code (str): The code of the project. + node_label (str): The label of the node. + api_version (str, optional): The version of the API to use. + Defaults to "v0". + """ + data = self.fetch_data(program_name, project_code, node_label, api_version=api_version, return_data=True) + return self.json_to_pd(data['data']) + + def fetch_data_json(self, program_name, project_code, node_label, api_version="v0"): + """ + Fetches data from the Gen3 API for a specific program, project, and node label. + + Args: + program_name (str): The name of the program. + project_code (str): The code of the project. + node_label (str): The label of the node. + api_version (str, optional): The version of the API to use. + Defaults to "v0". + """ + return self.fetch_data(program_name, project_code, node_label, api_version=api_version, return_data=True) From 41961572bd2a23a35d29330c77b6469a64189ce3 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 11 Jul 2025 15:11:24 +1000 Subject: [PATCH 15/16] Add unit tests for fetch_data_pd and fetch_data_json methods --- tests/test_gen3_metadata_parser.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_gen3_metadata_parser.py b/tests/test_gen3_metadata_parser.py index 400a6a4..2220fbd 100644 --- a/tests/test_gen3_metadata_parser.py +++ b/tests/test_gen3_metadata_parser.py @@ -212,3 +212,40 @@ def test_data_to_pd(gen3_metadata_parser, data_store): # Verify conversion assert test_key in gen3_metadata_parser.data_store_pd pd.testing.assert_frame_equal(gen3_metadata_parser.data_store_pd[test_key], expected_df) + + +@patch("requests.get") +def test_fetch_data_pd(mock_get, gen3_metadata_parser, fake_api_key): + """Test fetch_data for successful API response.""" + fake_response = {"data": [{"id": 1, "name": "test"}]} + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = fake_response + + program_name = "test_program" + project_code = "test_project" + node_label = "subjects" + + with patch("builtins.open", mock_open(read_data=json.dumps(fake_api_key))): + result = gen3_metadata_parser.fetch_data_pd(program_name, project_code, node_label) + key = f"{program_name}/{project_code}/{node_label}" + assert key in gen3_metadata_parser.data_store + assert isinstance(result, pd.DataFrame) + assert result.equals(pd.DataFrame(fake_response['data'])) + + +@patch("requests.get") +def test_fetch_data_json(mock_get, gen3_metadata_parser, fake_api_key): + """Test fetch_data_json for successful API response.""" + fake_response = {"data": [{"id": 1, "name": "test"}]} + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = fake_response + + program_name = "test_program" + project_code = "test_project" + node_label = "subjects" + + with patch("builtins.open", mock_open(read_data=json.dumps(fake_api_key))): + result = gen3_metadata_parser.fetch_data_json(program_name, project_code, node_label) + key = f"{program_name}/{project_code}/{node_label}" + assert key in gen3_metadata_parser.data_store + assert result == fake_response From ce42c62597c4e2c94a0bd7554d6ca25821a09d35 Mon Sep 17 00:00:00 2001 From: JoshuaHarris391 Date: Fri, 11 Jul 2025 15:23:02 +1000 Subject: [PATCH 16/16] Refactor example notebook and README for improved clarity and functionality. Removed dotenv usage, streamlined authentication process, and updated data fetching methods to return data as DataFrame and JSON. Enhanced README with clearer usage examples. --- README.md | 87 +++++++++++------------------------------- example_notebook.ipynb | 47 ++++------------------- 2 files changed, 31 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 473f378..2fce31f 100644 --- a/README.md +++ b/README.md @@ -1,93 +1,50 @@ # gen3-metadata User friendly tools for downloading and manipulating gen3 metadata - -## 1. Set up python venv -```bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -``` - -## 2. Create config file -```bash -echo credentials_path=\"/path/to/credentials.json\" > .env -``` - -## 3. Load library -```bash -pip install -e . -``` - - -## Alternatively you can build using: +## Python Installation ```bash +git clone https://github.com/AustralianBioCommons/gen3-metadata.git bash build.sh ``` -# Usage - -## 4. Run the notebook +## Usage Example - Notebook can be found in the `example_notebook.ipynb` file - Make sure to select .venv as the kernel in the notebook - -## 4. Usage Example - ```python -import os -from gen3_metadata.parser import Gen3MetadataParser -from dotenv import load_dotenv - -load_dotenv() -key_file = os.getenv('credentials_path') -# Set up credentials path -key_file = os.getenv('credentials_path') +from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser -# Initialize the Gen3MetadataParser +# Initialise +key_file = "path/to/credentials.json" gen3metadata = Gen3MetadataParser(key_file) -# authenticate +# Authenticate gen3metadata.authenticate() -# Fetch data for different categories -gen3metadata.fetch_data("program1", "AusDiab_Simulated", "subject") -gen3metadata.fetch_data("program1", "AusDiab_Simulated", "demographic") -gen3metadata.fetch_data("program1", "AusDiab_Simulated", "medical_history") - -# Convert fetched data to a pandas DataFrame -gen3metadata.data_to_pd() - -# Print the keys of the data sets that have been fetched -print(gen3metadata.data_store.keys()) - -# Return a json of one of the datasets -gen3metadata.data_store["program1/AusDiab_Simulated/subject"] +# Fetching data and returning as dataframe +program_name = "program1" +project_code = "project1" +node_label="medical_history" +pd_data = gen3metadata.fetch_data_pd(program_name, project_code, node_label=node_label) +pd_data -# Return the pandas dataframe of one of the datasets -gen3metadata.data_store_pd["program1/AusDiab_Simulated/subject"] +# Fetching data and returning as json +json_data = gen3metadata.fetch_data_json(program_name, project_code, node_label=node_label) +json_data ``` -The fetched data is stored in a dictionary within the `Gen3MetadataParser` instance. -Each category of data fetched is stored as a key-value pair in this dictionary, -where the key is the category name and the value is the corresponding data. -This allows for easy access and manipulation of the data after it has been fetched. - - - -## 5. Running Tests +## Running Tests The tests are written using the `pytest` framework. ```bash -pytest tests/ +pytest -vv tests/ ``` +--- - - -## Installation of the R version of gen3-metadata +# Installation of the R version of gen3-metadata You can install the gen3metadata R tool from [GitHub](https://github.com/) with: @@ -112,12 +69,14 @@ library("gen3metadata") ## Usage Example This is a basic example to authenticate and load some data. -You will need a credential file (stored in `key_file_path` in this example). ``` r # Load the library library("gen3metadata") +# Set the path to the credentials file +key_file_path <- "path/to/credentials.json" + # Create the Gen3 Metadata Parser object gen3 <- Gen3MetadataParser(key_file_path) diff --git a/example_notebook.ipynb b/example_notebook.ipynb index dba64b1..c13225f 100644 --- a/example_notebook.ipynb +++ b/example_notebook.ipynb @@ -15,7 +15,6 @@ "metadata": {}, "outputs": [], "source": [ - "import gen3_metadata\n", "from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser" ] }, @@ -25,12 +24,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Load credentiaals path from .env\n", - "import os\n", - "from dotenv import load_dotenv\n", - "from gen3_metadata.gen3_metadata_parser import Gen3MetadataParser\n", - "load_dotenv()\n", - "key_file = os.getenv('credentials_path')" + "# Initialise\n", + "key_file = \"path/to/credentials.json\"\n", + "gen3metadata = Gen3MetadataParser(key_file)" ] }, { @@ -39,18 +35,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Or define credentials path in this notebook\n", - "key_file = \"path/to/credentials.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Autheticating\n", - "gen3metadata = Gen3MetadataParser(key_file)\n", + "# Authenticate\n", "gen3metadata.authenticate()" ] }, @@ -60,19 +45,10 @@ "metadata": {}, "outputs": [], "source": [ - "# fetching data\n", + "# fetching data and returning as dataframe\n", "program_name= \"program1\"\n", "project_code= \"AusDiab_Simulated\"\n", - "\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"subject\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"demographic\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"exposure\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"lab_result\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"medical_history\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"medication\")\n", - "gen3metadata.fetch_data(program_name, project_code, node_label= \"blood_pressure_test\")\n", - "\n", - "gen3metadata.data_to_pd()" + "gen3metadata.fetch_data_pd(program_name, project_code, node_label= \"medical_history\")" ] }, { @@ -81,16 +57,9 @@ "metadata": {}, "outputs": [], "source": [ - "gen3metadata.data_store_pd[f\"{program_name}/{project_code}/subject\"]\n", - "gen3metadata.data_store_pd[f\"{program_name}/{project_code}/demographic\"]" + "# fetching data and returning as json\n", + "gen3metadata.fetch_data_json(program_name, project_code, node_label= \"medical_history\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {