From 2fcc8d1e5df959f95396cbdaaf92374efb091b9e Mon Sep 17 00:00:00 2001 From: jamshale Date: Thu, 18 Jul 2024 15:29:01 -0700 Subject: [PATCH 1/6] Allow lite plugins to be ignored by management and integration tests Signed-off-by: jamshale --- .github/workflows/pr-integration-tests.yaml | 29 +++++++++++++++++++++ lite_plugins | 1 + repo_manager.py | 8 +++--- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 lite_plugins diff --git a/.github/workflows/pr-integration-tests.yaml b/.github/workflows/pr-integration-tests.yaml index 63e87210f..c8bc9ba82 100644 --- a/.github/workflows/pr-integration-tests.yaml +++ b/.github/workflows/pr-integration-tests.yaml @@ -54,6 +54,35 @@ jobs: done done + # Remove any lite plugins from the changed_dirs array + readarray -t lite_plugin_array < lite_plugins + + echo "${changed_dirs[@]}" + echo "${lite_plugin_array[@]}" + # Function to remove items in array2 from array1 + remove_items() { + local -n source_array=$1 + local -n remove_array=$2 + local temp_array=() + + for item in "${source_array[@]}"; do + skip=false + for remove_item in "${remove_array[@]}"; do + if [[ "$item" == "$remove_item" ]]; then + skip=true + break + fi + done + if ! $skip; then + temp_array+=("$item") + fi + done + + source_array=("${temp_array[@]}") + } + + remove_items changed_dirs lite_plugin_array + echo "changed-plugins=${changed_dirs[*]}" >> $GITHUB_OUTPUT #---------------------------------------------- diff --git a/lite_plugins b/lite_plugins new file mode 100644 index 000000000..19aa00dc4 --- /dev/null +++ b/lite_plugins @@ -0,0 +1 @@ +basicmessage_storage \ No newline at end of file diff --git a/repo_manager.py b/repo_manager.py index 8e992e18d..c4aa06d5a 100644 --- a/repo_manager.py +++ b/repo_manager.py @@ -5,6 +5,7 @@ import sys from copy import deepcopy from enum import Enum +from pathlib import Path from typing import Optional GLOBAL_PLUGIN_DIR = "plugin_globals" @@ -289,10 +290,12 @@ def replace_global_sections(name: str) -> None: def is_plugin_directory(plugin_name: str) -> bool: # If there is a directory which is not a plugin it should be ignored here + lite_plugins = Path('lite_plugins').read_text().splitlines() return ( os.path.isdir(plugin_name) and plugin_name != GLOBAL_PLUGIN_DIR and not plugin_name.startswith(".") + and plugin_name not in lite_plugins ) @@ -302,10 +305,7 @@ def update_all_poetry_locks(): print(f"Updating poetry.lock in {root}") subprocess.run(["poetry", "lock"], cwd=root) -def upgrade_library_in_all_plugins(library: str = None): - if library is None: - library = input("Enter the library to upgrade: ") - +def upgrade_library_in_all_plugins(library: str = None): for root, _, files in os.walk("."): if "poetry.lock" in files: with open(f"{root}/poetry.lock", "r") as file: From cbafc6b1ae4d72307aecf681bf615ebd45dc4fb5 Mon Sep 17 00:00:00 2001 From: jamshale Date: Fri, 19 Jul 2024 08:44:01 -0700 Subject: [PATCH 2/6] Add upcoming lite plugins Signed-off-by: jamshale --- lite_plugins | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lite_plugins b/lite_plugins index 19aa00dc4..08d90c91d 100644 --- a/lite_plugins +++ b/lite_plugins @@ -1 +1,2 @@ -basicmessage_storage \ No newline at end of file +jwt_vc_json +mso_mdoc \ No newline at end of file From 2371ac7f1be4c135f48e77d3a7a52a6d6fd034d0 Mon Sep 17 00:00:00 2001 From: jamshale Date: Fri, 19 Jul 2024 08:45:01 -0700 Subject: [PATCH 3/6] Install all extras for unit tests workflow Signed-off-by: jamshale --- .github/workflows/pr-linting-and-unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-linting-and-unit-tests.yaml b/.github/workflows/pr-linting-and-unit-tests.yaml index 6c4aa28ff..75834765f 100644 --- a/.github/workflows/pr-linting-and-unit-tests.yaml +++ b/.github/workflows/pr-linting-and-unit-tests.yaml @@ -71,7 +71,7 @@ jobs: run: | for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do cd $dir - poetry install --no-interaction --no-root --extras "aca-py" + poetry install --no-interaction --no-root --all-extras cd .. done #---------------------------------------------- From fb752db91adcab5fcf35f4f38c75a3d395a6db43 Mon Sep 17 00:00:00 2001 From: jamshale Date: Mon, 22 Jul 2024 13:08:15 -0700 Subject: [PATCH 4/6] README update and another exclude condition in repo_manager Signed-off-by: jamshale --- README.md | 10 ++++++++++ repo_manager.py | 27 ++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 22c432398..dcca51e74 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,16 @@ A script was developed to help with maitenance of the repo called `repo_manager. Run `python repo_manager.py` and you will be met with 2 options. - (1) Is used for starting or adding a new plugin. It will generate all the common scaffolding for a plugin which has the expected format. - (2) Is used for updating and changing common poetry dependencies and configurations. It takes the poetry sections in the `pyproject.toml` files from the `plugin_globals` directory and combines them with the local plugin poetry sections. For the dependencies the common will be overridden by the globals. The other config sections will be replaced by the global configs. Then the lock files will be removed and re-installed. + - (3) Is used for updating the plugin versions in the `plugin_globals` directory. It will update the versions of the plugins in the `plugin_globals` directory to the latest version on the main branch of the plugin repo. It will also update the `plugin_globals` directory to the latest version on the main branch of the plugin repo. + - (4) This option is used by the CI/CD release pipeline. It updates the release notes and the individual plugins with a new version of aries_cloudagent. + - (5) This option is also used by the CI/CD release pipeline. It gets any plugins that have succeeded the tests after a new version of aries_cloudagent has been released if their changes were not reverted than the plugin has been updated to the new version of aries_cloudagent. + - (6) This option will run a general update for all poetry lock files in all plugins. + - (7) This option is used for upgrading a particular library for all plugins. It's useful for when you don't want to do a general upgrade for every library. + +## Lite plugins + +Sometimes is desirable to have a plugin that doesn't need integration tests or extra scaffholding. However, we need a way to avoid these plugins running integration tests in the CI/CD pipeline. To do this, we can simple add the plugin name to the `lite_plugins` file. Which is a line seperated list of plugin names. +``` ## Plugin Documentation diff --git a/repo_manager.py b/repo_manager.py index c4aa06d5a..37ea94803 100644 --- a/repo_manager.py +++ b/repo_manager.py @@ -284,20 +284,27 @@ def replace_global_sections(name: str) -> None: """ global_sections, plugin_sections = get_and_combine_main_poetry_sections(name) process_main_config_sections(name, plugin_sections, global_sections) - global_sections, plugin_sections = get_and_combine_integration_poetry_sections(name) - process_integration_config_sections(name, plugin_sections, global_sections) + if is_plugin_directory(name, True): + global_sections, plugin_sections = get_and_combine_integration_poetry_sections(name) + process_integration_config_sections(name, plugin_sections, global_sections) -def is_plugin_directory(plugin_name: str) -> bool: +def is_plugin_directory(plugin_name: str, exclude_lite_plugins: bool = False) -> bool: # If there is a directory which is not a plugin it should be ignored here - lite_plugins = Path('lite_plugins').read_text().splitlines() + if exclude_lite_plugins: + lite_plugins = Path('lite_plugins').read_text().splitlines() + return ( + os.path.isdir(plugin_name) + and plugin_name != GLOBAL_PLUGIN_DIR + and not plugin_name.startswith(".") + and plugin_name not in lite_plugins + ) return ( os.path.isdir(plugin_name) and plugin_name != GLOBAL_PLUGIN_DIR and not plugin_name.startswith(".") - and plugin_name not in lite_plugins ) - + def update_all_poetry_locks(): for root, _, files in os.walk("."): @@ -367,9 +374,11 @@ def main(arg_1=None, arg_2=None): print(f"Updating common poetry sections in {plugin_name}\n") replace_global_sections(plugin_name) os.system(f"cd {plugin_name} && rm poetry.lock && poetry lock") - os.system( - f"cd {plugin_name}/integration && rm poetry.lock && poetry lock" - ) + # Don't update lite plugin integration files (They don't have any) + if is_plugin_directory(plugin_name, True): + os.system( + f"cd {plugin_name}/integration && rm poetry.lock && poetry lock" + ) elif selection == "3": # Upgrade plugin globals lock file From 5747d0462812fadca155d5d1e3d7adf3031e5c72 Mon Sep 17 00:00:00 2001 From: jamshale Date: Tue, 23 Jul 2024 10:22:15 -0700 Subject: [PATCH 5/6] fix typing Signed-off-by: jamshale --- repo_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo_manager.py b/repo_manager.py index 37ea94803..54b132c30 100644 --- a/repo_manager.py +++ b/repo_manager.py @@ -6,7 +6,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Optional +from typing import Optional, Tuple GLOBAL_PLUGIN_DIR = "plugin_globals" @@ -165,7 +165,7 @@ def get_section_output( return j - i -def get_and_combine_main_poetry_sections(name: str) -> (dict, dict): +def get_and_combine_main_poetry_sections(name: str) -> Tuple[dict, dict]: """Get the global main sections and combine them with the plugin specific sections.""" global_sections = deepcopy(sections) plugin_sections = deepcopy(sections) From dce6256869640d57f39794638677d1ff6291099e Mon Sep 17 00:00:00 2001 From: jamshale Date: Tue, 23 Jul 2024 10:30:45 -0700 Subject: [PATCH 6/6] fix spelling Signed-off-by: jamshale --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcca51e74..e33cd5478 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Run `python repo_manager.py` and you will be met with 2 options. ## Lite plugins -Sometimes is desirable to have a plugin that doesn't need integration tests or extra scaffholding. However, we need a way to avoid these plugins running integration tests in the CI/CD pipeline. To do this, we can simple add the plugin name to the `lite_plugins` file. Which is a line seperated list of plugin names. +Sometimes is desirable to have a plugin that doesn't need integration tests or extra scaffholding. However, we need a way to avoid these plugins running integration tests in the CI/CD pipeline. To do this, we can simply add the plugin name to the `lite_plugins` file. Which is a line seperated list of plugin names. ``` ## Plugin Documentation