diff --git a/.github/workflows/pr-integration-tests.yaml b/.github/workflows/pr-integration-tests.yaml index 63e87210f..c8bc9ba82 100644 --- a/.github/workflows/pr-integration-tests.yaml +++ b/.github/workflows/pr-integration-tests.yaml @@ -54,6 +54,35 @@ jobs: done done + # Remove any lite plugins from the changed_dirs array + readarray -t lite_plugin_array < lite_plugins + + echo "${changed_dirs[@]}" + echo "${lite_plugin_array[@]}" + # Function to remove items in array2 from array1 + remove_items() { + local -n source_array=$1 + local -n remove_array=$2 + local temp_array=() + + for item in "${source_array[@]}"; do + skip=false + for remove_item in "${remove_array[@]}"; do + if [[ "$item" == "$remove_item" ]]; then + skip=true + break + fi + done + if ! $skip; then + temp_array+=("$item") + fi + done + + source_array=("${temp_array[@]}") + } + + remove_items changed_dirs lite_plugin_array + echo "changed-plugins=${changed_dirs[*]}" >> $GITHUB_OUTPUT #---------------------------------------------- diff --git a/.github/workflows/pr-linting-and-unit-tests.yaml b/.github/workflows/pr-linting-and-unit-tests.yaml index 6c4aa28ff..75834765f 100644 --- a/.github/workflows/pr-linting-and-unit-tests.yaml +++ b/.github/workflows/pr-linting-and-unit-tests.yaml @@ -71,7 +71,7 @@ jobs: run: | for dir in ${{ steps.changed-plugins.outputs.changed-plugins }}; do cd $dir - poetry install --no-interaction --no-root --extras "aca-py" + poetry install --no-interaction --no-root --all-extras cd .. done #---------------------------------------------- diff --git a/README.md b/README.md index 22c432398..e33cd5478 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,16 @@ A script was developed to help with maitenance of the repo called `repo_manager. Run `python repo_manager.py` and you will be met with 2 options. - (1) Is used for starting or adding a new plugin. It will generate all the common scaffolding for a plugin which has the expected format. - (2) Is used for updating and changing common poetry dependencies and configurations. It takes the poetry sections in the `pyproject.toml` files from the `plugin_globals` directory and combines them with the local plugin poetry sections. For the dependencies the common will be overridden by the globals. The other config sections will be replaced by the global configs. Then the lock files will be removed and re-installed. + - (3) Is used for updating the plugin versions in the `plugin_globals` directory. It will update the versions of the plugins in the `plugin_globals` directory to the latest version on the main branch of the plugin repo. It will also update the `plugin_globals` directory to the latest version on the main branch of the plugin repo. + - (4) This option is used by the CI/CD release pipeline. It updates the release notes and the individual plugins with a new version of aries_cloudagent. + - (5) This option is also used by the CI/CD release pipeline. It gets any plugins that have succeeded the tests after a new version of aries_cloudagent has been released if their changes were not reverted than the plugin has been updated to the new version of aries_cloudagent. + - (6) This option will run a general update for all poetry lock files in all plugins. + - (7) This option is used for upgrading a particular library for all plugins. It's useful for when you don't want to do a general upgrade for every library. + +## Lite plugins + +Sometimes is desirable to have a plugin that doesn't need integration tests or extra scaffholding. However, we need a way to avoid these plugins running integration tests in the CI/CD pipeline. To do this, we can simply add the plugin name to the `lite_plugins` file. Which is a line seperated list of plugin names. +``` ## Plugin Documentation diff --git a/lite_plugins b/lite_plugins new file mode 100644 index 000000000..08d90c91d --- /dev/null +++ b/lite_plugins @@ -0,0 +1,2 @@ +jwt_vc_json +mso_mdoc \ No newline at end of file diff --git a/repo_manager.py b/repo_manager.py index 8e992e18d..54b132c30 100644 --- a/repo_manager.py +++ b/repo_manager.py @@ -5,7 +5,8 @@ import sys from copy import deepcopy from enum import Enum -from typing import Optional +from pathlib import Path +from typing import Optional, Tuple GLOBAL_PLUGIN_DIR = "plugin_globals" @@ -164,7 +165,7 @@ def get_section_output( return j - i -def get_and_combine_main_poetry_sections(name: str) -> (dict, dict): +def get_and_combine_main_poetry_sections(name: str) -> Tuple[dict, dict]: """Get the global main sections and combine them with the plugin specific sections.""" global_sections = deepcopy(sections) plugin_sections = deepcopy(sections) @@ -283,18 +284,27 @@ def replace_global_sections(name: str) -> None: """ global_sections, plugin_sections = get_and_combine_main_poetry_sections(name) process_main_config_sections(name, plugin_sections, global_sections) - global_sections, plugin_sections = get_and_combine_integration_poetry_sections(name) - process_integration_config_sections(name, plugin_sections, global_sections) + if is_plugin_directory(name, True): + global_sections, plugin_sections = get_and_combine_integration_poetry_sections(name) + process_integration_config_sections(name, plugin_sections, global_sections) -def is_plugin_directory(plugin_name: str) -> bool: +def is_plugin_directory(plugin_name: str, exclude_lite_plugins: bool = False) -> bool: # If there is a directory which is not a plugin it should be ignored here + if exclude_lite_plugins: + lite_plugins = Path('lite_plugins').read_text().splitlines() + return ( + os.path.isdir(plugin_name) + and plugin_name != GLOBAL_PLUGIN_DIR + and not plugin_name.startswith(".") + and plugin_name not in lite_plugins + ) return ( os.path.isdir(plugin_name) and plugin_name != GLOBAL_PLUGIN_DIR and not plugin_name.startswith(".") ) - + def update_all_poetry_locks(): for root, _, files in os.walk("."): @@ -302,10 +312,7 @@ def update_all_poetry_locks(): print(f"Updating poetry.lock in {root}") subprocess.run(["poetry", "lock"], cwd=root) -def upgrade_library_in_all_plugins(library: str = None): - if library is None: - library = input("Enter the library to upgrade: ") - +def upgrade_library_in_all_plugins(library: str = None): for root, _, files in os.walk("."): if "poetry.lock" in files: with open(f"{root}/poetry.lock", "r") as file: @@ -367,9 +374,11 @@ def main(arg_1=None, arg_2=None): print(f"Updating common poetry sections in {plugin_name}\n") replace_global_sections(plugin_name) os.system(f"cd {plugin_name} && rm poetry.lock && poetry lock") - os.system( - f"cd {plugin_name}/integration && rm poetry.lock && poetry lock" - ) + # Don't update lite plugin integration files (They don't have any) + if is_plugin_directory(plugin_name, True): + os.system( + f"cd {plugin_name}/integration && rm poetry.lock && poetry lock" + ) elif selection == "3": # Upgrade plugin globals lock file