From fd93800e50dc1472b64f9ba910081a56c822bf3b Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 17:54:58 -0600 Subject: [PATCH 01/38] Move log statement --- .workflow/builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workflow/builds.py b/.workflow/builds.py index 8e3f13ea40..1eac99850e 100644 --- a/.workflow/builds.py +++ b/.workflow/builds.py @@ -8,8 +8,8 @@ def add_build_for_envs( orch ): dm_opt = [ "ON", "OFF" ] build_types = [ "Release", "Debug" ] configurations = { "ARW" : [ "EM_REAL", "EM_FIRE", "EM_B_WAVE" ] } + orch.log( f"Creating builds for permutations..." ) for core, env, build_type, sm, dm in itertools.product( configurations, envs, build_types, sm_opt, dm_opt ): - orch.log( f"Creating builds for permutation core: {core} env: {env} build_type: {build_type} sm: {sm} dm: {dm}" ) for case in configurations[core]: sm_desc = "_sm" if sm == "ON" else "" dm_desc = "_dm" if dm == "ON" else "" From d02fb2b48c2004b2ef5c8a9d3cd5adddc4cc4529 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:00:47 -0600 Subject: [PATCH 02/38] Fix names and paths --- .github/workflows/ci_rework.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index a39e70065a..ef35c35a58 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -64,6 +64,7 @@ jobs: - host : derecho name : "Make Compilation Tests" id : make-tests + label: buildcmake args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch : - priority : -99 @@ -99,7 +100,7 @@ jobs: state: 'pending' }) - - uses: ./.github/actions/sane-workflows + - uses: ./.github/actions/sane_workflows name: ${{ inputs.name }} with: # Everything below this should remain the same and comes from the workflow matrix @@ -121,13 +122,13 @@ jobs: repo: context.repo.repo, sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ inputs.name }}', - context: '${{ inputs.host }}/${{ inputs.id }}', + description: '${{ matrix.workflow.name }}', + context: '${{ matrix.workflow.host }}/${{ matrix.workflow.id }}', state: '${{ job.status == 'success' && 'success' || 'failure' }}' }) - - name : Remove '${{ inputs.label }}' label - if : ${{ !cancelled() && inputs.event_label == inputs.label }} + - name : Remove '${{ matrix.workflow.label }}' label + if : ${{ !cancelled() && inputs.event_label == matrix.workflow.label }} env: PR_NUMBER: ${{ inputs.event_number }} run: | @@ -135,7 +136,7 @@ jobs: -X DELETE \ -H "Accept: application/vnd.github.v3+json" \ -H 'Authorization: token ${{ github.token }}' \ - https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ matrix.workflow.label }} # In the event that 'all-tests' is used, this final job will be the one to remove # the label from the PR From 4a2e612f6f54f80d7f42e2766706418a65d3aaef Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:03:39 -0600 Subject: [PATCH 03/38] Checkout --- .github/workflows/ci_rework.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index ef35c35a58..8c5742c701 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -100,6 +100,11 @@ jobs: state: 'pending' }) + - uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} + - uses: ./.github/actions/sane_workflows name: ${{ inputs.name }} with: From a7a1830153643c010618176ed15ff0aad91f5920 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:04:56 -0600 Subject: [PATCH 04/38] Syntax issue --- .github/workflows/ci_rework.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 8c5742c701..da1d30b111 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -100,10 +100,10 @@ jobs: state: 'pending' }) - - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} + - uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} - uses: ./.github/actions/sane_workflows name: ${{ inputs.name }} From b3875931176afbec5308546b021ee268eaff13d5 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:10:49 -0600 Subject: [PATCH 05/38] More syntax issues --- .github/actions/sane_workflows/action.yml | 4 +++- .github/workflows/ci_rework.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index b0f09be08c..087df70fb1 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -45,6 +45,7 @@ runs: - if: ${{ inputs.cache == 'false' || steps.cache-.outputs.cache-hit != 'true' }} name: Install sane-workflows id : install-sane + shell: bash run: | if [ -x "${{ inputs.venv }}/bin/activate" ]; then . ${{ inputs.venv }}/bin/activate @@ -60,6 +61,7 @@ runs: - name: Run ${{ inputs.id }} id : runTest + shell: bash run: | NAME=${{ inputs.run_id }}-${{ inputs.id }} @@ -77,7 +79,7 @@ runs: PATCH="-p .patch-${{ inputs.id }}/" fi - sane_runner ${{ input.args }} $ACTIONS $HOST $PATCH \ + sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \ -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ -sl .${NAME}_saves \ -ll ${NAME}_logs diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index da1d30b111..bc9f7786d6 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -106,7 +106,7 @@ jobs: ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} - uses: ./.github/actions/sane_workflows - name: ${{ inputs.name }} + name: ${{ matrix.workflow.name }} with: # Everything below this should remain the same and comes from the workflow matrix id : ${{ matrix.workflow.id }} From 19229be5e94f3d567d08c8ee19ce894e1e87ccf0 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:14:01 -0600 Subject: [PATCH 06/38] Bad args to action --- .github/workflows/ci_rework.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index bc9f7786d6..e37a51ac03 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -112,7 +112,6 @@ jobs: id : ${{ matrix.workflow.id }} host : ${{ matrix.workflow.host }} actions : ${{ toJson( matrix.workflow.actions ) }} - mkdirs : ${{ matrix.workflow.mkdirs }} args : ${{ matrix.workflow.args }} patch : '${{ toJson( matrix.workflow.patch ) }}' From d97fd071016446333159a91d510f5b0dcf459162 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 18:56:52 -0600 Subject: [PATCH 07/38] Fix syntax --- .github/workflows/ci_rework.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index e37a51ac03..c893dc8221 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -67,11 +67,11 @@ jobs: label: buildcmake args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch : - - priority : -99 - - hosts : - - '[.*derecho.*]' : - - account : NMMM0012 - - queue : main + priority : -99 + hosts : + '[.*derecho.*]' : + account : NMMM0012 + queue : main actions : - build_cmake_arw_em_real_gnu_release_dm - build_cmake_arw_em_real_gnu_debug_dm From d3916fd74c21ed6b874ca7f8ad84264294968d15 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 19:11:55 -0600 Subject: [PATCH 08/38] Add missing inputs --- .github/actions/sane_workflows/action.yml | 27 ++++++++++++++++++++--- .github/workflows/ci_rework.yml | 3 +++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 087df70fb1..0f50588299 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -30,6 +30,26 @@ inputs: type : string default : "" + cache: + required : false + type : boolean + default : false + cache_id: + required : false + type : string + default : "" + venv: + required : false + type : string + default : "./venv/sane" + paths: + required : true + type : string + python_dependencies: + required : false + type : string + default : "" + runs: using: "composite" steps: @@ -63,7 +83,7 @@ runs: id : runTest shell: bash run: | - NAME=${{ inputs.run_id }}-${{ inputs.id }} + NAME=${{ github.run_id }}-${{ inputs.id }} if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" @@ -78,6 +98,7 @@ runs: echo '{ "patch" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json PATCH="-p .patch-${{ inputs.id }}/" fi + . ${{ inputs.venv }}/bin/activate sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \ -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ @@ -88,6 +109,6 @@ runs: name: Upload test logs uses : actions/upload-artifact@v4 with: - name: ${{ inputs.run_id }}-${{ inputs.id }}_logfiles - path: ${{ inputs.run_id }}-${{ inputs.id }}_logs + name: ${{ github.run_id }}-${{ inputs.id }}_logfiles + path: ${{ github.run_id }}-${{ inputs.id }}_logs include-hidden-files: true diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index c893dc8221..eed2406d19 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -75,6 +75,8 @@ jobs: actions : - build_cmake_arw_em_real_gnu_release_dm - build_cmake_arw_em_real_gnu_debug_dm + paths: + - .workflow/ # I am leaving this here for posterity if this is to be replicated in private repositories for testing permissions: contents: read @@ -114,6 +116,7 @@ jobs: actions : ${{ toJson( matrix.workflow.actions ) }} args : ${{ matrix.workflow.args }} patch : '${{ toJson( matrix.workflow.patch ) }}' + paths : '${{ toJson( matrix.workflow.paths ) }}' - name: Set completed status if: ${{ always() }} From 61124a937ae0d2fad1303900dd1482f0c0e9173c Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 19:14:46 -0600 Subject: [PATCH 09/38] More issues --- .github/actions/sane_workflows/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 0f50588299..4476c57b62 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -53,7 +53,7 @@ inputs: runs: using: "composite" steps: - - if: ${{ inputs.cache }} + - if: ${{ inputs.cache == 'true' }} name: Cache python environment id: cache-venv uses: actions/cache@v3 @@ -62,7 +62,7 @@ runs: ${{ inputs.venv }} key: venv-${{ inputs.cache_id || inputs.id }} - - if: ${{ inputs.cache == 'false' || steps.cache-.outputs.cache-hit != 'true' }} + - if: ${{ inputs.cache == 'false' || steps.cache-venv.outputs.cache-hit != 'true' }} name: Install sane-workflows id : install-sane shell: bash @@ -103,7 +103,8 @@ runs: sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \ -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ -sl .${NAME}_saves \ - -ll ${NAME}_logs + -ll ${NAME}_logs \ + -d - if : ${{ inputs.upload && failure() }} name: Upload test logs From f3e193f700e1f4c3949af44d373243be45589389 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 21:57:35 -0600 Subject: [PATCH 10/38] Try to short circuit json parsing if not available --- .github/actions/sane_workflows/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 4476c57b62..cc32da1d37 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -76,7 +76,7 @@ runs: python3 -m pip install sane-workflows if [ ! -z "${{ inputs.python_dependencies }}" ]; then - python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} + python3 -m pip install ${{ inputs.python_dependencies == "" || join( fromJson( inputs.python_dependencies ), ' ' ) }} fi - name: Run ${{ inputs.id }} @@ -88,7 +88,7 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ ! -z "${{ inputs.actions }}" ]; then - ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" + ACTIONS="-a ${{ inputs.actions == "" || join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" From 6edcf4f0d89705ca376d83637c5dab19471e4140 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:01:30 -0600 Subject: [PATCH 11/38] Try using single quotes --- .github/actions/sane_workflows/action.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index cc32da1d37..c0071d81ce 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -4,7 +4,7 @@ inputs: archive : required : false type : string - default : "./" + default : './' id: required : true @@ -12,23 +12,23 @@ inputs: host: required : true type : string - default : "" + default : '' actions: required : false type : string - default : "" + default : '' actions_filter: required : false type : string - default : "" + default : '' args : required : false type : string - default : "" + default : '' patch: required : false type : string - default : "" + default : '' cache: required : false @@ -37,7 +37,7 @@ inputs: cache_id: required : false type : string - default : "" + default : '' venv: required : false type : string @@ -48,7 +48,7 @@ inputs: python_dependencies: required : false type : string - default : "" + default : '' runs: using: "composite" @@ -76,7 +76,7 @@ runs: python3 -m pip install sane-workflows if [ ! -z "${{ inputs.python_dependencies }}" ]; then - python3 -m pip install ${{ inputs.python_dependencies == "" || join( fromJson( inputs.python_dependencies ), ' ' ) }} + python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} fi - name: Run ${{ inputs.id }} @@ -88,7 +88,7 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ ! -z "${{ inputs.actions }}" ]; then - ACTIONS="-a ${{ inputs.actions == "" || join( fromJson( inputs.actions ), ' ' ) }}" + ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" From 79a0fddb5cd738615fc4a5d535f7b8c6eb0dae1d Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:13:48 -0600 Subject: [PATCH 12/38] Set default string JSON --- .github/actions/sane_workflows/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index c0071d81ce..c5e927f09b 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -16,7 +16,7 @@ inputs: actions: required : false type : string - default : '' + default : '[]' actions_filter: required : false type : string @@ -48,7 +48,7 @@ inputs: python_dependencies: required : false type : string - default : '' + default : '[]' runs: using: "composite" From 45fc86a36649ddc9391940973290208861b9407d Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:17:24 -0600 Subject: [PATCH 13/38] Change if string check --- .github/actions/sane_workflows/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index c5e927f09b..ee09009604 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -75,7 +75,7 @@ runs: fi python3 -m pip install sane-workflows - if [ ! -z "${{ inputs.python_dependencies }}" ]; then + if [ "${{ inputs.python_dependencies }}" != "[]" ]; then python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} fi @@ -87,7 +87,7 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" - elif [ ! -z "${{ inputs.actions }}" ]; then + elif [ "${{ inputs.actions }}" != "[]" ]; then ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then From 372fe10c3c66bb4ba1c257031bb90f5a53340671 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:28:52 -0600 Subject: [PATCH 14/38] Note build is cmake --- .workflow/builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.workflow/builds.py b/.workflow/builds.py index 1eac99850e..1a3400008a 100644 --- a/.workflow/builds.py +++ b/.workflow/builds.py @@ -15,7 +15,7 @@ def add_build_for_envs( orch ): dm_desc = "_dm" if dm == "ON" else "" id = f"{core}_{case}_{env}_{build_type}{sm_desc}{dm_desc}".lower() - action = sane.Action( f"build_{id}" ) + action = sane.Action( f"build_cmake_{id}" ) action.config["command"] = ".workflow/scripts/buildCMake.sh" args = [] From c85be917abdf891fc9e4c16060843450d8467c13 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:51:35 -0600 Subject: [PATCH 15/38] Real run --- .github/actions/sane_workflows/action.yml | 6 +++--- .github/workflows/ci_rework.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index ee09009604..32ba2b330a 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -88,14 +88,14 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ "${{ inputs.actions }}" != "[]" ]; then - ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" + ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" fi if [ ! -z "${{ inputs.patch }}" ]; then mkdir -p .patch-${{ inputs.id }}/ - echo '{ "patch" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json + echo '{ "patches" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json PATCH="-p .patch-${{ inputs.id }}/" fi . ${{ inputs.venv }}/bin/activate @@ -104,7 +104,7 @@ runs: -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ -sl .${NAME}_saves \ -ll ${NAME}_logs \ - -d + -r - if : ${{ inputs.upload && failure() }} name: Upload test logs diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index eed2406d19..7949a5836d 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -52,7 +52,7 @@ on: # - < next test > # https://stackoverflow.com/a/68940067 jobs: - buildcmake: + compile-tests: if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }} name : Test ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }} runs-on: ${{ matrix.workflow.host }} @@ -64,7 +64,7 @@ jobs: - host : derecho name : "Make Compilation Tests" id : make-tests - label: buildcmake + label: compile-tests args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch : priority : -99 @@ -97,8 +97,8 @@ jobs: repo: context.repo.repo, sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ inputs.name }}', - context: '${{ inputs.host }}/${{ inputs.id }}', + description: '${{ matrix.workflow.name }}', + context: '${{ matrix.workflow.host }}/${{ matrix.workflow.id }}', state: 'pending' }) From 0a379338a25a622f61a7796a55d1f7e84e6c0e45 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:52:11 -0600 Subject: [PATCH 16/38] Change name --- .github/workflows/ci_rework.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 7949a5836d..8bbb894978 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -62,8 +62,8 @@ jobs: matrix: workflow : - host : derecho - name : "Make Compilation Tests" - id : make-tests + name : "CMake Compilation Tests" + id : cmake-tests label: compile-tests args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch : From 84ace9e50be7da4ce62437ada18c0cb13a5e4637 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Wed, 22 Oct 2025 22:53:24 -0600 Subject: [PATCH 17/38] Fix deps --- .github/workflows/ci_rework.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 8bbb894978..1102bb24a6 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -151,7 +151,7 @@ jobs: if : ${{ !cancelled() && inputs.event_label == 'all-tests' }} name : Remove 'all-tests' label runs-on: ubuntu-latest - needs : [ buildcmake ] # Put tests here to make this wait for the tests to complete + needs : [ compile-tests ] # Put tests here to make this wait for the tests to complete permissions: pull-requests: write steps: From 8d9585fdc4aa7a4a8c992be3e3350cef86283482 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 14:00:06 -0600 Subject: [PATCH 18/38] Add job name --- .github/workflows/ci_rework.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 1102bb24a6..88f502c149 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -70,8 +70,9 @@ jobs: priority : -99 hosts : '[.*derecho.*]' : - account : NMMM0012 - queue : main + account : NMMM0012 + queue : main + job_suffix: '-${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}' actions : - build_cmake_arw_em_real_gnu_release_dm - build_cmake_arw_em_real_gnu_debug_dm From 4bc2df53ef0b7e31cc932c2c3d64d245050d902a Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 14:01:54 -0600 Subject: [PATCH 19/38] Syntax issue --- .github/workflows/ci_rework.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 88f502c149..2f201f840a 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -72,7 +72,7 @@ jobs: '[.*derecho.*]' : account : NMMM0012 queue : main - job_suffix: '-${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}' + job_suffix: -${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} actions : - build_cmake_arw_em_real_gnu_release_dm - build_cmake_arw_em_real_gnu_debug_dm From d3d9831c5a4a24e8c5c0a8e4939f2190caa3f81b Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 15:12:50 -0600 Subject: [PATCH 20/38] Allow venv upgrade --- .github/actions/sane_workflows/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 32ba2b330a..dcbdc72177 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -49,6 +49,10 @@ inputs: required : false type : string default : '[]' + upgrade: : + required : false + type : boolean + default : true runs: using: "composite" @@ -73,10 +77,11 @@ runs: python3 -m venv ${{ inputs.venv }} . ${{ inputs.venv }}/bin/activate fi - python3 -m pip install sane-workflows + + python3 -m pip install sane-workflows ${{ inputs.upgrade && '--upgrade' }} if [ "${{ inputs.python_dependencies }}" != "[]" ]; then - python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} + python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} ${{ inputs.upgrade && '--upgrade' }} fi - name: Run ${{ inputs.id }} From 3dc08d2fdef9d3d668ba7919d63642a869cdfdea Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 15:13:19 -0600 Subject: [PATCH 21/38] simplify name, upload logs on failure --- .github/actions/sane_workflows/action.yml | 10 ++++------ .github/workflows/ci_rework.yml | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index dcbdc72177..612c587865 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -88,8 +88,6 @@ runs: id : runTest shell: bash run: | - NAME=${{ github.run_id }}-${{ inputs.id }} - if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ "${{ inputs.actions }}" != "[]" ]; then @@ -107,14 +105,14 @@ runs: sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \ -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ - -sl .${NAME}_saves \ - -ll ${NAME}_logs \ + -sl .${{ inputs.id }}_saves \ + -ll ${{ inputs.id }}_logs \ -r - if : ${{ inputs.upload && failure() }} name: Upload test logs uses : actions/upload-artifact@v4 with: - name: ${{ github.run_id }}-${{ inputs.id }}_logfiles - path: ${{ github.run_id }}-${{ inputs.id }}_logs + name: ${{ inputs.id }}_logfiles + path: ${{ inputs.id }}_logs include-hidden-files: true diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 2f201f840a..1ceac25cc4 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -54,7 +54,7 @@ on: jobs: compile-tests: if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }} - name : Test ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }} + name : Run ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }} runs-on: ${{ matrix.workflow.host }} strategy: max-parallel: 4 @@ -72,7 +72,7 @@ jobs: '[.*derecho.*]' : account : NMMM0012 queue : main - job_suffix: -${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} + job_suffix: .compile-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} actions : - build_cmake_arw_em_real_gnu_release_dm - build_cmake_arw_em_real_gnu_debug_dm @@ -118,6 +118,7 @@ jobs: args : ${{ matrix.workflow.args }} patch : '${{ toJson( matrix.workflow.patch ) }}' paths : '${{ toJson( matrix.workflow.paths ) }}' + upload : true - name: Set completed status if: ${{ always() }} From 14d5c58d75b9d7f8a03416450f1e36feeb54c1ac Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 15:16:32 -0600 Subject: [PATCH 22/38] Syntax issue --- .github/actions/sane_workflows/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 612c587865..c683a1af3a 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -1,10 +1,10 @@ name: sane-workflows description: "Run a set of actions using sane-workflows" inputs: - archive : + upload : required : false - type : string - default : './' + type : boolean + default : false id: required : true @@ -49,7 +49,7 @@ inputs: required : false type : string default : '[]' - upgrade: : + upgrade: required : false type : boolean default : true From 9d326811ddead57a27285374cb031e63a8d7c5cb Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 17:24:48 -0600 Subject: [PATCH 23/38] Force no cache dir --- .github/actions/sane_workflows/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index c683a1af3a..618512bfd9 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -78,10 +78,10 @@ runs: . ${{ inputs.venv }}/bin/activate fi - python3 -m pip install sane-workflows ${{ inputs.upgrade && '--upgrade' }} + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} sane-workflows if [ "${{ inputs.python_dependencies }}" != "[]" ]; then - python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} ${{ inputs.upgrade && '--upgrade' }} + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} fi - name: Run ${{ inputs.id }} From b2d21d852f461a57224b25169653a81f5e22ddce Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 18:31:51 -0600 Subject: [PATCH 24/38] Try using complex inputs --- .github/actions/sane_workflows/action.yml | 53 ++++++++++++++--------- .github/workflows/ci_rework.yml | 6 +-- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 618512bfd9..7582d54ed9 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -1,54 +1,67 @@ name: sane-workflows description: "Run a set of actions using sane-workflows" inputs: - upload : - required : false - type : boolean - default : false - id: + description: ID of workflow run, i.e. name required : true type : string - host: + paths: + description: List of paths pointing to workflow(s) required : true + type : array + + # Everything else is optional + host: + description: Specific host to run workflow as + required : false type : string default : '' actions: + description: Actions list to run, mutually exclusive with actions_filter required : false - type : string - default : '[]' + type : array + default : [] actions_filter: + description: Actions filter to run, mutually exclusive with actions required : false type : string default : '' args : + description: Additional args to pass to sane_runner required : false type : string default : '' patch: + description: One-time patch allowed for modification of workflow, need be required : false - type : string - default : '' + type : object + default : {} + upload : + description: Whether to upload logs as an artifact on failure + required : false + type : boolean + default : false + # Python environment management cache: + description: Whether to cache the python virtual environment required : false type : boolean default : false cache_id: + description: Specific cache id to use, if not set the key will be ${{ inputs.id }} required : false type : string default : '' venv: + description: Path to setup python virtual environment at required : false type : string - default : "./venv/sane" - paths: - required : true - type : string + default : './venv/sane' python_dependencies: required : false - type : string - default : '[]' + type : array + default : [] upgrade: required : false type : boolean @@ -81,7 +94,7 @@ runs: python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} sane-workflows if [ "${{ inputs.python_dependencies }}" != "[]" ]; then - python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( inputs.python_dependencies, ' ' ) }} fi - name: Run ${{ inputs.id }} @@ -91,14 +104,14 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ "${{ inputs.actions }}" != "[]" ]; then - ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" + ACTIONS="-a ${{ join( inputs.actions, ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" fi if [ ! -z "${{ inputs.patch }}" ]; then mkdir -p .patch-${{ inputs.id }}/ - echo '{ "patches" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json + echo '{ "patches" : toJSON( ${{ inputs.patch }} ) }' >> .patch-${{ inputs.id }}/patch.json PATCH="-p .patch-${{ inputs.id }}/" fi . ${{ inputs.venv }}/bin/activate @@ -107,7 +120,7 @@ runs: -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ -sl .${{ inputs.id }}_saves \ -ll ${{ inputs.id }}_logs \ - -r + -d - if : ${{ inputs.upload && failure() }} name: Upload test logs diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 1ceac25cc4..9500e10bf5 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -114,10 +114,10 @@ jobs: # Everything below this should remain the same and comes from the workflow matrix id : ${{ matrix.workflow.id }} host : ${{ matrix.workflow.host }} - actions : ${{ toJson( matrix.workflow.actions ) }} + actions : ${{ matrix.workflow.actions }} #${{ toJson( matrix.workflow.actions ) }} args : ${{ matrix.workflow.args }} - patch : '${{ toJson( matrix.workflow.patch ) }}' - paths : '${{ toJson( matrix.workflow.paths ) }}' + patch : ${{ matrix.workflow.patch }} #'${{ toJson( matrix.workflow.patch ) }}' + paths : ${{ matrix.workflow.paths }} #'${{ toJson( matrix.workflow.paths ) }}' upload : true - name: Set completed status From c35b4782a9848b5d52596eae473092052020bbb8 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 18:34:15 -0600 Subject: [PATCH 25/38] Remove defaults for complex --- .github/actions/sane_workflows/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 7582d54ed9..9530917405 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -20,7 +20,6 @@ inputs: description: Actions list to run, mutually exclusive with actions_filter required : false type : array - default : [] actions_filter: description: Actions filter to run, mutually exclusive with actions required : false @@ -35,7 +34,6 @@ inputs: description: One-time patch allowed for modification of workflow, need be required : false type : object - default : {} upload : description: Whether to upload logs as an artifact on failure required : false @@ -61,7 +59,6 @@ inputs: python_dependencies: required : false type : array - default : [] upgrade: required : false type : boolean From 53c067857914ebd2bcf676e990af887bde1eb758 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 18:35:38 -0600 Subject: [PATCH 26/38] Fix description --- .github/actions/sane_workflows/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index 9530917405..aa39551293 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -47,7 +47,7 @@ inputs: type : boolean default : false cache_id: - description: Specific cache id to use, if not set the key will be ${{ inputs.id }} + description: Specific cache id to use, if not set the key will be inputs.id required : false type : string default : '' From 70065980b79bffbb63710cc049eec6934b4e943c Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 18:42:08 -0600 Subject: [PATCH 27/38] Complex input did not work --- .github/actions/sane_workflows/action.yml | 17 +++++++++++------ .github/workflows/ci_rework.yml | 6 +++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index aa39551293..16913dfbea 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -19,7 +19,8 @@ inputs: actions: description: Actions list to run, mutually exclusive with actions_filter required : false - type : array + type : string + default : '[]' actions_filter: description: Actions filter to run, mutually exclusive with actions required : false @@ -31,9 +32,10 @@ inputs: type : string default : '' patch: - description: One-time patch allowed for modification of workflow, need be + description: One-time patch dict allowed for modification of workflow, need be required : false - type : object + type : string + default : '' upload : description: Whether to upload logs as an artifact on failure required : false @@ -57,9 +59,12 @@ inputs: type : string default : './venv/sane' python_dependencies: + description: Extra python dependencies needed for workflow required : false type : array + default : '[]' upgrade: + description: Use pip install --upgrade when installing dependencies, not recommended with cache required : false type : boolean default : true @@ -91,7 +96,7 @@ runs: python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} sane-workflows if [ "${{ inputs.python_dependencies }}" != "[]" ]; then - python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( inputs.python_dependencies, ' ' ) }} + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} fi - name: Run ${{ inputs.id }} @@ -101,14 +106,14 @@ runs: if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" elif [ "${{ inputs.actions }}" != "[]" ]; then - ACTIONS="-a ${{ join( inputs.actions, ' ' ) }}" + ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" fi if [ ! -z "${{ inputs.patch }}" ]; then mkdir -p .patch-${{ inputs.id }}/ - echo '{ "patches" : toJSON( ${{ inputs.patch }} ) }' >> .patch-${{ inputs.id }}/patch.json + echo '{ "patches" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json PATCH="-p .patch-${{ inputs.id }}/" fi . ${{ inputs.venv }}/bin/activate diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 9500e10bf5..addd62ff1d 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -114,10 +114,10 @@ jobs: # Everything below this should remain the same and comes from the workflow matrix id : ${{ matrix.workflow.id }} host : ${{ matrix.workflow.host }} - actions : ${{ matrix.workflow.actions }} #${{ toJson( matrix.workflow.actions ) }} + actions : '${{ toJson( matrix.workflow.actions ) }}' args : ${{ matrix.workflow.args }} - patch : ${{ matrix.workflow.patch }} #'${{ toJson( matrix.workflow.patch ) }}' - paths : ${{ matrix.workflow.paths }} #'${{ toJson( matrix.workflow.paths ) }}' + patch : '${{ toJson( matrix.workflow.patch ) }}' + paths : '${{ toJson( matrix.workflow.paths ) }}' upload : true - name: Set completed status From deef31d29fd64d185715f0c9a10ceaad60a46d3b Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 21:02:01 -0600 Subject: [PATCH 28/38] Use separate action and base matrix --- .github/workflows/ci_rework.yml | 44 +++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index addd62ff1d..a0e97d0049 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -60,24 +60,36 @@ jobs: max-parallel: 4 fail-fast: false matrix: - workflow : + base : - host : derecho - name : "CMake Compilation Tests" - id : cmake-tests label: compile-tests - args : -vr='{"cpus":128,"timelimit":"00:10:00"}' + paths: + - .workflow/ patch : priority : -99 hosts : '[.*derecho.*]' : account : NMMM0012 queue : main - job_suffix: .compile-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} + job_suffix: .{0}.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} + workflow: + - name : "CMake GNU Compilation Tests" + id : cmake-gnu + args : -vr='{"cpus":128,"timelimit":"00:10:00"}' actions : - build_cmake_arw_em_real_gnu_release_dm + - build_cmake_arw_em_real_gnu_release - build_cmake_arw_em_real_gnu_debug_dm - paths: - - .workflow/ + - build_cmake_arw_em_em_b_wave_gnu_debug + - name: "CMake Intel Compilation Tests" + id : cmake-intel + args : -vr='{"cpus":128,"timelimit":"00:35:00"}' + actions: + - build_cmake_arw_em_real_intel_release_dm + - build_cmake_arw_em_real_intel_release + - build_cmake_arw_em_real_intel_debug_dm + - build_cmake_arw_em_b_wave_intel_debug + # I am leaving this here for posterity if this is to be replicated in private repositories for testing permissions: contents: read @@ -99,7 +111,7 @@ jobs: sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', description: '${{ matrix.workflow.name }}', - context: '${{ matrix.workflow.host }}/${{ matrix.workflow.id }}', + context: '${{ matrix.base.host }}/${{ matrix.workflow.id }}', state: 'pending' }) @@ -108,16 +120,16 @@ jobs: submodules: true ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} - - uses: ./.github/actions/sane_workflows + - uses: islas/sane-workflows-action@v1.0.0 name: ${{ matrix.workflow.name }} with: # Everything below this should remain the same and comes from the workflow matrix id : ${{ matrix.workflow.id }} - host : ${{ matrix.workflow.host }} + host : ${{ matrix.base.host }} actions : '${{ toJson( matrix.workflow.actions ) }}' args : ${{ matrix.workflow.args }} - patch : '${{ toJson( matrix.workflow.patch ) }}' - paths : '${{ toJson( matrix.workflow.paths ) }}' + patch : '${{ format( toJson( matrix.base.patch ), matrix.workflow.id }}' + paths : '${{ toJson( matrix.base.paths ) }}' upload : true - name: Set completed status @@ -132,12 +144,12 @@ jobs: sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', description: '${{ matrix.workflow.name }}', - context: '${{ matrix.workflow.host }}/${{ matrix.workflow.id }}', + context: '${{ matrix.base.host }}/${{ matrix.workflow.id }}', state: '${{ job.status == 'success' && 'success' || 'failure' }}' }) - - name : Remove '${{ matrix.workflow.label }}' label - if : ${{ !cancelled() && inputs.event_label == matrix.workflow.label }} + - name : Remove '${{ matrix.base.label }}' label + if : ${{ !cancelled() && inputs.event_label == matrix.base.label }} env: PR_NUMBER: ${{ inputs.event_number }} run: | @@ -145,7 +157,7 @@ jobs: -X DELETE \ -H "Accept: application/vnd.github.v3+json" \ -H 'Authorization: token ${{ github.token }}' \ - https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ matrix.workflow.label }} + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ matrix.base.label }} # In the event that 'all-tests' is used, this final job will be the one to remove # the label from the PR From 2e82febc3ff5236b5e2bfd0e05378fe8a785dc2c Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 21:23:27 -0600 Subject: [PATCH 29/38] Try to not use matrix --- .github/workflows/ci_rework.yml | 92 +++++++++++++++------------------ 1 file changed, 41 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index a0e97d0049..54c74d0fad 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -54,42 +54,33 @@ on: jobs: compile-tests: if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }} - name : Run ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }} - runs-on: ${{ matrix.workflow.host }} - strategy: - max-parallel: 4 - fail-fast: false - matrix: - base : - - host : derecho - label: compile-tests - paths: - - .workflow/ - patch : - priority : -99 - hosts : - '[.*derecho.*]' : - account : NMMM0012 - queue : main - job_suffix: .{0}.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }} - workflow: - - name : "CMake GNU Compilation Tests" - id : cmake-gnu - args : -vr='{"cpus":128,"timelimit":"00:10:00"}' - actions : - - build_cmake_arw_em_real_gnu_release_dm - - build_cmake_arw_em_real_gnu_release - - build_cmake_arw_em_real_gnu_debug_dm - - build_cmake_arw_em_em_b_wave_gnu_debug - - name: "CMake Intel Compilation Tests" - id : cmake-intel - args : -vr='{"cpus":128,"timelimit":"00:35:00"}' - actions: - - build_cmake_arw_em_real_intel_release_dm - - build_cmake_arw_em_real_intel_release - - build_cmake_arw_em_real_intel_debug_dm - - build_cmake_arw_em_b_wave_intel_debug - + env: + host: derecho + label: cmake-tests + name: "CMake Compilation Tests" + paths: '[ ./workflow ]' + patch: | + { + 'priority':-99, + 'hosts': + { + '[.*derecho.*]' : + { + 'account' : 'NMMM0012', + 'queue' : 'main', + 'job_suffix' : 'cmake-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}' + } + } + } + args : -vr='{"cpus":128,"timelimit":"00:10:00"}' + actions : | + [ build_cmake_arw_em_real_gnu_release_dm, + build_cmake_arw_em_real_gnu_release, + build_cmake_arw_em_real_gnu_debug_dm, + build_cmake_arw_em_em_b_wave_gnu_debug + ] + name : Run ${{ env.name }} on ${{ env.host }} + runs-on: ${{ env.host }} # I am leaving this here for posterity if this is to be replicated in private repositories for testing permissions: contents: read @@ -110,8 +101,8 @@ jobs: repo: context.repo.repo, sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ matrix.workflow.name }}', - context: '${{ matrix.base.host }}/${{ matrix.workflow.id }}', + description: '${{ env.name }}', + context: '${{ env.host }}/${{ env.label }}', state: 'pending' }) @@ -121,15 +112,14 @@ jobs: ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} - uses: islas/sane-workflows-action@v1.0.0 - name: ${{ matrix.workflow.name }} + name: ${{ env.name }} with: - # Everything below this should remain the same and comes from the workflow matrix - id : ${{ matrix.workflow.id }} - host : ${{ matrix.base.host }} - actions : '${{ toJson( matrix.workflow.actions ) }}' - args : ${{ matrix.workflow.args }} - patch : '${{ format( toJson( matrix.base.patch ), matrix.workflow.id }}' - paths : '${{ toJson( matrix.base.paths ) }}' + id : ${{ env.label }} + host : ${{ env.host }} + actions : ${{ env.actions }} + args : ${{ env.args }} + patch : ${{ env.patch }} + paths : ${{ env.paths ) }} upload : true - name: Set completed status @@ -143,13 +133,13 @@ jobs: repo: context.repo.repo, sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ matrix.workflow.name }}', - context: '${{ matrix.base.host }}/${{ matrix.workflow.id }}', + description: '${{ env.name }}', + context: '${{ env.host }}/${{ env.label }}', state: '${{ job.status == 'success' && 'success' || 'failure' }}' }) - - name : Remove '${{ matrix.base.label }}' label - if : ${{ !cancelled() && inputs.event_label == matrix.base.label }} + - name : Remove '${{ env.label }}' label + if : ${{ !cancelled() && inputs.event_label == env.label }} env: PR_NUMBER: ${{ inputs.event_number }} run: | @@ -157,7 +147,7 @@ jobs: -X DELETE \ -H "Accept: application/vnd.github.v3+json" \ -H 'Authorization: token ${{ github.token }}' \ - https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ matrix.base.label }} + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ env.label }} # In the event that 'all-tests' is used, this final job will be the one to remove # the label from the PR From 48f3b0ad253b82d8f219c51f14bb1af25bf3edcd Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 21:50:36 -0600 Subject: [PATCH 30/38] Back to reusable workflow --- .github/workflows/labeled_test.yml | 147 +++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 .github/workflows/labeled_test.yml diff --git a/.github/workflows/labeled_test.yml b/.github/workflows/labeled_test.yml new file mode 100644 index 0000000000..a07e1aaa59 --- /dev/null +++ b/.github/workflows/labeled_test.yml @@ -0,0 +1,147 @@ +on : + workflow_call : + inputs : + # Label control + label : + required : true + type : string + name : + required : true + type : string + event_name : + required : true + type : string + event_number: + required : true + type : string + event_label: + required : true + type : string + ref : + required : true + type : string + sha : + required : true + type : string + + # SANE Workflow + paths: + required : true + type : string + host: + required : false + type : string + actions: + required : false + type : string + default : '[]' + actions_filter: + required : false + type : string + args : + required : false + type : string + patch: + required : false + type : string + upload : + required : false + type : boolean + cache: + required : false + type : boolean + cache_id: + required : false + type : string + venv: + required : false + type : string + python_dependencies: + required : false + type : string + default : '[]' + upgrade: + required : false + type : boolean + default : true + +jobs: + labeled_test: + # Is 5 days a reasonable wait time for testing? + timeout-minutes: 7200 + name: Run ${{ inputs.name }} on ${{ inputs.host }} + runs-on: ${{ inputs.host }} + steps: + # Don't use gh checks as they are woefully underdeveloped as a feature leading + # to confusing UI and misplaced metrics + # https://github.com/orgs/community/discussions/24616 + - name: Set pending status + id: check_run_start + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', + target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', + description: '${{ inputs.name }}', + context: '${{ inputs.host }}/${{ inputs.label }}', + state: 'pending' + }) + + - uses: actions/checkout@v4 + with: + path : main + submodules: true + ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} + + - uses: islas/sane-workflows-action@v1.0.0 + name: ${{ inputs.name }} + with: + id : ${{ inputs.label }} + paths : ${{ inputs.paths }} + host : ${{ inputs.host }} + actions : ${{ inputs.actions }} + actions_filter : ${{ inputs.actions_filter }} + args : ${{ inputs.args }} + patch : ${{ inputs.patch }} + upload : ${{ inputs.upload }} + cache : ${{ inputs.cache }} + cache_id : ${{ inputs.cache_id }} + venv : ${{ inputs.venv }} + python_dependencies : ${{ inputs.python_dependencies }} + upgrade : ${{ inputs.upgrade }} + + - name: Set completed status + if: ${{ always() }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', + target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', + description: '${{ inputs.name }}', + context: '${{ inputs.host }}/${{ inputs.label }}', + state: '${{ job.status == 'success' && 'success' || 'failure' }}' + }) + + - name : Remove '${{ inputs.label }}' label + if : ${{ !cancelled() && inputs.event_label == inputs.label }} + env: + PR_NUMBER: ${{ inputs.event_number }} + run: | + curl \ + -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ github.token }}' \ + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + + + + + From a34ae699c9ef4da708b2c164a71d8252597f75ac Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 21:50:52 -0600 Subject: [PATCH 31/38] Use simple reusable workflow --- .github/workflows/ci_rework.yml | 90 +++++++-------------------------- 1 file changed, 17 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 54c74d0fad..4224d29783 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -54,11 +54,18 @@ on: jobs: compile-tests: if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }} - env: - host: derecho - label: cmake-tests - name: "CMake Compilation Tests" + # I am leaving this here for posterity if this is to be replicated in private repositories for testing + permissions: + contents: read + pull-requests: write + statuses: write + uses : ./.github/workflows/labeled_test.yml + with : + label : cmake-compile-tests + name : "CMake Compilation Tests" + host : derecho paths: '[ ./workflow ]' + args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch: | { 'priority':-99, @@ -72,82 +79,19 @@ jobs: } } } - args : -vr='{"cpus":128,"timelimit":"00:10:00"}' actions : | [ build_cmake_arw_em_real_gnu_release_dm, build_cmake_arw_em_real_gnu_release, build_cmake_arw_em_real_gnu_debug_dm, build_cmake_arw_em_em_b_wave_gnu_debug ] - name : Run ${{ env.name }} on ${{ env.host }} - runs-on: ${{ env.host }} - # I am leaving this here for posterity if this is to be replicated in private repositories for testing - permissions: - contents: read - pull-requests: write - statuses: write - steps: - # Don't use gh checks as they are woefully underdeveloped as a feature leading - # to confusing UI and misplaced metrics - # https://github.com/orgs/community/discussions/24616 - - name: Set pending status - id: check_run_start - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', - target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ env.name }}', - context: '${{ env.host }}/${{ env.label }}', - state: 'pending' - }) - - - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} - - - uses: islas/sane-workflows-action@v1.0.0 - name: ${{ env.name }} - with: - id : ${{ env.label }} - host : ${{ env.host }} - actions : ${{ env.actions }} - args : ${{ env.args }} - patch : ${{ env.patch }} - paths : ${{ env.paths ) }} - upload : true - - name: Set completed status - if: ${{ always() }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', - target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ env.name }}', - context: '${{ env.host }}/${{ env.label }}', - state: '${{ job.status == 'success' && 'success' || 'failure' }}' - }) - - - name : Remove '${{ env.label }}' label - if : ${{ !cancelled() && inputs.event_label == env.label }} - env: - PR_NUMBER: ${{ inputs.event_number }} - run: | - curl \ - -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - -H 'Authorization: token ${{ github.token }}' \ - https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ env.label }} + # required to do event trigger + event_name : ${{ inputs.event_name }} + event_number : ${{ inputs.event_number }} + event_label : ${{ inputs.test }} + ref : ${{ inputs.ref }} + sha : ${{ inputs.sha }} # In the event that 'all-tests' is used, this final job will be the one to remove # the label from the PR From 6110705254010bec9d1fd607f0c369276376b80a Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:02:00 -0600 Subject: [PATCH 32/38] Fixes, fixes --- .github/workflows/ci_rework.yml | 1 + .github/workflows/labeled_test.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 4224d29783..a12b77a41d 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -60,6 +60,7 @@ jobs: pull-requests: write statuses: write uses : ./.github/workflows/labeled_test.yml + name : CMake Compilation Tests with : label : cmake-compile-tests name : "CMake Compilation Tests" diff --git a/.github/workflows/labeled_test.yml b/.github/workflows/labeled_test.yml index a07e1aaa59..43e0510c02 100644 --- a/.github/workflows/labeled_test.yml +++ b/.github/workflows/labeled_test.yml @@ -56,6 +56,7 @@ on : venv: required : false type : string + default : './venv/sane' python_dependencies: required : false type : string From 44a7908c0dde151ca61aa4883fb6d5ae59b51cac Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:14:38 -0600 Subject: [PATCH 33/38] Wrap in quotes --- .github/workflows/ci_rework.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index a12b77a41d..7222c9089e 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -81,10 +81,10 @@ jobs: } } actions : | - [ build_cmake_arw_em_real_gnu_release_dm, - build_cmake_arw_em_real_gnu_release, - build_cmake_arw_em_real_gnu_debug_dm, - build_cmake_arw_em_em_b_wave_gnu_debug + [ 'build_cmake_arw_em_real_gnu_release_dm', + 'build_cmake_arw_em_real_gnu_release', + 'build_cmake_arw_em_real_gnu_debug_dm', + 'build_cmake_arw_em_em_b_wave_gnu_debug' ] # required to do event trigger From 9b179d2a55ac9630768f21756a3e2e480fec5a52 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:18:44 -0600 Subject: [PATCH 34/38] Wrap in quotes double time --- .github/workflows/ci_rework.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 7222c9089e..a1bd1ea800 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -65,7 +65,8 @@ jobs: label : cmake-compile-tests name : "CMake Compilation Tests" host : derecho - paths: '[ ./workflow ]' + paths : | + [ './workflow' ] args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch: | { From 4210bdcee209c7912663c4fa81a189647460ec67 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:24:44 -0600 Subject: [PATCH 35/38] Wrap in double quotes quadruple time --- .github/workflows/ci_rework.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index a1bd1ea800..81579fc823 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -66,26 +66,26 @@ jobs: name : "CMake Compilation Tests" host : derecho paths : | - [ './workflow' ] + [ "./workflow" ] args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch: | { - 'priority':-99, - 'hosts': + "priority" : -99, + "hosts" : { - '[.*derecho.*]' : + "[.*derecho.*]" : { - 'account' : 'NMMM0012', - 'queue' : 'main', - 'job_suffix' : 'cmake-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}' + "account" : "NMMM0012", + "queue" : "main", + "job_suffix" : "cmake-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}" } } } actions : | - [ 'build_cmake_arw_em_real_gnu_release_dm', - 'build_cmake_arw_em_real_gnu_release', - 'build_cmake_arw_em_real_gnu_debug_dm', - 'build_cmake_arw_em_em_b_wave_gnu_debug' + [ "build_cmake_arw_em_real_gnu_release_dm", + "build_cmake_arw_em_real_gnu_release", + "build_cmake_arw_em_real_gnu_debug_dm", + "build_cmake_arw_em_em_b_wave_gnu_debug" ] # required to do event trigger From b276ed6cf77708dc3e8ddaddc3fe0537bdef715d Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:29:40 -0600 Subject: [PATCH 36/38] Operate in one directory --- .github/workflows/labeled_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/labeled_test.yml b/.github/workflows/labeled_test.yml index 43e0510c02..d977e2622e 100644 --- a/.github/workflows/labeled_test.yml +++ b/.github/workflows/labeled_test.yml @@ -94,7 +94,6 @@ jobs: - uses: actions/checkout@v4 with: - path : main submodules: true ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} From 38b881bd73483ac1298eee5521ff8958b48769c4 Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:36:20 -0600 Subject: [PATCH 37/38] Fix workflow path --- .github/workflows/ci_rework.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index 81579fc823..da4dca6fb4 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -66,7 +66,7 @@ jobs: name : "CMake Compilation Tests" host : derecho paths : | - [ "./workflow" ] + [ ".workflow/" ] args : -vr='{"cpus":128,"timelimit":"00:10:00"}' patch: | { From 9b295d177963711222d2ed88dbee88db9fa931fd Mon Sep 17 00:00:00 2001 From: Anthony Islas Date: Thu, 23 Oct 2025 22:39:37 -0600 Subject: [PATCH 38/38] I thought I fixed this typo... --- .github/workflows/ci_rework.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index da4dca6fb4..5ac90467cd 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -85,7 +85,7 @@ jobs: [ "build_cmake_arw_em_real_gnu_release_dm", "build_cmake_arw_em_real_gnu_release", "build_cmake_arw_em_real_gnu_debug_dm", - "build_cmake_arw_em_em_b_wave_gnu_debug" + "build_cmake_arw_em_b_wave_gnu_debug" ] # required to do event trigger