Gha test

.github/workflows/README.md

@@ -0,0 +1,25 @@
+### Github Actions Workflow
+
+This build file replaces the existing `Jenkins.ci` build process.
+
+`lint.yaml` replaces the `Static code validation` stage of the Jenkins build.
+
+`build.yaml` replaces the `Build / Test` stage of the Jenkins build.
+
+Many of the other stages are replaced merely by the fact we're using Github Actions, we use prebuild Docker containers so we don't have to replicate the steps for building containers.
+
+The `Build result notification` stage was not moved to GHA, build failures will be reports via GHA.
+
+The build process for `Jenkins.nightly` was not ported to GHA.
+
+#### Configuring actions
+
+If you are cloning or forking this repo you will need to configure two secrets for Actions to run correctly.
+
+Secrets can be set via Settings -> Secrets -> New repository secret.
+
+CR_USER is your GH username.
+CR_PAT can be created by following [these directions](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token)
+
+Once you have run the build once with those secrets, you have to make then package public.
+Access the package at https://ghcr.io/USER/indy-node/indy-node-build or https://ghcr.io/USER/indy-node/indy-node-lint then change the visibility in 'Package Settings' to 'Public' then re-run the build.

.github/workflows/build-u18004.yml

@@ -0,0 +1,254 @@
+name: indy-node-build
+on: [ push, pull_request, workflow_dispatch ]
+
+jobs:
+  workflow-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }}
+      CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }}
+      # Expose the lowercase version of the GitHub repository name
+      # to all subsequent jobs that reference image repositories
+      # as the push and pull operations require the URL of the repository 
+      # to be in lowercase.
+      GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Set outputs
+        id: cache
+        run: |
+          echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}"
+          echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}"
+          echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"
+
+  build-lint-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-lint
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_LINT }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-lint.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }}
+  build-test-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-build
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_BUILD }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-build.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }}
+  indy_node:
+    name: Build Indy Node
+    needs: build-test-image
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    strategy:
+      matrix:
+        module: [indy_node, indy_common]
+        slice: [1, 2, 3, 4 ,5, 6, 7,8, 9, 10, 11]
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: pip install .[tests]
+        continue-on-error: true
+
+      - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/${{ strategy.job-total }}
+        run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv --junitxml=test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml" --dir "${{ matrix.module }}" --output "test-result-indy-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}"
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        continue-on-error: true
+        with:
+          check_name: Indy Node ${{ matrix.module }} ${{ matrix.slice }}/${{ strategy.job-total }} Test Report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/adenishchenko/indy-node/indy-node-lint
+    needs: [workflow-setup, build-lint-image]
+    steps:      
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: flake8
+        run: python3 -m flake8
+
+# name: indy-node-build-u2004
+# on: [ push, pull_request ]
+
+# jobs:
+#   workflow-setup:
+#     runs-on: ubuntu-20.04
+#     outputs:
+#       CACHE_KEY_CI_IMAGE: ${{ steps.cache.outputs.CACHE_KEY_CI_IMAGE }}
+#       CACHE_KEY_BUILD_IMAGE: ${{ steps.cache.outputs.CACHE_KEY_BUILD_IMAGE }}
+#       GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }}
+#       DOCKER_FILE_POSTFIX: ${{ steps.cache.outputs.DOCKER_FILE_POSTFIX }}
+#     steps:
+#       - name: Checkout
+#         uses: actions/checkout@v2
+
+#       - name: Set outputs
+#         id: cache
+#         run: |
+#           echo "::set-output name=CACHE_KEY_CI_IMAGE::${{ hashFiles('.github/workflows/test/Dockerfile') }}-rc1"
+#           echo "::set-output name=CACHE_KEY_BUILD_IMAGE::${{ hashFiles('.github/workflows/build/Dockerfile') }}-rc1"
+#           # Convert the GitHub repository name to lowercase
+#           echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"
+#           echo "::set-output name=DOCKER_FILE_POSTFIX::ubuntu-20-04"
+
+
+#   test-image:
+#     # Reference to workflow-setup job is required to access its various outputs.
+#     needs: workflow-setup
+#     runs-on: ubuntu-latest
+#     env:
+#       DOCKER_BUILDKIT: 1
+#       CACHE_KEY_CI_IMAGE: ${{ needs.workflow-setup.outputs.CACHE_KEY_CI_IMAGE }}
+#       GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+#       DOCKER_FILE_POSTFIX: ${{ needs.workflow-setup.outputs.DOCKER_FILE_POSTFIX }}
+#     steps:
+#       - name: Git checkout
+#         uses: actions/checkout@v2
+
+#       - name: Try load from cache.
+#         id: cache-image
+#         uses: actions/cache@v2
+#         with:
+#           path: ${GITHUB_WORKSPACE}/cache
+#           key: ${{ env.CACHE_KEY_CI_IMAGE}}
+
+#       - name: If NOT found in cache, build and push image.
+#         if: steps.cache-image.outputs.cache-hit != 'true'
+#         run: |
+#           echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+#           docker build -f .github/workflows/test/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/node-test:${{ env.CACHE_KEY_CI_IMAGE }} .
+#           docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/node-test:${{ env.CACHE_KEY_CI_IMAGE }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-test:${{ env.DOCKER_FILE_POSTFIX }}
+#           docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-test:${{ env.DOCKER_FILE_POSTFIX }}
+#           mkdir -p ${GITHUB_WORKSPACE}/cache
+#           touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_CI_IMAGE }}
+
+#   build-image:
+#     # Reference to workflow-setup job is required to access its various outputs.
+#     needs: workflow-setup
+#     runs-on: ubuntu-latest
+#     env:
+#       DOCKER_BUILDKIT: 1
+#       CACHE_KEY_BUILD_IMAGE: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD_IMAGE }}
+#       GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+#       DOCKER_FILE_POSTFIX: ${{ needs.workflow-setup.outputs.DOCKER_FILE_POSTFIX }}
+#     steps:
+#       - name: Git checkout
+#         uses: actions/checkout@v2
+
+#       - name: Try load from cache.
+#         id: cache-image
+#         uses: actions/cache@v2
+#         with:
+#           path: ${GITHUB_WORKSPACE}/cache
+#           key: ${{ env.CACHE_KEY_BUILD_IMAGE }}
+
+#       - name: If NOT found in cache, build and push image.
+#         if: steps.cache-image.outputs.cache-hit != 'true'
+#         run: |
+#           echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+#           docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/node-build:${{ env.CACHE_KEY_BUILD_IMAGE }} .
+#           docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/node-build:${{ env.CACHE_KEY_BUILD_IMAGE }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-build:${{ env.DOCKER_FILE_POSTFIX }}
+#           docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/node-build:${{ env.DOCKER_FILE_POSTFIX }}
+#           mkdir -p ${GITHUB_WORKSPACE}/cache
+#           touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD_IMAGE }}
+
+
+
+
+
+
+
+
+#   indy-common:
+#     name: Build Indy Common
+#     runs-on: ubuntu-latest
+#     #TODO: move this to hyperledger?
+#     container: udosson/indy-node-ci-u1804:latest
+#     steps:
+#       - name: Check out code
+#         uses: actions/checkout@v2
+
+#       - name: Install dependencies
+#         run: pip install .[tests]
+#         continue-on-error: true
+
+#       - name: Run Indy Common test
+#         run: python3 -m pytest -l -vv --junitxml=test-result-indy-common.xml --dir indy_common --output "test-result-indy-common.txt"
+
+#   indy-node:
+#     name: Build Indy Node
+#     runs-on: ubuntu-18.04
+#     #TODO: move this to hyperledger?
+#     container: udosson/indy-node-ci-u1804:latest
+#     steps:
+#       - name: Check out code
+#         uses: actions/checkout@v2
+
+#       - name: Install dependencies
+#         run: pip install .[tests]
+#         continue-on-error: true
+
+#       - name: Run Indy Node test
+#         run: python3 -m pytest -l -vv --junitxml=test-result-indy-node.xml --dir indy_node --output "test-result-indy-node.txt"
+
+
+
+#   lint:
+#     name: Lint
+#     runs-on: ubuntu-18.04
+#     #TODO: move this to hyperledger?
+#     container: udosson/indy-node-ci-u1804:latest
+#     steps:
+#       - name: Check out code
+#         uses: actions/checkout@v2
+
+#       - name: flake8
+#         run: python3 -m flake8

.github/workflows/build.yaml

@@ -0,0 +1,117 @@
+name: indy-node-build
+on: [ push, pull_request, workflow_dispatch ]
+
+jobs:
+  workflow-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      CACHE_KEY_LINT: ${{ steps.cache.outputs.CACHE_KEY_LINT }}
+      CACHE_KEY_BUILD: ${{ steps.cache.outputs.CACHE_KEY_BUILD }}
+      # Expose the lowercase version of the GitHub repository name
+      # to all subsequent jobs that reference image repositories
+      # as the push and pull operations require the URL of the repository 
+      # to be in lowercase.
+      GITHUB_REPOSITORY_NAME: ${{ steps.cache.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Set outputs
+        id: cache
+        run: |
+          echo "::set-output name=CACHE_KEY_LINT::${{ hashFiles('.github/workflows/lint/Dockerfile') }}"
+          echo "::set-output name=CACHE_KEY_BUILD::${{ hashFiles('.github/workflows/build/Dockerfile') }}"
+          echo "::set-output name=GITHUB_REPOSITORY_NAME::$(echo ${GITHUB_REPOSITORY,,})"
+
+  build-lint-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_LINT: ${{ needs.workflow-setup.outputs.CACHE_KEY_LINT }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-lint
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_LINT }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-lint.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/lint/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:${{ env.CACHE_KEY_LINT }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-lint:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_LINT }}
+
+  build-test-image:
+    needs: workflow-setup
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: 1
+      CACHE_KEY_BUILD: ${{ needs.workflow-setup.outputs.CACHE_KEY_BUILD }}
+      GITHUB_REPOSITORY_NAME: ${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Try load from cache.
+        id: cache-image-build
+        uses: actions/cache@v2
+        with:
+          path: ${GITHUB_WORKSPACE}/cache
+          key: ${{ env.CACHE_KEY_BUILD }}
+      - name: If NOT found in cache, build and push image.
+        if: steps.cache-image-build.outputs.cache-hit != 'true'
+        run: |
+          echo ${{ secrets.CR_PAT }} | docker login ghcr.io --username ${{ secrets.CR_USER }} --password-stdin
+          docker build -f .github/workflows/build/Dockerfile --no-cache -t ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} .
+          docker tag ${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:${{ env.CACHE_KEY_BUILD }} ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          docker push ghcr.io/${{ env.GITHUB_REPOSITORY_NAME }}/indy-node-build:latest
+          mkdir -p ${GITHUB_WORKSPACE}/cache
+          touch ${GITHUB_WORKSPACE}/cache/${{ env.CACHE_KEY_BUILD }}
+
+  indy_node:
+    name: Build Indy Node
+    needs: build-test-image
+    runs-on: ubuntu-18.04
+    container:
+      image: ghcr.io/${{ github.repository }}/indy-node-build
+    strategy:
+      matrix:
+        module: [indy_node, indy_common]
+        slice: [1, 2, 3, 4 ,5, 6, 7,8, 9, 10, 11]
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: pip install .[tests]
+        continue-on-error: true
+
+      - name: Run Indy Node ${{ matrix.module }} test slice ${{ matrix.slice }}/${{ strategy.job-total }}
+        run: RUSTPYTHONASYNCIODEBUG=0 python3 runner.py --pytest "python3 -m pytest -l -vv --junitxml=test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml" --dir "${{ matrix.module }}" --output "test-result-indy-node-${{ matrix.slice }}.txt" --test-only-slice "${{ matrix.slice }}/${{ strategy.job-total }}"
+
+      - name: Publish Test Report
+        uses: scacap/action-surefire-report@v1
+        with:
+          check_name: Indy Node ${{ matrix.module }} ${{ matrix.slice }}/${{ strategy.job-total }} Test Report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: test-result-indy-node-${{ matrix.module }}-${{ matrix.slice }}.xml
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/${{ needs.workflow-setup.outputs.GITHUB_REPOSITORY_NAME }}/indy-node-lint
+    needs: [workflow-setup, build-lint-image]
+    steps:      
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: flake8
+        run: python3 -m flake8