Coverage script #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

payalcha wants to merge 16 commits into develop from coverage-report

.coverage

Binary file not shown.

.github/actions/tr_post_test_run/action.yml

-Original file line number
+Diff line change
@@ Expand Up / @@ -24,7 +24,7 @@ runs: @@
           id: tar_files
           if: ${{ always() }}
           run: |
-            tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" --exclude="tensor.db" --exclude="workspace.tar" $HOME/results
+            tar -cvf result.tar --exclude="cert" --exclude="data" --exclude="__pycache__" --exclude="tensor.db" --exclude="workspace.tar" --exclude="minio_data" $HOME/results
             # Model name might contain forward slashes, convert them to underscore.
             tmp=${{ env.MODEL_NAME }}
             echo "MODEL_NAME_MODIFIED=${tmp//\//_}" >> $GITHUB_ENV
@@ Expand Down @@

.github/scripts/.coverage

Binary file not shown.

.github/scripts/coverage-report.sh

-Original file line number
+Diff line change
@@ -0,0 +1,60 @@
+    #!/bin/bash
+    set -Eeuo pipefail
+    rm -rf build
+    pip uninstall openfl -y
+    pip install -e .
+    pip install -r test-requirements.txt
+    pip install -r openfl-tutorials/experimental/workflow/workflow_interface_requirements.txt
+    pip install coverage
+    pip install pytest-cov
+    fx experimental deactivate
+    rm -rf .coverage
+    python -m pytest -rA --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_via_native --model_name torch/mnist --num_rounds 2 --disable_client_auth --secure_agg --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_via_native --model_name keras/jax/mnist --num_rounds 2 --disable_tls --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py -k test_log_memory_usage_basic --model_name torch/histology --num_rounds 2 --log_memory_usage --secure_agg --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/tr_resiliency_tests.py --model_name keras/torch/mnist --num_rounds 25 --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/tr_flower_tests.py -k test_flower_app_pytorch_native --model_name flower-app-pytorch --num_rounds 1 --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m task_runner_dockerized_ws --num_rounds 2 --model_name keras/torch/mnist --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/tr_with_fedeval_tests.py -m task_runner_basic --model_name xgb_higgs  --num_rounds 2 --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/wf_local_func_tests.py --num_rounds 2 --cov-report=term-missing --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/wf_local_func_tests.py --workflow_backend ray --num_rounds 2 --cov-report=term-missing --cov-append --cov=openfl
+    fx experimental activate
+    python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_runtime_301_watermarking --cov-report=term-missing  --cov-append --cov=openfl
+    python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_runtime_secure_aggregation --cov-report=term-missing  --cov-append --cov=openfl
+    # python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_evaluation --cov-report=term-missing  --cov-append --cov=openfl
+    fx experimental deactivate
+    # Combine and generate the final coverage report
+    coverage report

.github/workflows/pq_pipeline.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -80,7 +80,7 @@ jobs:
  
          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||

          (github.event_name == 'workflow_dispatch')

        name: TaskRunner E2E

        needs: set_commit_id_for_all_jobs

        needs: task_runner_connectivity_e2e

        uses: ./.github/workflows/task_runner_basic_e2e.yml

        with:

          commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

    @@ -90,7 +90,7 @@ jobs:
  
          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||

          (github.event_name == 'workflow_dispatch')

        name: TaskRunner Resiliency E2E

        needs: task_runner_e2e

        needs: task_runner_connectivity_e2e

        uses: ./.github/workflows/task_runner_resiliency_e2e.yml

        with:

          commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

    @@ -158,6 +158,26 @@ jobs:
  
        with:

          commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

      task_runner_fed_analytics_e2e:

        if: |

          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||

          (github.event_name == 'workflow_dispatch')

        name: TaskRunner Federated Analytics E2E

        needs: task_runner_connectivity_e2e

        uses: ./.github/workflows/task_runner_fed_analytics_e2e.yml

        with:

          commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

      tr_verifiable_dataset_e2e:

        if: |

          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||

          (github.event_name == 'workflow_dispatch')

        name: TaskRunner Verifiable Dataset E2E

        needs: task_runner_e2e

        uses: ./.github/workflows/tr_verifiable_dataset_e2e.yml

        with:

          commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

      run_trivy:

        if: |

          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||

    @@ -198,14 +218,17 @@ jobs:
  
          wf_mnist_local_runtime,

          wf_watermark_e2e,

          wf_secagg_e2e,

          task_runner_connectivity_e2e,

          task_runner_e2e,

          task_runner_fed_analytics_e2e,

          task_runner_resiliency_e2e,

          task_runner_fedeval_e2e,

          task_runner_secure_agg_e2e,

          task_runner_straggler_e2e,

          task_runner_dockerized_e2e,

          task_runner_secret_ssl_e2e,

          task_runner_flower_app_pytorch,

          tr_verifiable_dataset_e2e,

          run_trivy,

          run_bandit

        ]

.github/workflows/task_runner_fed_analytics_e2e.yml

-Original file line number
+Diff line change
@@ -0,0 +1,108 @@
+    ---
+    # Task Runner Federated Analytics E2E tests for bare metal approach
+    name: Task_Runner_Fed_Analytics_E2E  # Please do not modify the name as it is used in the composite action
+    on:
+      workflow_call:
+        inputs:
+          commit_id:
+            required: false
+            type: string
+      workflow_dispatch:
+        inputs:
+          num_collaborators:
+            description: "Number of collaborators"
+            required: false
+            default: "2"
+            type: string
+          python_version:
+            description: "Python version"
+            required: false
+            default: "3.10"
+            type: choice
+            options:
+              - "3.10"
+              - "3.11"
+              - "3.12"
+    permissions:
+      contents: read
+    # Environment variables common for all the jobs
+    # DO NOT use double quotes for the values of the environment variables
+    env:
+      NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }}
+      COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
+    jobs:
+      test_fed_analytics_histogram:
+        name: With TLS (federated_analytics/histogram, 3.11) # DO NOT change this name.
+        runs-on: ubuntu-22.04
+        timeout-minutes: 30
+        if: |
+          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+          (github.event_name == 'workflow_dispatch') ||
+          (github.event.pull_request.draft == false)
+        env:
+          MODEL_NAME: 'federated_analytics/histogram'
+          PYTHON_VERSION: ${{ inputs.python_version || '3.11' }}
+        steps:
+          - name: Checkout OpenFL repository
+            id: checkout_openfl
+            uses: actions/checkout@v4
+            with:
+              ref: ${{ env.COMMIT_ID }}
+          - name: Pre test run
+            uses: ./.github/actions/tr_pre_test_run
+            if: ${{ always() }}
+          - name: Run Federated Analytics Histogram
+            id: run_tests
+            run: |
+              python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
+              -m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
+              echo "Federated analytics histogram test run completed"
+          - name: Post test run
+            uses: ./.github/actions/tr_post_test_run
+            if: ${{ always() }}
+            with:
+              test_type: "Sepal_Histogram_Analytics"
+      test_fed_analytics_smokers_health:
+        name: With TLS (federated_analytics/smokers_health, 3.12) # DO NOT change this name.
+        runs-on: ubuntu-22.04
+        timeout-minutes: 30
+        if: |
+          (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
+          (github.event_name == 'workflow_dispatch') ||
+          (github.event.pull_request.draft == false)
+        env:
+          MODEL_NAME: 'federated_analytics/smokers_health'
+          PYTHON_VERSION: ${{ inputs.python_version || '3.12' }}
+        steps:
+          - name: Checkout OpenFL repository
+            id: checkout_openfl
+            uses: actions/checkout@v4
+            with:
+              ref: ${{ env.COMMIT_ID }}
+          - name: Pre test run
+            uses: ./.github/actions/tr_pre_test_run
+            if: ${{ always() }}
+          - name: Run Federated Analytics Smokers Health
+            id: run_tests
+            run: |
+              python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
+              -m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
+              echo "Federated analytics smokers health test run completed"
+          - name: Post test run
+            uses: ./.github/actions/tr_post_test_run
+            if: ${{ always() }}
+            with:
+              test_type: "Smokers_Health_Analytics"

.github/workflows/tr_verifiable_dataset_e2e.yml

-Original file line number
+Diff line change
@@ -0,0 +1,81 @@
+    ---
+    # Task Runner Verifiable Dataset E2E
+    name: TR_Verifiable_Dataset_E2E  # Please do not modify the name as it is used in the composite action
+    on:
+      workflow_call:
+        inputs:
+          commit_id:
+            required: false
+            type: string
+      workflow_dispatch:
+        inputs:
+          num_rounds:
+            description: "Number of rounds to train"
+            required: false
+            default: "2"
+            type: string
+          num_collaborators:
+            description: "Number of collaborators"
+            required: false
+            default: "2"
+            type: string
+    permissions:
+      contents: read
+    # Environment variables common for all the jobs
+    # DO NOT use double quotes for the values of the environment variables
+    env:
+      NUM_ROUNDS: ${{ inputs.num_rounds || 2 }}
+      NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }}
+      COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
+    jobs:
+      test_with_s3: # Run it only if the runner machine has enough memory and CPU
+        name: With S3 (torch/histology_s3, 3.11)
+        runs-on: ubuntu-22.04
+        timeout-minutes: 120
+        env:
+          MODEL_NAME: "torch/histology_s3"
+          PYTHON_VERSION: "3.11"
+        steps:
+          - name: Checkout OpenFL repository
+            id: checkout_openfl
+            uses: actions/checkout@v4
+            with:
+              ref: ${{ env.COMMIT_ID }}
+          - name: Pre test run
+            uses: ./.github/actions/tr_pre_test_run
+            if: ${{ always() }}
+          - name: Install MinIO
+            id: install_minio
+            run: |
+              wget https://dl.min.io/server/minio/release/linux-amd64/minio
+              chmod +x minio
+              sudo mv minio /usr/local/bin/
+          - name: Install MinIO Client
+            id: install_minio_client
+            run: |
+              wget https://dl.min.io/client/mc/release/linux-amd64/mc
+              chmod +x mc
+              sudo mv mc /usr/local/bin/
+          - name: Run Task Runner E2E tests with S3
+            id: run_tests
+            run: |
+              python -m pytest -s tests/end_to_end/test_suites/tr_verifiable_dataset_tests.py \
+              -m task_runner_with_s3 --model_name ${{ env.MODEL_NAME }} \
+              --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
+              echo "Task Runner E2E tests with S3 run completed"
+          - name: Post test run
+            uses: ./.github/actions/tr_post_test_run
+            if: ${{ always() }}
+            with:
+              test_type: "With_S3"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Coverage script #14

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

Coverage script #14

Are you sure you want to change the base?

Uh oh!

Coverage script #14

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!