[wip] update coverage details to improve reporting

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -127,10 +127,22 @@ jobs:
                 '-m (not require_docker_in_docker)' ||
                 '-m require_docker_in_docker'
             }}"
+      - name: Find Coverage Report
+        id: find-coverage
+        run: |
+          # Locate the actual coverage.xml, ignoring JUnit XMLs
+          REPORT=$(find sdks/python -name "coverage.xml" | grep -v "pytest_" | head -n 1)
+          if [ -z "$REPORT" ]; then
+            echo "No coverage.xml found"
+            exit 1
+          fi
+          echo "REPORT_PATH=$REPORT" >> $GITHUB_OUTPUT
       - uses: codecov/codecov-action@v3
+        if: steps.find-coverage.outputs.REPORT_PATH != ''
         with:
           flags: python
-          directory: sdks/python
+          file: ${{ steps.find-coverage.outputs.REPORT_PATH }}
+          fail_ci_if_error: true
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v4
         if: failure()

sdks/python/.coveragerc

@@ -16,4 +16,11 @@
 #
 
 [run]
-omit = target/*
+parallel = True
+omit = target/*
+
+[paths]
+source =
+    apache_beam
+    **/site-packages/apache_beam
+    **/build/srcs/sdks/python/apache_beam

sdks/python/scripts/run_pytest.sh

@@ -25,7 +25,6 @@
 # $2 - additional arguments not parsed by tox (typically module names or
 #   '-k keyword')
 # $3 - optional arguments to pytest
-#!/bin/bash
 
 envname=${1?First argument required: suite base name}
 posargs=$2
@@ -134,22 +133,47 @@ echo "Running parallel tests with: pytest -m \"$marker_for_parallel_tests\" $pyt
 pytest -v -rs -o junit_suite_name=${envname} \
   --junitxml=pytest_${envname}.xml -m "$marker_for_parallel_tests" -n 6 --import-mode=importlib ${pytest_args} ${pytest_command_args}
 status1=$?
+echo "Parallel tests finished with status $status1"
 
 # Run tests sequentially.
 echo "Running sequential tests with: pytest -m \"$marker_for_sequential_tests\" $pytest_command_args"
 pytest -v -rs -o junit_suite_name=${envname}_no_xdist \
   --junitxml=pytest_${envname}_no_xdist.xml -m "$marker_for_sequential_tests" --import-mode=importlib ${pytest_args} ${pytest_command_args}
 status2=$?
+echo "Sequential tests finished with status $status2"
 
 # Exit with error if no tests were run in either suite (status code 5).
 if [[ $status1 == 5 && $status2 == 5 ]]; then
   exit $status1
 fi
 
-# Exit with error if one of the statuses has an error that's not 5.
+# Combine coverage data if parallel was used.
+# Combine coverage data if parallel was used.
+# If .coveragerc is not found locally, look for it in the directory of the script or sdk root.
+COVERAGERC_PATH=".coveragerc"
+if [ ! -f "$COVERAGERC_PATH" ]; then
+  COVERAGERC_PATH="$(dirname "$0")/../.coveragerc"
+fi
+
+if [ -f "$COVERAGERC_PATH" ] && grep -q "parallel = True" "$COVERAGERC_PATH"; then
+  echo "Combining coverage data from $COVERAGERC_PATH..."
+  # coverage combine will use the .coveragerc to find data files.
+  coverage combine --rcfile="$COVERAGERC_PATH"
+  if [[ $pytest_args == *"--cov-report=xml"* ]]; then
+    echo "Generating XML report..."
+    coverage xml --rcfile="$COVERAGERC_PATH"
+  fi
+fi
+
+# Exit with error if any of the statuses has an error that's not 5.
 if [[ $status1 != 0 && $status1 != 5 ]]; then
+  echo "Exiting with status1: $status1"
   exit $status1
 fi
 if [[ $status2 != 0 && $status2 != 5 ]]; then
+  echo "Exiting with status2: $status2"
   exit $status2
 fi
+
+echo "Exiting with success"
+exit 0