workflows: add ubuntu-latest to beam_PreCommit_Python_ML, beam_PostCommit_Python, and beam_PreCommit_Python_Coverage

.github/workflows/beam_PostCommit_Python.yml

@@ -53,21 +53,15 @@ env:
 
 jobs:
   beam_PostCommit_Python:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 240
     strategy:
       fail-fast: false
       matrix:
-        job_name: [beam_PostCommit_Python]
-        job_phrase: [Run Python PostCommit]
+        job_name: ['beam_PostCommit_Python']
+        job_phrase: ['Run Python PostCommit']
         python_version: ['3.9', '3.10', '3.11', '3.12']
-        # Run on both self-hosted and GitHub-hosted runners.
-        # Some tests (marked require_docker_in_docker) can't run on Beam's
-        # self-hosted runners due to Docker-in-Docker environment constraint.
-        # These tests will only execute on ubuntu-latest (GitHub-hosted).
-        # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
         os: [[self-hosted, ubuntu-20.04, highmem22]]
     if: |
       github.event_name == 'workflow_dispatch' ||
@@ -81,7 +75,7 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -106,19 +100,15 @@ jobs:
           arguments: |
             -Pjava21Home=$JAVA_HOME_21_X64 \
             -PuseWheelDistribution \
-            -Pposargs="${{
-              contains(matrix.os, 'self-hosted') &&
-                '-m (not require_docker_in_docker)' ||
-                '-m require_docker_in_docker'
-            }}" \
+            -Pposargs="-m (not require_docker_in_docker)" \
             -PpythonVersion=${{ matrix.python_version }} \
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results
+          name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -128,3 +118,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
+          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

.github/workflows/beam_PreCommit_Python_Coverage.yml

@@ -71,8 +71,7 @@ jobs:
         # self-hosted runners due to Docker-in-Docker environment constraint.
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
         # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
-        os: [[self-hosted, ubuntu-20.04, highmem]]
+        os: [[self-hosted, ubuntu-20.04, highmem], [ubuntu-latest]]
     timeout-minutes: 180
     if: |
       github.event_name == 'push' ||

.github/workflows/beam_PreCommit_Python_ML.yml

@@ -57,7 +57,7 @@ env:
 
 jobs:
   beam_PreCommit_Python_ML:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
     strategy:
@@ -70,9 +70,18 @@ jobs:
         # Some tests (marked require_docker_in_docker) can't run on Beam's
         # self-hosted runners due to Docker-in-Docker environment constraint.
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
-        # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
-        os: [[self-hosted, ubuntu-20.04, main]]
+        # Context: https://github.com/apache/beam/pull/35585.
+        os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
+        exclude:
+          # Temporary exclude Python 3.9, 3.10, 3.11 from ubuntu-latest. This
+          # results in pip dependency resolution exceeded maximum depth issue.
+          # Context: https://github.com/apache/beam/pull/35816.
+          - python_version: '3.9'
+            os: [ubuntu-latest]
+          - python_version: '3.10'
+            os: [ubuntu-latest]
+          - python_version: '3.11'
+            os: [ubuntu-latest]
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
@@ -86,7 +95,7 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -113,7 +122,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results
+          name: Python ${{ matrix.python_version }} Test Results ${{ matrix.os }}
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -123,3 +132,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
+          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py

@@ -574,7 +574,7 @@ def test_empty_input_chunks(self):
 
     expected_chunks = []
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -702,7 +702,7 @@ def test_filtered_search_with_cosine_similarity_and_batching(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -807,7 +807,7 @@ def test_filtered_search_with_bm25_full_text_and_batching(self):
             embedding=Embedding())
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -948,7 +948,7 @@ def test_vector_search_with_euclidean_distance(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1088,7 +1088,7 @@ def test_vector_search_with_inner_product_similarity(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1153,7 +1153,7 @@ def test_keyword_search_with_inner_product_sparse_embedding(self):
                 sparse_embedding=([1, 2, 3, 4], [0.05, 0.41, 0.05, 0.41])))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1226,7 +1226,7 @@ def test_hybrid_search(self):
             embedding=Embedding(dense_embedding=[0.1, 0.2, 0.3]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,

sdks/python/test-suites/direct/common.gradle

@@ -419,33 +419,6 @@ task feastIntegrationTest {
     }
 }
 
-// Integration tests that require Docker-in-Docker capabilities.
-// These tests are marked with the `require_docker_in_docker` pytest marker
-// because they rely on Docker-in-Docker configurations that are not supported
-// on Beam's self-hosted GitHub Actions runners. Docker-in-Docker works on
-// ubuntu-latest GitHub-hosted runners but not on self-hosted environments due
-// to containerization architecture differences.
-// Context: https://github.com/apache/beam/pull/35585
-task dockerInDockerIntegrationTest {
-  dependsOn 'installGcpTest'
-  dependsOn ':sdks:python:sdist'
-
-  doLast {
-      def testOpts = basicTestOpts
-      def argMap = [
-          "test_opts": testOpts,
-          "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
-          "collect": "require_docker_in_docker",
-          "runner": "TestDirectRunner",
-      ]
-      def cmdArgs = mapToArgString(argMap)
-      exec {
-        executable 'sh'
-        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
-      }
-    }
-}
-
 // Add all the RunInference framework IT tests to this gradle task that runs on Direct Runner Post commit suite.
 project.tasks.register("inferencePostCommitIT") {
   dependsOn = [
@@ -456,7 +429,6 @@ project.tasks.register("inferencePostCommitIT") {
   'transformersInferenceTest',
   'testcontainersTest',
   'feastIntegrationTest',
-  'dockerInDockerIntegrationTest',
   // (TODO) https://github.com/apache/beam/issues/25799
    // uncomment tfx bsl tests once tfx supports protobuf 4.x
   // 'tfxInferenceTest',