From 082873ad91ca0e432b975ce737dab69830582dca Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Thu, 7 Aug 2025 19:34:32 +0000
Subject: [PATCH 01/15] workflows: add `ubuntu-latest` to postcommit/ml

---
 .github/workflows/beam_PostCommit_Python.yml   | 2 +-
 .github/workflows/beam_PreCommit_Python_ML.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml
index fef02dc8f92f..c883697d52cf 100644
--- a/.github/workflows/beam_PostCommit_Python.yml
+++ b/.github/workflows/beam_PostCommit_Python.yml
@@ -68,7 +68,7 @@ jobs:
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
         # Context: https://github.com/apache/beam/pull/35585
         # Temporary removed the ubuntu-latest env till resolving deps issues.
-        os: [[self-hosted, ubuntu-20.04, highmem22]]
+        os: [[self-hosted, ubuntu-20.04, highmem22], [ubuntu-latest]]
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'pull_request_target' ||
diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml
index de920428a24b..60fffc3a2a43 100644
--- a/.github/workflows/beam_PreCommit_Python_ML.yml
+++ b/.github/workflows/beam_PreCommit_Python_ML.yml
@@ -72,7 +72,7 @@ jobs:
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
         # Context: https://github.com/apache/beam/pull/35585
         # Temporary removed the ubuntu-latest env till resolving deps issues.
-        os: [[self-hosted, ubuntu-20.04, main]]
+        os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||

From bd56d2c3c6decdeb048b8cb3167d6d92683fbcee Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Fri, 15 Aug 2025 22:43:18 +0000
Subject: [PATCH 02/15] workflows: include os runner as part of job name and
 test results

---
 .github/workflows/beam_PostCommit_Python.yml   | 5 +++--
 .github/workflows/beam_PreCommit_Python_ML.yml | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml
index c883697d52cf..f36a8a4eaf00 100644
--- a/.github/workflows/beam_PostCommit_Python.yml
+++ b/.github/workflows/beam_PostCommit_Python.yml
@@ -53,7 +53,7 @@ env:
 
 jobs:
   beam_PostCommit_Python:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) [${{ matrix.os }}]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 240
     strategy:
@@ -118,7 +118,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results
+          name: Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -128,3 +128,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
+          check_name: 'Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]'
diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml
index 60fffc3a2a43..4dce59d64495 100644
--- a/.github/workflows/beam_PreCommit_Python_ML.yml
+++ b/.github/workflows/beam_PreCommit_Python_ML.yml
@@ -57,7 +57,7 @@ env:
 
 jobs:
   beam_PreCommit_Python_ML:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) [${{ matrix.os }}]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
     strategy:
@@ -113,7 +113,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results
+          name: Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -123,3 +123,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
+          check_name: 'Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]'

From 475289f86bf4cb313f59f2d009d9ce4d56be0a3f Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sat, 16 Aug 2025 12:46:54 +0000
Subject: [PATCH 03/15] sdks/python: properly make milvus as extra dependency

---
 sdks/python/setup.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index f7cd2a808a9a..382f7c263741 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -160,6 +160,8 @@ def cythonize(*args, **kwargs):
     'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3',
 ]
 
+milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']
+
 
 def find_by_ext(root_dir, ext):
   for root, _, files in os.walk(root_dir):
@@ -444,9 +446,8 @@ def get_portability_package_data():
               'mysql-connector-python>=9.3.0',
               'python-tds>=1.16.1',
               'sqlalchemy-pytds>=1.0.2',
-              'oracledb>=3.1.1',
-              'milvus'
-          ],
+              'oracledb>=3.1.1'
+          ] + milvus_dependency,
           'gcp': [
               'cachetools>=3.1.0,<7',
               'google-api-core>=2.0.0,<3',
@@ -589,7 +590,7 @@ def get_portability_package_data():
           ],
           'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'],
           'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'],
-          'milvus': ['pymilvus>=2.5.10,<3.0.0']
+          'milvus': milvus_dependency
       },
       zip_safe=False,
       # PyPI package information.

From ad4cc0c4ef92ae73a298b134c671d3920d31f403 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sat, 16 Aug 2025 14:00:42 +0000
Subject: [PATCH 04/15] sdks/python: fix linting issues

---
 sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
index 58df738c6e5f..6782472d9611 100644
--- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
+++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
@@ -27,16 +27,15 @@
 from typing import Union
 
 from google.protobuf.json_format import MessageToDict
-
-from apache_beam.ml.rag.types import Chunk
-from apache_beam.ml.rag.types import Embedding
-from apache_beam.transforms.enrichment import EnrichmentSourceHandler
 from pymilvus import AnnSearchRequest
 from pymilvus import Hit
 from pymilvus import Hits
 from pymilvus import MilvusClient
 from pymilvus import SearchResult
 
+from apache_beam.ml.rag.types import Chunk
+from apache_beam.ml.rag.types import Embedding
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
 
 class SearchStrategy(Enum):
   """Search strategies for information retrieval.

From 7f3b197f556cc8c881c4d7cc229186bd6d1390d2 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sat, 16 Aug 2025 14:24:08 +0000
Subject: [PATCH 05/15] sdks/python: fix formatting issues

---
 sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
index 6782472d9611..a0f597f5366f 100644
--- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
+++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py
@@ -37,6 +37,7 @@
 from apache_beam.ml.rag.types import Embedding
 from apache_beam.transforms.enrichment import EnrichmentSourceHandler
 
+
 class SearchStrategy(Enum):
   """Search strategies for information retrieval.
 

From 76c77df27ef9fec882baebc4c73ea0875c05d600 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sat, 16 Aug 2025 14:24:37 +0000
Subject: [PATCH 06/15] .github: trigger postcommit python

---
 .github/trigger_files/beam_PostCommit_Python.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
index c6ec17f48412..393a34b8da4e 100644
--- a/.github/trigger_files/beam_PostCommit_Python.json
+++ b/.github/trigger_files/beam_PostCommit_Python.json
@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 35
+  "modification": 350
 }
 

From ea7c8866bd98f3b5e07d101bd175f6c91bd580c6 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Tue, 19 Aug 2025 18:51:19 +0000
Subject: [PATCH 07/15] workflows: properly show job names

---
 .github/workflows/beam_PostCommit_Python.yml   | 15 +++++++--------
 .github/workflows/beam_PreCommit_Python_ML.yml | 11 +++++------
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml
index f36a8a4eaf00..69d66531de03 100644
--- a/.github/workflows/beam_PostCommit_Python.yml
+++ b/.github/workflows/beam_PostCommit_Python.yml
@@ -53,21 +53,20 @@ env:
 
 jobs:
   beam_PostCommit_Python:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) [${{ matrix.os }}]
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 240
     strategy:
       fail-fast: false
       matrix:
-        job_name: [beam_PostCommit_Python]
-        job_phrase: [Run Python PostCommit]
+        job_name: ['beam_PostCommit_Python']
+        job_phrase: ['Run Python PostCommit']
         python_version: ['3.9', '3.10', '3.11', '3.12']
         # Run on both self-hosted and GitHub-hosted runners.
         # Some tests (marked require_docker_in_docker) can't run on Beam's
         # self-hosted runners due to Docker-in-Docker environment constraint.
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
-        # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
+        # Context: https://github.com/apache/beam/pull/35585.
         os: [[self-hosted, ubuntu-20.04, highmem22], [ubuntu-latest]]
     if: |
       github.event_name == 'workflow_dispatch' ||
@@ -81,7 +80,7 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -118,7 +117,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]
+          name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -128,4 +127,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
-          check_name: 'Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]'
+          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"
diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml
index 4dce59d64495..e2d4ccafae5f 100644
--- a/.github/workflows/beam_PreCommit_Python_ML.yml
+++ b/.github/workflows/beam_PreCommit_Python_ML.yml
@@ -57,7 +57,7 @@ env:
 
 jobs:
   beam_PreCommit_Python_ML:
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) [${{ matrix.os }}]
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
     strategy:
@@ -70,8 +70,7 @@ jobs:
         # Some tests (marked require_docker_in_docker) can't run on Beam's
         # self-hosted runners due to Docker-in-Docker environment constraint.
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
-        # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
+        # Context: https://github.com/apache/beam/pull/35585.
         os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
     if: |
       github.event_name == 'push' ||
@@ -86,7 +85,7 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
@@ -113,7 +112,7 @@ jobs:
         uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]
+          name: Python ${{ matrix.python_version }} Test Results ${{ matrix.os }}
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -123,4 +122,4 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/pytest*.xml'
           large_files: true
-          check_name: 'Python ${{ matrix.python_version }} Test Results [${{ matrix.os }}]'
+          check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

From fdabfdc81d3c677cb78c613e7b350935a441bb20 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Tue, 19 Aug 2025 20:29:35 +0000
Subject: [PATCH 08/15] .github: trigger beam postcommit python

---
 .github/trigger_files/beam_PostCommit_Python.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json
index 393a34b8da4e..ed56f65ef50f 100644
--- a/.github/trigger_files/beam_PostCommit_Python.json
+++ b/.github/trigger_files/beam_PostCommit_Python.json
@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 350
+  "modification": 32
 }
 

From ab93ebb2a113eed273fd8452d9a0739b62a57817 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Thu, 28 Aug 2025 17:49:52 +0000
Subject: [PATCH 09/15] workflows: add `ubuntu-latest` to
 `beam_PreCommit_Python_Coverage`

---
 .github/workflows/beam_PreCommit_Python_Coverage.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml
index 3da51a2eceda..7c675c01183b 100644
--- a/.github/workflows/beam_PreCommit_Python_Coverage.yml
+++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml
@@ -71,8 +71,7 @@ jobs:
         # self-hosted runners due to Docker-in-Docker environment constraint.
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
         # Context: https://github.com/apache/beam/pull/35585
-        # Temporary removed the ubuntu-latest env till resolving deps issues.
-        os: [[self-hosted, ubuntu-20.04, highmem]]
+        os: [[self-hosted, ubuntu-20.04, highmem], [ubuntu-latest]]
     timeout-minutes: 180
     if: |
       github.event_name == 'push' ||
@@ -112,7 +111,7 @@ jobs:
           TESTCONTAINERS_HOST_OVERRIDE: ${{ contains(matrix.os, 'self-hosted') && env.DIND_IP || '' }}
           TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/docker.sock"
           TESTCONTAINERS_RYUK_DISABLED: "false"
-          TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true"          
+          TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true"
           PYTEST_ADDOPTS: "-v --tb=short --maxfail=3 --durations=20 --reruns=2 --reruns-delay=5"
           TC_TIMEOUT: "120"
           TC_MAX_TRIES: "120"

From c8d88b02467f99ecbe31a56b88db510d81823c4a Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Fri, 29 Aug 2025 18:33:02 +0000
Subject: [PATCH 10/15] actions+sdks/python: upgrade pip

---
 .github/actions/setup-environment-action/action.yml | 7 +++++++
 sdks/python/tox.ini                                 | 2 --
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml
index d5f1f879a072..6525103f83a5 100644
--- a/.github/actions/setup-environment-action/action.yml
+++ b/.github/actions/setup-environment-action/action.yml
@@ -56,6 +56,13 @@ runs:
           sdks/python/setup.py
           sdks/python/tox.ini
 
+    - name: Upgrade pip to latest version
+      if: ${{ inputs.python-version != '' }}
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip --version
+
     - name: Cache tox environments
       if: ${{ inputs.python-version != '' && inputs.tox-cache == 'true' }}
       uses: actions/cache@v3
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 157dfedecc47..3ac91bdb4c93 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -106,7 +106,6 @@ commands =
 [testenv:py{39,310,311}-ml]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 deps =
-  pip==25.0.1
   accelerate>=1.6.0
 setenv =
 extras = test,gcp,dataframe,ml_test
@@ -516,7 +515,6 @@ deps =
   latest: accelerate>=1.6.0
   tensorflow==2.12.0
   protobuf==4.25.5
-  pip==25.0.1
 extras = test,gcp,ml_test
 commands =
   # Log transformers and its dependencies version for debugging

From 8e37726cb24a231b8b9c5fe217a64554ffc2a031 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Fri, 5 Sep 2025 13:59:00 +0000
Subject: [PATCH 11/15] workflows: exclude `ubuntu-latest` from
 `beam_PostCommit_Python`

---
 .github/workflows/beam_PostCommit_Python.yml | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml
index 69d66531de03..b96067b498e7 100644
--- a/.github/workflows/beam_PostCommit_Python.yml
+++ b/.github/workflows/beam_PostCommit_Python.yml
@@ -62,12 +62,7 @@ jobs:
         job_name: ['beam_PostCommit_Python']
         job_phrase: ['Run Python PostCommit']
         python_version: ['3.9', '3.10', '3.11', '3.12']
-        # Run on both self-hosted and GitHub-hosted runners.
-        # Some tests (marked require_docker_in_docker) can't run on Beam's
-        # self-hosted runners due to Docker-in-Docker environment constraint.
-        # These tests will only execute on ubuntu-latest (GitHub-hosted).
-        # Context: https://github.com/apache/beam/pull/35585.
-        os: [[self-hosted, ubuntu-20.04, highmem22], [ubuntu-latest]]
+        os: [[self-hosted, ubuntu-20.04, highmem22]]
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'pull_request_target' ||
@@ -105,11 +100,7 @@ jobs:
           arguments: |
             -Pjava21Home=$JAVA_HOME_21_X64 \
             -PuseWheelDistribution \
-            -Pposargs="${{
-              contains(matrix.os, 'self-hosted') &&
-                '-m (not require_docker_in_docker)' ||
-                '-m require_docker_in_docker'
-            }}" \
+            -Pposargs="-m (not require_docker_in_docker)" \
             -PpythonVersion=${{ matrix.python_version }} \
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}

From 7c40ae36e17bc0579c48d757832b193041dccdeb Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Fri, 5 Sep 2025 14:00:49 +0000
Subject: [PATCH 12/15] sdks/python: force tests to run only on precommit

---
 .../ml/rag/enrichment/milvus_search_it_test.py     | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py
index 4dabcafe6703..81ceb6b69e71 100644
--- a/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py
+++ b/sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py
@@ -574,7 +574,7 @@ def test_empty_input_chunks(self):
 
     expected_chunks = []
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -702,7 +702,7 @@ def test_filtered_search_with_cosine_similarity_and_batching(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -807,7 +807,7 @@ def test_filtered_search_with_bm25_full_text_and_batching(self):
             embedding=Embedding())
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -948,7 +948,7 @@ def test_vector_search_with_euclidean_distance(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1088,7 +1088,7 @@ def test_vector_search_with_inner_product_similarity(self):
             embedding=Embedding(dense_embedding=[0.3, 0.4, 0.5]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1153,7 +1153,7 @@ def test_keyword_search_with_inner_product_sparse_embedding(self):
                 sparse_embedding=([1, 2, 3, 4], [0.05, 0.41, 0.05, 0.41])))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,
@@ -1226,7 +1226,7 @@ def test_hybrid_search(self):
             embedding=Embedding(dense_embedding=[0.1, 0.2, 0.3]))
     ]
 
-    with TestPipeline(is_integration_test=True) as p:
+    with TestPipeline() as p:
       result = (p | beam.Create(test_chunks) | Enrichment(handler))
       assert_that(
           result,

From e58bfada0cac1d144af71b8b8439805207e0d2ac Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sun, 7 Sep 2025 18:53:17 +0000
Subject: [PATCH 13/15] Revert "actions+sdks/python: upgrade pip"

This reverts commit c8d88b02467f99ecbe31a56b88db510d81823c4a.
---
 .github/actions/setup-environment-action/action.yml | 7 -------
 sdks/python/tox.ini                                 | 2 ++
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml
index 6525103f83a5..d5f1f879a072 100644
--- a/.github/actions/setup-environment-action/action.yml
+++ b/.github/actions/setup-environment-action/action.yml
@@ -56,13 +56,6 @@ runs:
           sdks/python/setup.py
           sdks/python/tox.ini
 
-    - name: Upgrade pip to latest version
-      if: ${{ inputs.python-version != '' }}
-      shell: bash
-      run: |
-        python -m pip install --upgrade pip
-        pip --version
-
     - name: Cache tox environments
       if: ${{ inputs.python-version != '' && inputs.tox-cache == 'true' }}
       uses: actions/cache@v3
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 3ac91bdb4c93..157dfedecc47 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -106,6 +106,7 @@ commands =
 [testenv:py{39,310,311}-ml]
 # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests.
 deps =
+  pip==25.0.1
   accelerate>=1.6.0
 setenv =
 extras = test,gcp,dataframe,ml_test
@@ -515,6 +516,7 @@ deps =
   latest: accelerate>=1.6.0
   tensorflow==2.12.0
   protobuf==4.25.5
+  pip==25.0.1
 extras = test,gcp,ml_test
 commands =
   # Log transformers and its dependencies version for debugging

From 0cd2f8723ab906823a118e43bd99560f7694bbab Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Sun, 7 Sep 2025 18:53:57 +0000
Subject: [PATCH 14/15] workflows: run python 3.12 only on precommit ml
 `ubuntu-latest` os

---
 .github/workflows/beam_PreCommit_Python_ML.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/beam_PreCommit_Python_ML.yml b/.github/workflows/beam_PreCommit_Python_ML.yml
index e2d4ccafae5f..471dcf953be5 100644
--- a/.github/workflows/beam_PreCommit_Python_ML.yml
+++ b/.github/workflows/beam_PreCommit_Python_ML.yml
@@ -72,6 +72,16 @@ jobs:
         # These tests will only execute on ubuntu-latest (GitHub-hosted).
         # Context: https://github.com/apache/beam/pull/35585.
         os: [[self-hosted, ubuntu-20.04, main], [ubuntu-latest]]
+        exclude:
+          # Temporary exclude Python 3.9, 3.10, 3.11 from ubuntu-latest. This
+          # results in pip dependency resolution exceeded maximum depth issue.
+          # Context: https://github.com/apache/beam/pull/35816.
+          - python_version: '3.9'
+            os: [ubuntu-latest]
+          - python_version: '3.10'
+            os: [ubuntu-latest]
+          - python_version: '3.11'
+            os: [ubuntu-latest]
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||

From cc9f74c4d793a27f5b9f3434213202b0dbd4c812 Mon Sep 17 00:00:00 2001
From: Mohamed Awnallah <mohamedmohey2352@gmail.com>
Date: Wed, 10 Sep 2025 15:52:23 +0000
Subject: [PATCH 15/15] test-suites/direct/common.gradle: remove milvus
 postcommit gradle hook

---
 sdks/python/test-suites/direct/common.gradle | 28 --------------------
 1 file changed, 28 deletions(-)

diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index 3ca4591bc16f..1dd15ecb09f9 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -419,33 +419,6 @@ task feastIntegrationTest {
     }
 }
 
-// Integration tests that require Docker-in-Docker capabilities.
-// These tests are marked with the `require_docker_in_docker` pytest marker
-// because they rely on Docker-in-Docker configurations that are not supported
-// on Beam's self-hosted GitHub Actions runners. Docker-in-Docker works on
-// ubuntu-latest GitHub-hosted runners but not on self-hosted environments due
-// to containerization architecture differences.
-// Context: https://github.com/apache/beam/pull/35585
-task dockerInDockerIntegrationTest {
-  dependsOn 'installGcpTest'
-  dependsOn ':sdks:python:sdist'
-
-  doLast {
-      def testOpts = basicTestOpts
-      def argMap = [
-          "test_opts": testOpts,
-          "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
-          "collect": "require_docker_in_docker",
-          "runner": "TestDirectRunner",
-      ]
-      def cmdArgs = mapToArgString(argMap)
-      exec {
-        executable 'sh'
-        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
-      }
-    }
-}
-
 // Add all the RunInference framework IT tests to this gradle task that runs on Direct Runner Post commit suite.
 project.tasks.register("inferencePostCommitIT") {
   dependsOn = [
@@ -456,7 +429,6 @@ project.tasks.register("inferencePostCommitIT") {
   'transformersInferenceTest',
   'testcontainersTest',
   'feastIntegrationTest',
-  'dockerInDockerIntegrationTest',
   // (TODO) https://github.com/apache/beam/issues/25799
    // uncomment tfx bsl tests once tfx supports protobuf 4.x
   // 'tfxInferenceTest',