From 034653bf7aff8d6899a99e1a27b2e8c2d4ce62c9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 23 Nov 2025 23:33:44 +0900 Subject: [PATCH 1/5] feat(pipstar): extract the wheel without python With this we start extracting the wheel without Python and it becomes a requirement only when patching (we will extract the wheel without Python, patch it and then re-compress it which makes a very inefficient process). This should result in much faster executions. --- python/private/pypi/whl_library.bzl | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 5db7bc49a1..e986f6eeb5 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -377,20 +377,9 @@ def _whl_library_impl(rctx): # # Remove non-pipstar and config_load check when we release rules_python 2. if enable_pipstar: - pypi_repo_utils.execute_checked( - rctx, - op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), - python = python_interpreter, - arguments = args + [ - "--whl-file", - whl_path, - "--enable-pipstar", - ], - srcs = rctx.attr._python_srcs, - environment = environment, - quiet = rctx.attr.quiet, - timeout = rctx.attr.timeout, - logger = logger, + rctx.extract( + archive = whl_path, + output = "site-packages", ) metadata = whl_metadata( From 82d97a8290c72d7b0deb99979447edf1008418a5 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:23:13 +0900 Subject: [PATCH 2/5] stop passing the interpreter to whl pipstar targets --- python/private/pypi/hub_builder.bzl | 15 +++++++++++---- tests/pypi/hub_builder/hub_builder_tests.bzl | 10 ---------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 69359bcf8a..e3490798f7 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -440,6 +440,9 @@ def _create_whl_repos( pip_attr = pip_attr, enable_pipstar = enable_pipstar, ) + + interpreter = _detect_interpreter(self, pip_attr) + for whl in requirements_by_platform: whl_library_args = common_args | _whl_library_args( self, @@ -456,6 +459,7 @@ def _create_whl_repos( auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(pip_attr.python_version), is_multiple_versions = whl.is_multiple_versions, + interpreter = interpreter, enable_pipstar = enable_pipstar, ) _add_whl_library( @@ -467,8 +471,6 @@ def _create_whl_repos( ) def _common_args(self, module_ctx, *, pip_attr, enable_pipstar): - interpreter = _detect_interpreter(self, pip_attr) - # Construct args separately so that the lock file can be smaller and does not include unused # attrs. whl_library_args = dict( @@ -483,8 +485,6 @@ def _common_args(self, module_ctx, *, pip_attr, enable_pipstar): environment = pip_attr.environment, envsubst = pip_attr.envsubst, pip_data_exclude = pip_attr.pip_data_exclude, - python_interpreter = interpreter.path, - python_interpreter_target = interpreter.target, ) if not enable_pipstar: maybe_args["experimental_target_platforms"] = pip_attr.experimental_target_platforms @@ -536,6 +536,7 @@ def _whl_repo( auth_patterns, python_version, use_downloader, + interpreter, enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line @@ -548,6 +549,12 @@ def _whl_repo( # need to pass the extra args there, so only pop this for whls args["extra_pip_args"] = src.extra_pip_args + if not (enable_pipstar and is_whl): + if interpreter.path: + args["python_interpreter"] = interpreter.path + if interpreter.target: + args["python_interpreter_target"] = interpreter.target + if not src.url or (not is_whl and download_only): if download_only and use_downloader: # If the user did not allow using sdists and we are using the downloader diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 6d061f4d56..75d1ebf9a1 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -320,7 +320,6 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple[foo]==0.0.1", "sha256": "deadbeef", "urls": ["https://example.com/simple-0.0.1-py3-none-any.whl"], @@ -329,7 +328,6 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1", "sha256": "deadbeef", "urls": ["https://example.com/simple-0.0.1-py3-none-any.whl"], @@ -663,7 +661,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], @@ -672,7 +669,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], @@ -681,7 +677,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", "urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], @@ -690,7 +685,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], @@ -979,7 +973,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "direct_without_sha==0.0.1", "sha256": "", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], @@ -1002,7 +995,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1", "sha256": "deadb00f", "urls": ["example2.org"], @@ -1011,7 +1003,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "some_pkg-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_pkg==0.0.1", "sha256": "deadbaaf", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], @@ -1020,7 +1011,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "some-other-pkg-0.0.1-py3-none-any.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "some_other_pkg==0.0.1", "sha256": "deadb33f", "urls": ["example2.org/index/some_other_pkg/"], From 2a67682c4a692a642bd18f3ff932574a68610784 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:39:14 +0900 Subject: [PATCH 3/5] symlink the whl to a zip file to work around extract limitations in older bazels --- python/private/internal_config_repo.bzl | 2 ++ python/private/pypi/patch_whl.bzl | 2 ++ python/private/pypi/whl_library.bzl | 9 ++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index b208037c13..91f786c64e 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -32,6 +32,7 @@ config = struct( enable_pystar = True, enable_pipstar = {enable_pipstar}, enable_deprecation_warnings = {enable_deprecation_warnings}, + bazel_8_or_later = {bazel_8_or_later}, bazel_9_or_later = {bazel_9_or_later}, BuiltinPyInfo = getattr(getattr(native, "legacy_globals", None), "PyInfo", {builtin_py_info_symbol}), BuiltinPyRuntimeInfo = getattr(getattr(native, "legacy_globals", None), "PyRuntimeInfo", {builtin_py_runtime_info_symbol}), @@ -107,6 +108,7 @@ def _internal_config_repo_impl(rctx): builtin_py_info_symbol = builtin_py_info_symbol, builtin_py_runtime_info_symbol = builtin_py_runtime_info_symbol, builtin_py_cc_link_params_provider = builtin_py_cc_link_params_provider, + bazel_8_or_later = str(bazel_major_version >= 8), bazel_9_or_later = str(bazel_major_version >= 9), )) diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index 7af9c4da2f..e315989dd9 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -87,6 +87,8 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): # symlink to a zip file to use bazel's extract so that we can use bazel's # repository_ctx patch implementation. The whl file may be in a different # external repository. + # + # TODO @aignas 2025-11-24: remove this symlinking workaround when we drop support for bazel 7 whl_file_zip = whl_input.basename + ".zip" rctx.symlink(whl_input, whl_file_zip) rctx.extract(whl_file_zip) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index e986f6eeb5..6b515a56a4 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -377,10 +377,17 @@ def _whl_library_impl(rctx): # # Remove non-pipstar and config_load check when we release rules_python 2. if enable_pipstar: + if rp_config.bazel_8_or_later: + extract_path = whl_path + else: + extract_path = rctx.path(whl_path.basename + ".zip") + rctx.symlink(whl_path, extract_path) rctx.extract( - archive = whl_path, + archive = extract_path, output = "site-packages", ) + if not rp_config.bazel_8_or_later: + rctx.delete(extract_path) metadata = whl_metadata( install_dir = whl_path.dirname.get_child("site-packages"), From 916dd959b1f43404541e13075fd53401cc0da19a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 24 Nov 2025 15:52:36 +0900 Subject: [PATCH 4/5] fix a bug --- python/private/pypi/hub_builder.bzl | 2 +- tests/pypi/hub_builder/hub_builder_tests.bzl | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index e3490798f7..1378e2f122 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -549,7 +549,7 @@ def _whl_repo( # need to pass the extra args there, so only pop this for whls args["extra_pip_args"] = src.extra_pip_args - if not (enable_pipstar and is_whl): + if "whl_patches" in args or not (enable_pipstar and is_whl): if interpreter.path: args["python_interpreter"] = interpreter.path if interpreter.target: diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 75d1ebf9a1..4968e6234c 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -44,6 +44,7 @@ def hub_builder( debug = False, config = None, minor_mapping = {}, + whl_overrides = {}, evaluate_markers_fn = None, simpleapi_download_fn = None, available_interpreters = {}): @@ -76,7 +77,7 @@ def hub_builder( netrc = None, auth_patterns = None, ), - whl_overrides = {}, + whl_overrides = whl_overrides, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, available_interpreters = available_interpreters or { "python_3_15_host": "unit_test_interpreter_target", @@ -832,6 +833,11 @@ def _test_simple_get_index(env): builder = hub_builder( env, simpleapi_download_fn = mocksimpleapi_download, + whl_overrides = { + "direct_without_sha": { + "my_patch": 1, + }, + }, ) builder.pip_parse( _mock_mctx( @@ -973,9 +979,14 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", + "python_interpreter_target": "unit_test_interpreter_target", "requirement": "direct_without_sha==0.0.1", "sha256": "", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], + # NOTE @aignas 2025-11-24: any patching still requires the python interpreter from the + # hermetic toolchain or the system. This is so that we can rezip it back to a wheel and + # verify the metadata so that it is installable by any installer out there. + "whl_patches": {"my_patch": "1"}, }, "pypi_315_git_dep": { "config_load": "@pypi//:config.bzl", From dfdf707800437638ec27f7fa8ae02c57b0703f71 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 25 Nov 2025 11:44:41 +0900 Subject: [PATCH 5/5] Remove python_interpreter_target from config Removed 'python_interpreter_target' from the configuration. --- tests/pypi/hub_builder/hub_builder_tests.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 22129f48d5..414ad1250e 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1026,7 +1026,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", - "python_interpreter_target": "unit_test_interpreter_target", "requirement": "plat_pkg==0.0.4", "sha256": "deadb44f", "urls": ["example2.org/index/plat_pkg/"],