From 57cdae582936d09ec93edae4a7d692a74381a0e9 Mon Sep 17 00:00:00 2001 From: Nathanael Mercaldo Date: Sun, 28 Dec 2025 23:03:18 -0500 Subject: [PATCH 1/8] Initial implementation --- e2e/cases/bin-to-bin-deps/BUILD.bazel | 32 +++++++++++++ e2e/cases/bin-to-bin-deps/bin_a.py | 1 + e2e/cases/bin-to-bin-deps/bin_b.py | 5 ++ e2e/cases/bin-to-bin-deps/lib.py | 3 ++ e2e/cases/bin-to-bin-deps/test.py | 4 ++ py/private/BUILD.bazel | 15 ++++++ py/private/bin_to_lib.bzl | 68 +++++++++++++++++++++++++++ py/private/py_binary.bzl | 9 +++- py/private/py_library.bzl | 4 +- 9 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 e2e/cases/bin-to-bin-deps/BUILD.bazel create mode 100644 e2e/cases/bin-to-bin-deps/bin_a.py create mode 100644 e2e/cases/bin-to-bin-deps/bin_b.py create mode 100644 e2e/cases/bin-to-bin-deps/lib.py create mode 100644 e2e/cases/bin-to-bin-deps/test.py create mode 100644 py/private/bin_to_lib.bzl diff --git a/e2e/cases/bin-to-bin-deps/BUILD.bazel b/e2e/cases/bin-to-bin-deps/BUILD.bazel new file mode 100644 index 00000000..c4e12216 --- /dev/null +++ b/e2e/cases/bin-to-bin-deps/BUILD.bazel @@ -0,0 +1,32 @@ +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_test", "py_library") + +py_library( + name = "lib", + imports = ["."], + srcs = ["lib.py"] +) + +py_binary( + name = "bin_a", + srcs = ["bin_a.py"], + deps = [ + ":lib" + ] +) + +py_binary( + name = "bin_b", + srcs = ["bin_b.py"], + deps = [ + ":bin_a" + ] +) + +py_test( + name = "test", + srcs = ["test.py"], + deps = [ + ":bin_a", + ":bin_b", + ] +) \ No newline at end of file diff --git a/e2e/cases/bin-to-bin-deps/bin_a.py b/e2e/cases/bin-to-bin-deps/bin_a.py new file mode 100644 index 00000000..8b367f3b --- /dev/null +++ b/e2e/cases/bin-to-bin-deps/bin_a.py @@ -0,0 +1 @@ +print("bin_a: Hello!") \ No newline at end of file diff --git a/e2e/cases/bin-to-bin-deps/bin_b.py b/e2e/cases/bin-to-bin-deps/bin_b.py new file mode 100644 index 00000000..1e3d3f8e --- /dev/null +++ b/e2e/cases/bin-to-bin-deps/bin_b.py @@ -0,0 +1,5 @@ +from lib import get_msg + +print("bin_b: Hello!") + +print(get_msg()) \ No newline at end of file diff --git a/e2e/cases/bin-to-bin-deps/lib.py b/e2e/cases/bin-to-bin-deps/lib.py new file mode 100644 index 00000000..2917ee32 --- /dev/null +++ b/e2e/cases/bin-to-bin-deps/lib.py @@ -0,0 +1,3 @@ + +def get_msg(): + return "Hello from lib!" \ No newline at end of file diff --git a/e2e/cases/bin-to-bin-deps/test.py b/e2e/cases/bin-to-bin-deps/test.py new file mode 100644 index 00000000..e5c17241 --- /dev/null +++ b/e2e/cases/bin-to-bin-deps/test.py @@ -0,0 +1,4 @@ +from lib import get_msg + +def test_lib(): + assert "Hello from lib!" == get_msg() \ No newline at end of file diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index eb36e61b..d8c54690 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -1,4 +1,5 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_visibility = ["//py:__subpackages__"]) @@ -10,6 +11,20 @@ exports_files( visibility = ["//visibility:public"], ) +bool_flag( + name = "bin_to_lib_flag", + build_setting_default = True, + visibility = ["//:__subpackages__"] +) + +exports_files( + [ + "run.tmpl.sh", + "pytest.py.tmpl", + ], + visibility = ["//visibility:public"], +) + bzl_library( name = "py_image_layer", srcs = ["py_image_layer.bzl"], diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl new file mode 100644 index 00000000..bd62ab80 --- /dev/null +++ b/py/private/bin_to_lib.bzl @@ -0,0 +1,68 @@ +""" + +Sometimes it is desirable for a python binary or venv to depend on another binary target or venv. +This package reifies bin-dep-bin semantics by transitioning a binary into a library if it is + dependend upon by another binary. + +This functionality has the following benefits: +1. py_venv_binary behaves more like the original py_binary rule. +2. Easier composition of venvs +3. Better merge semantics for existing + +""" + + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_python//python:defs.bzl", "PyInfo") +load("//py/private:providers.bzl", "PyVirtualInfo") + +def _bin_to_lib_transition_impl(settings, attr): + return { + "//py/private:bin_to_lib_flag": False, + } + +bin_to_lib_transition = transition( + implementation = _bin_to_lib_transition_impl, + inputs = [ + "//py/private:bin_to_lib_flag", + ], + outputs = [ + "//py/private:bin_to_lib_flag", + ], +) + + +def wrap_with_bin_to_lib(bin_rule, lib_rule): + def helper(ctx): + if not ctx.attr._binary_mode: + fail("Wrapped rule missing required attributes.") + + if ctx.attr._binary_mode[BuildSettingInfo].value: + return bin_rule(ctx) + + # Ugly hack: Is there a better way? + dummy_file = ctx.actions.declare_file( + ctx.label.name + "_dummy_bin" + ) + ctx.actions.write( + output = dummy_file, + content = "\n\n", + ) + return lib_rule(ctx, dummy_file) + return helper + +bin_to_lib = struct( + wrapper = wrap_with_bin_to_lib, + attribs = dict({ + "_binary_mode": attr.label( + default="//py/private:bin_to_lib_flag" + ), + "deps": attr.label_list( + doc = "Targets that produce Python code, commonly `py_library` rules.", + providers = [[PyInfo], [PyVirtualInfo], [CcInfo]], + cfg = bin_to_lib_transition + ), + }) +) + diff --git a/py/private/py_binary.bzl b/py/private/py_binary.bzl index cfc0ea3f..668fba42 100644 --- a/py/private/py_binary.bzl +++ b/py/private/py_binary.bzl @@ -8,6 +8,8 @@ load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private/toolchain:types.bzl", "PY_TOOLCHAIN", "VENV_TOOLCHAIN") load(":transitions.bzl", "python_version_transition") +load(":bin_to_lib.bzl", "bin_to_lib") + def _dict_to_exports(env): return [ "export %s=\"%s\"" % (k, v) @@ -228,8 +230,11 @@ py_base = struct( py_binary = rule( doc = "Run a Python program under Bazel. Most users should use the [py_binary macro](#py_binary) instead of loading this directly.", - implementation = py_base.implementation, - attrs = py_base.attrs, + implementation = bin_to_lib.wrapper( + py_base.implementation, + _py_library.implementation + ), + attrs = py_base.attrs | bin_to_lib.attribs, toolchains = py_base.toolchains, executable = True, cfg = py_base.cfg, diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl index ffc1abcc..6fcb1609 100644 --- a/py/private/py_library.bzl +++ b/py/private/py_library.bzl @@ -171,7 +171,7 @@ def _make_merged_runfiles(ctx, extra_depsets = [], extra_runfiles = [], extra_ru return runfiles -def _py_library_impl(ctx): +def _py_library_impl(ctx, executable=None): transitive_srcs = _make_srcs_depset(ctx) imports = _make_imports_depset(ctx) virtuals = _make_virtual_depset(ctx) @@ -183,6 +183,8 @@ def _py_library_impl(ctx): DefaultInfo( files = depset(direct = ctx.files.srcs), default_runfiles = runfiles, + # Ugly hack: Inject dummy executable in case this library is actually a transitioned binary. + executable = executable ), PyInfo( imports = imports, From 97171f2e94e0eede16a5ed3a33cc5fe4b5391ec9 Mon Sep 17 00:00:00 2001 From: ParaLock Date: Sun, 28 Dec 2025 23:32:15 -0500 Subject: [PATCH 2/8] Remove copy-paste error --- py/private/BUILD.bazel | 8 -------- 1 file changed, 8 deletions(-) diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index d8c54690..79da4178 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -3,14 +3,6 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_visibility = ["//py:__subpackages__"]) -exports_files( - [ - "run.tmpl.sh", - "pytest.py.tmpl", - ], - visibility = ["//visibility:public"], -) - bool_flag( name = "bin_to_lib_flag", build_setting_default = True, From 1064f42315b852d0cee4af1995faab3f90b5261e Mon Sep 17 00:00:00 2001 From: ParaLock Date: Mon, 29 Dec 2025 00:29:35 -0500 Subject: [PATCH 3/8] Refactor lib executable --- py/private/bin_to_lib.bzl | 51 +++++++++++++++++++++++++++------------ py/private/py_library.bzl | 4 +-- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index bd62ab80..2789d97c 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -2,16 +2,15 @@ Sometimes it is desirable for a python binary or venv to depend on another binary target or venv. This package reifies bin-dep-bin semantics by transitioning a binary into a library if it is - dependend upon by another binary. + depended upon by another binary. This functionality has the following benefits: 1. py_venv_binary behaves more like the original py_binary rule. 2. Easier composition of venvs -3. Better merge semantics for existing +3. Better merge semantics for existing binary rules. """ - load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("@rules_python//python:defs.bzl", "PyInfo") @@ -32,37 +31,57 @@ bin_to_lib_transition = transition( ], ) +def _add_executable(ctx, providers): + new_providers = [] + + # Ugly hack: Is there a better way? + dummy_file = ctx.actions.declare_file( + ctx.label.name + "_dummy_bin", + ) + ctx.actions.write( + output = dummy_file, + content = "\n\n", + ) + + for p in providers: + if type(p) == "DefaultInfo": + new_providers.append( + DefaultInfo( + files = p.files, + default_runfiles = p.default_runfiles, + executable = dummy_file, + ), + ) + else: + new_providers.append(p) + + return new_providers def wrap_with_bin_to_lib(bin_rule, lib_rule): def helper(ctx): if not ctx.attr._binary_mode: fail("Wrapped rule missing required attributes.") - + if ctx.attr._binary_mode[BuildSettingInfo].value: return bin_rule(ctx) - # Ugly hack: Is there a better way? - dummy_file = ctx.actions.declare_file( - ctx.label.name + "_dummy_bin" - ) - ctx.actions.write( - output = dummy_file, - content = "\n\n", + return _add_executable( + ctx, + lib_rule(ctx), ) - return lib_rule(ctx, dummy_file) + return helper bin_to_lib = struct( wrapper = wrap_with_bin_to_lib, attribs = dict({ "_binary_mode": attr.label( - default="//py/private:bin_to_lib_flag" + default = "//py/private:bin_to_lib_flag", ), "deps": attr.label_list( doc = "Targets that produce Python code, commonly `py_library` rules.", providers = [[PyInfo], [PyVirtualInfo], [CcInfo]], - cfg = bin_to_lib_transition + cfg = bin_to_lib_transition, ), - }) + }), ) - diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl index 6fcb1609..ffc1abcc 100644 --- a/py/private/py_library.bzl +++ b/py/private/py_library.bzl @@ -171,7 +171,7 @@ def _make_merged_runfiles(ctx, extra_depsets = [], extra_runfiles = [], extra_ru return runfiles -def _py_library_impl(ctx, executable=None): +def _py_library_impl(ctx): transitive_srcs = _make_srcs_depset(ctx) imports = _make_imports_depset(ctx) virtuals = _make_virtual_depset(ctx) @@ -183,8 +183,6 @@ def _py_library_impl(ctx, executable=None): DefaultInfo( files = depset(direct = ctx.files.srcs), default_runfiles = runfiles, - # Ugly hack: Inject dummy executable in case this library is actually a transitioned binary. - executable = executable ), PyInfo( imports = imports, From 842699443dee410498ab880884b864345db50270 Mon Sep 17 00:00:00 2001 From: ParaLock Date: Mon, 29 Dec 2025 11:54:31 -0500 Subject: [PATCH 4/8] Try propagating binaries --- py/private/bin_to_lib.bzl | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index 2789d97c..7edc8afa 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -1,7 +1,7 @@ """ Sometimes it is desirable for a python binary or venv to depend on another binary target or venv. -This package reifies bin-dep-bin semantics by transitioning a binary into a library if it is +Here we reify bin-dep-bin semantics by transitioning a binary into a library if it is depended upon by another binary. This functionality has the following benefits: @@ -31,30 +31,28 @@ bin_to_lib_transition = transition( ], ) -def _add_executable(ctx, providers): - new_providers = [] - - # Ugly hack: Is there a better way? - dummy_file = ctx.actions.declare_file( - ctx.label.name + "_dummy_bin", - ) - ctx.actions.write( - output = dummy_file, - content = "\n\n", - ) +def _find_provider(providers, prov_type_name): + for provider in providers: + if type(provider) == prov_type_name: + return provider + return None - for p in providers: +def _add_executable(ctx, lib_providers, bin_providers): + new_providers = [] + bin_default_info = _find_provider(bin_providers, "DefaultInfo") + for p in lib_providers: if type(p) == "DefaultInfo": new_providers.append( DefaultInfo( files = p.files, default_runfiles = p.default_runfiles, - executable = dummy_file, + # ugly hack: For some reason files_to_run is None so we must infer executable + # from file list. + executable = bin_default_info.files.to_list()[0], ), ) - else: - new_providers.append(p) - + continue + new_providers.append(p) return new_providers def wrap_with_bin_to_lib(bin_rule, lib_rule): @@ -62,12 +60,15 @@ def wrap_with_bin_to_lib(bin_rule, lib_rule): if not ctx.attr._binary_mode: fail("Wrapped rule missing required attributes.") + bin_providers = bin_rule(ctx) if ctx.attr._binary_mode[BuildSettingInfo].value: - return bin_rule(ctx) + return bin_providers + lib_providers = lib_rule(ctx) return _add_executable( ctx, - lib_rule(ctx), + lib_providers, + bin_providers, ) return helper From 7170e21173315f687ff5bbec9717ec0ee22ae68b Mon Sep 17 00:00:00 2001 From: ParaLock Date: Mon, 29 Dec 2025 11:58:07 -0500 Subject: [PATCH 5/8] Added comment --- py/private/bin_to_lib.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index 7edc8afa..75c02804 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -65,6 +65,9 @@ def wrap_with_bin_to_lib(bin_rule, lib_rule): return bin_providers lib_providers = lib_rule(ctx) + + # It appears that one cannot transition the executable status of a binary. + # This means we need to resolve an executable for binaries transitioned to libraries. return _add_executable( ctx, lib_providers, From 913fd7af73e98b0d90511ded785def8c47e4d739 Mon Sep 17 00:00:00 2001 From: ParaLock Date: Mon, 29 Dec 2025 12:12:43 -0500 Subject: [PATCH 6/8] Found better way to get at executable --- py/private/bin_to_lib.bzl | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index 75c02804..3882c13e 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -31,24 +31,15 @@ bin_to_lib_transition = transition( ], ) -def _find_provider(providers, prov_type_name): - for provider in providers: - if type(provider) == prov_type_name: - return provider - return None - -def _add_executable(ctx, lib_providers, bin_providers): +def _add_executable(ctx, lib_providers): new_providers = [] - bin_default_info = _find_provider(bin_providers, "DefaultInfo") for p in lib_providers: if type(p) == "DefaultInfo": new_providers.append( DefaultInfo( files = p.files, default_runfiles = p.default_runfiles, - # ugly hack: For some reason files_to_run is None so we must infer executable - # from file list. - executable = bin_default_info.files.to_list()[0], + executable = ctx.outputs.executable, ), ) continue @@ -64,14 +55,11 @@ def wrap_with_bin_to_lib(bin_rule, lib_rule): if ctx.attr._binary_mode[BuildSettingInfo].value: return bin_providers - lib_providers = lib_rule(ctx) - # It appears that one cannot transition the executable status of a binary. # This means we need to resolve an executable for binaries transitioned to libraries. return _add_executable( ctx, - lib_providers, - bin_providers, + lib_rule(ctx), ) return helper From eeb85410cdf0acde527c455780eeebd65c6e255e Mon Sep 17 00:00:00 2001 From: ParaLock Date: Mon, 29 Dec 2025 12:17:37 -0500 Subject: [PATCH 7/8] Move transition to the right location --- py/private/bin_to_lib.bzl | 16 +--------------- py/private/transitions.bzl | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index 3882c13e..a89fe058 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -15,21 +15,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("@rules_python//python:defs.bzl", "PyInfo") load("//py/private:providers.bzl", "PyVirtualInfo") - -def _bin_to_lib_transition_impl(settings, attr): - return { - "//py/private:bin_to_lib_flag": False, - } - -bin_to_lib_transition = transition( - implementation = _bin_to_lib_transition_impl, - inputs = [ - "//py/private:bin_to_lib_flag", - ], - outputs = [ - "//py/private:bin_to_lib_flag", - ], -) +load(":transitions.bzl", "bin_to_lib_transition") def _add_executable(ctx, lib_providers): new_providers = [] diff --git a/py/private/transitions.bzl b/py/private/transitions.bzl index 13660e4e..5d91e370 100644 --- a/py/private/transitions.bzl +++ b/py/private/transitions.bzl @@ -30,5 +30,20 @@ python_transition = transition( ], ) +def _bin_to_lib_transition_impl(settings, attr): + return { + "//py/private:bin_to_lib_flag": False, + } + +bin_to_lib_transition = transition( + implementation = _bin_to_lib_transition_impl, + inputs = [ + "//py/private:bin_to_lib_flag", + ], + outputs = [ + "//py/private:bin_to_lib_flag", + ], +) + # The old name, FIXME: refactor this out python_version_transition = python_transition From 2e83626f6fbff55b4918370a1b60699a69a11350 Mon Sep 17 00:00:00 2001 From: ParaLock Date: Tue, 30 Dec 2025 20:43:57 -0500 Subject: [PATCH 8/8] Switch over py_venv_binary. (this breaks tests) --- py/private/bin_to_lib.bzl | 20 +++++++++++++++----- py/private/py_venv/py_venv.bzl | 16 ++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/py/private/bin_to_lib.bzl b/py/private/bin_to_lib.bzl index a89fe058..81885b0c 100644 --- a/py/private/bin_to_lib.bzl +++ b/py/private/bin_to_lib.bzl @@ -17,13 +17,21 @@ load("@rules_python//python:defs.bzl", "PyInfo") load("//py/private:providers.bzl", "PyVirtualInfo") load(":transitions.bzl", "bin_to_lib_transition") -def _add_executable(ctx, lib_providers): +def _find_provider(providers, prov_type_name): + for provider in providers: + if type(provider) == prov_type_name: + return provider + return None + +def _resolve_providers(ctx, lib_providers, bin_providers): new_providers = [] + bin_default_info = _find_provider(bin_providers, "DefaultInfo") for p in lib_providers: if type(p) == "DefaultInfo": new_providers.append( DefaultInfo( files = p.files, + data_runfiles = p.data_runfiles, default_runfiles = p.default_runfiles, executable = ctx.outputs.executable, ), @@ -41,11 +49,13 @@ def wrap_with_bin_to_lib(bin_rule, lib_rule): if ctx.attr._binary_mode[BuildSettingInfo].value: return bin_providers - # It appears that one cannot transition the executable status of a binary. - # This means we need to resolve an executable for binaries transitioned to libraries. - return _add_executable( + lib_providers = lib_rule(ctx) + + # Looks like we need to extract and propagate some files from the original binary. + return _resolve_providers( ctx, - lib_rule(ctx), + lib_providers, + bin_providers, ) return helper diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index 0f3788da..e56e77b7 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -2,6 +2,7 @@ load("@aspect_bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variables") load("@aspect_bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_rlocation_path") +load("//py/private:bin_to_lib.bzl", "bin_to_lib") load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "python_version_transition") @@ -392,8 +393,11 @@ py_venv_base = struct( _py_venv = rule( doc = """Build a Python virtual environment and execute its interpreter.""", - implementation = _py_venv_rule_impl, - attrs = py_venv_base.attrs, + implementation = bin_to_lib.wrapper( + _py_venv_rule_impl, + _py_library.implementation, + ), + attrs = py_venv_base.attrs | bin_to_lib.attribs, toolchains = py_venv_base.toolchains, executable = True, cfg = py_venv_base.cfg, @@ -411,8 +415,12 @@ def _wrap_with_debug(rule): _py_venv_binary = rule( doc = """Run a Python program under Bazel using a virtualenv.""", - implementation = _py_venv_binary_impl, - attrs = py_venv_base.attrs | py_venv_base.binary_attrs, + implementation = bin_to_lib.wrapper( + _py_venv_binary_impl, + _py_library.implementation, + ), + #implementation = _py_venv_binary_impl, + attrs = py_venv_base.attrs | py_venv_base.binary_attrs | bin_to_lib.attribs, toolchains = py_venv_base.toolchains, executable = True, cfg = py_venv_base.cfg,