Split Pex .whl into two .whls.

.github/workflows/ci.yml

@@ -72,7 +72,8 @@ jobs:
         run: |
           uv run dev-cmd package -- \
             --additional-format sdist \
-            --additional-format wheel \
+            --additional-format whl \
+            --additional-format whl-3.12-plus \
             --embed-docs \
             --clean-docs \
             --scies \

.github/workflows/release.yml

@@ -64,8 +64,15 @@ jobs:
           python-version: "3.11"
       - name: Install uv
         uses: astral-sh/setup-uv@v5
-      - name: Build sdist and wheel
-        run: uv run dev-cmd package -- --no-pex --additional-format sdist --additional-format wheel --embed-docs --clean-docs
+      - name: Build sdist and wheels
+        run: |
+          uv run dev-cmd package -- \
+            --no-pex \
+            --additional-format sdist \
+            --additional-format whl \
+            --additional-format whl-3.12-plus \
+            --embed-docs \
+            --clean-docs
       - name: Publish Pex ${{ needs.determine-tag.outputs.release-tag }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

CHANGES.md

@@ -1,5 +1,17 @@
 # Release Notes
 
+## 2.77.0
+
+This release has no fixes or new features per-se, but just changes the set of distributions that
+Pex releases to PyPI. Previously Pex released an sdist and a universal (`py2.py3-none-any`) `.whl`.
+Pex now releases two wheels in addition to the sdist. The `py3.py312-none-any.whl` targets
+Python>=3.12 and has un-needed vendored libraries elided making it bith a smaller `.whl` and less
+prone to false-positive security scan issues since unused vendored code is now omitted. The other
+wheel carries the same contents as prior and supports creating PEXes for Python 2.7 and
+Python>=3.5,<3.12.
+
+* Split Pex `.whl` into two `.whl`s. (#3057)
+
 ## 2.76.1
 
 This release fixes bootstrapping of Pips specified via `--pip-version` to respect Pex Pip

build-backend/pex_build/__init__.py

@@ -14,11 +14,25 @@
 from pex.common import safe_mkdir, safe_mkdtemp
 from pex.third_party.packaging.markers import Marker
 from pex.typing import TYPE_CHECKING
+from pex.vendor import iter_vendor_specs
 
 if TYPE_CHECKING:
     from typing import Callable, Iterator, Optional
 
-INCLUDE_DOCS = os.environ.get("__PEX_BUILD_INCLUDE_DOCS__", "False").lower() in ("1", "true")
+
+def read_bool_env(
+    name,  # type: str
+    default,  # type: bool
+):
+    # type: (...) -> bool
+    value = os.environ.get(name)
+    if value is None:
+        return default
+    return value.lower() in ("1", "true")
+
+
+INCLUDE_DOCS = read_bool_env("__PEX_BUILD_INCLUDE_DOCS__", default=False)
+WHEEL_3_12_PLUS = read_bool_env("__PEX_BUILD_WHL_3_12_PLUS__", default=False)
 
 
 @contextmanager
@@ -128,3 +142,8 @@ def modify_wheel(
                 out_dir,
             ]
         )
+    if WHEEL_3_12_PLUS:
+        for vendor_spec in iter_vendor_specs():
+            if vendor_spec.key in ("pip", "setuptools", "toml", "tomli"):
+                shutil.rmtree(os.path.join(wheel_dir, vendor_spec.relpath))
+                print("Removed un-needed vendored library", vendor_spec.relpath, file=sys.stderr)

pex/environment.py

@@ -59,7 +59,7 @@ def _import_pkg_resources():
     except ImportError:
         from pex import third_party
 
-        third_party.install(expose=["setuptools"])
+        third_party.install(expose_if_available=["setuptools"])
         try:
             import pkg_resources  # vendor:skip
 

pex/pep_425.py

@@ -49,7 +49,9 @@ class RankedTag(object):
     rank = attr.ib()  # type: TagRank
 
     def select_higher_rank(self, other):
-        # type: (RankedTag) -> RankedTag
+        # type: (Optional[RankedTag]) -> RankedTag
+        if other is None:
+            return self
         return Rank.select_highest_rank(
             self, other, extract_rank=lambda ranked_tag: ranked_tag.rank
         )

pex/pep_427.py

@@ -76,8 +76,8 @@ class InstallableType(Enum["InstallableType.Value"]):
     class Value(Enum.Value):
         pass
 
-    INSTALLED_WHEEL_CHROOT = Value("installed wheel chroot")
-    WHEEL_FILE = Value(".whl file")
+    INSTALLED_WHEEL_CHROOT = Value("installed-wheel-chroot")
+    WHEEL_FILE = Value(".whl-file")
 
 
 InstallableType.seal()
@@ -974,8 +974,9 @@ def is_entry_point_script(script_path):
         if not compile and installed_file.path.endswith(".pyc"):
             continue
 
-        src_file = os.path.realpath(os.path.join(wheel.location, installed_file.path))
-        if not os.path.exists(src_file):
+        src_file = os.path.normpath(os.path.join(wheel.location, installed_file.path))
+        src_file_realpath = os.path.realpath(src_file)
+        if not os.path.exists(src_file_realpath):
             if not warned_bad_record:
                 pex_warnings.warn(
                     "The wheel {whl} has a bad RECORD. Skipping install of non-existent file "
@@ -989,17 +990,24 @@ def is_entry_point_script(script_path):
         dst_components = None  # type: Optional[Tuple[Text, Text, bool]]
         for path_name, installed_path in wheel.iter_install_paths_by_name():
             installed_path = os.path.realpath(installed_path)
-            if installed_path == commonpath((installed_path, src_file)):
+
+            src_path = None  # type: Optional[Text]
+            if installed_path == commonpath((installed_path, src_file_realpath)):
+                src_path = src_file_realpath
+            elif installed_path == commonpath((installed_path, src_file)):
+                src_path = src_file
+
+            if src_path:
                 rewrite_script = False
                 if "scripts" == path_name:
-                    if is_entry_point_script(src_file):
+                    if is_entry_point_script(src_path):
                         # This entry point script will be installed afresh below as needed.
                         break
                     rewrite_script = interpreter is not None and is_python_script(
-                        src_file, check_executable=False
+                        src_path, check_executable=False
                     )
 
-                dst_rel_path = os.path.relpath(src_file, installed_path)
+                dst_rel_path = os.path.relpath(src_path, installed_path)
                 dst_components = path_name, dst_rel_path, rewrite_script
                 break
         else:

pex/pex_builder.py

@@ -543,7 +543,11 @@ def _prepare_bootstrap(self):
         # NB: We use pip here in the builder, but that's only at build time, and
         # although we don't use pyparsing directly, packaging.markers, which we
         # do use at runtime, does.
-        root_module_names = ["appdirs", "attr", "colors", "packaging", "pkg_resources", "pyparsing"]
+        root_module_names = ["appdirs", "attr", "colors", "packaging", "pyparsing"]
+        for vendor_spec in vendor.iter_vendor_specs():
+            if vendor_spec.key == "setuptools":
+                root_module_names.append("pkg_resources")
+
         prepared_sources = vendor.vendor_runtime(
             chroot=self._chroot,
             dest_basedir=self._pex_info.bootstrap,

pex/pip/installation.py

@@ -278,14 +278,7 @@ def bootstrap_pip():
         try:
             bootstrap_pip_version = PipVersion.for_value(pip_version.raw)
         except ValueError:
-            # Backstop with the version of Pip CPython 3.12.0 shipped with. This should be the
-            # oldest Pip we need in the Pip bootstrap process (which is only required for
-            # Python >= 3.12 which have distutils removed rendering vendored Pip unusable).
-            bootstrap_pip_version = PipVersion.v23_2
-            for rev in sorted(PipVersion.values(), reverse=True):
-                if pip_version > rev.version:
-                    bootstrap_pip_version = rev
-                    break
+            bootstrap_pip_version = PipVersionValue(pip_version.raw)
 
         pip = BootstrapPip(
             pip_venv=PipVenv(
@@ -388,7 +381,7 @@ def _resolved_installation(
             warn=False,
         )
     )
-    if bootstrap_pip_version is PipVersion.VENDORED:
+    if bootstrap_pip_version is PipVersion.VENDORED and PipVersion.VENDORED.available:
 
         def resolve_distribution_locations():
             for resolved_distribution in resolver.resolve_requirements(
@@ -555,7 +548,7 @@ def get_pip(
     pip = _PIP.get(installation)
     if pip is None:
         installation.check_python_applies()
-        if installation.version is PipVersion.VENDORED:
+        if installation.version is PipVersion.VENDORED and PipVersion.VENDORED.available:
             pip = _vendored_installation(
                 interpreter=interpreter,
                 resolver=resolver,

pex/pip/tool.py

@@ -365,7 +365,7 @@ def _calculate_resolver_version_args(
         if (
             resolver_version == ResolverVersion.PIP_2020
             and interpreter.version[0] == 2
-            and self.version.version < PipVersion.v22_3.version
+            and PipVersion.VENDORED <= self.version < PipVersion.v22_3
         ):
             yield "--use-feature"
             yield "2020-resolver"
@@ -398,7 +398,7 @@ def _spawn_pip_isolated(
             # We are not interactive.
             "--no-input",
         ]
-        if self.version < PipVersion.v25_0:
+        if PipVersion.VENDORED <= self.version < PipVersion.v25_0:
             # If we want to warn about a version of python we support, we should do it, not pip.
             # That said, the option does nothing in Pip 25.0 and is deprecated and slated for
             # removal.

pex/pip/version.py

@@ -47,7 +47,7 @@ def overridden(cls):
     @staticmethod
     def _to_requirement(
         project_name,  # type: str
-        project_version,  # type: Union[str, Version]
+        project_version=None,  # type: Optional[Union[str, Version]]
     ):
         # type: (...) -> Requirement
 
@@ -63,13 +63,14 @@ def _to_requirement(
     def __init__(
         self,
         version,  # type: str
-        setuptools_version,  # type: str
-        wheel_version,  # type: str
+        setuptools_version=None,  # type: Optional[str]
+        wheel_version=None,  # type: Optional[str]
         requires_python=None,  # type: Optional[str]
         name=None,  # type: Optional[str]
         requirement=None,  # type: Optional[str]
         setuptools_requirement=None,  # type: Optional[str]
         hidden=False,  # type: bool
+        available=True,  # type: bool
     ):
         # type: (...) -> None
         super(PipVersionValue, self).__init__(name or version, enum_type=PipVersionValue)
@@ -87,6 +88,7 @@ def __init__(
         self.wheel_version = wheel_version
         self.wheel_requirement = self._to_requirement("wheel", wheel_version)
         self.hidden = hidden
+        self.available = available
 
     def cache_dir_name(self):
         # type: () -> str
@@ -283,6 +285,7 @@ def latest_compatible(cls, target=None):
         setuptools_requirement="setuptools",
         wheel_version="0.37.1",
         requires_python="<3.12",
+        available=vendor.PIP_SPEC.exists,
     )
 
     v22_2_2 = PipVersionValue(

pex/targets.py

@@ -6,6 +6,7 @@
 import os
 
 from pex.dist_metadata import Constraint, Distribution
+from pex.dist_metadata import requires_python as dist_requires_python
 from pex.interpreter import PythonInterpreter
 from pex.interpreter_implementation import InterpreterImplementation
 from pex.orderedset import OrderedSet
@@ -31,6 +32,22 @@ class RequiresPythonError(Exception):
 
 @attr.s(frozen=True)
 class WheelEvaluation(object):
+    @classmethod
+    def select_best_match(cls, evals):
+        # type: (Iterable[WheelEvaluation]) -> Optional[WheelEvaluation]
+        match = None  # type: Optional[WheelEvaluation]
+        for wheel_eval in evals:
+            if not wheel_eval.applies:
+                continue
+            if match is None:
+                match = wheel_eval
+            elif wheel_eval.best_match and (
+                wheel_eval.best_match.select_higher_rank(match.best_match) == wheel_eval.best_match
+            ):
+                match = wheel_eval
+        return match
+
+    wheel = attr.ib()  # type: str
     tags = attr.ib()  # type: Tuple[Tag, ...]
     best_match = attr.ib()  # type: Optional[RankedTag]
     requires_python = attr.ib()  # type: Optional[SpecifierSet]
@@ -182,20 +199,27 @@ def requirement_applies(
         return False
 
     def wheel_applies(self, wheel):
-        # type: (Distribution) -> WheelEvaluation
+        # type: (Union[str, Distribution]) -> WheelEvaluation
+
         wheel_tags = CompatibilityTags.from_wheel(wheel)
         ranked_tag = self.supported_tags.best_match(wheel_tags)
+        requires_python = (
+            wheel.metadata.requires_python
+            if isinstance(wheel, Distribution)
+            else dist_requires_python(wheel)
+        )
+        wheel_location = wheel.location if isinstance(wheel, Distribution) else wheel
+
         return WheelEvaluation(
+            wheel=wheel_location,
             tags=tuple(wheel_tags),
             best_match=ranked_tag,
-            requires_python=wheel.metadata.requires_python,
+            requires_python=requires_python,
             applies=(
                 ranked_tag is not None
                 and (
-                    not wheel.metadata.requires_python
-                    or self.requires_python_applies(
-                        wheel.metadata.requires_python, source=wheel.location
-                    )
+                    not requires_python
+                    or self.requires_python_applies(requires_python, source=wheel_location)
                 )
             ),
         )
@@ -217,8 +241,15 @@ def __repr__(self):
 class LocalInterpreter(Target):
     @classmethod
     def create(cls, interpreter=None):
-        # type: (Optional[PythonInterpreter]) -> LocalInterpreter
-        python_interpreter = interpreter or PythonInterpreter.get()
+        # type: (Optional[Union[str, PythonInterpreter]]) -> LocalInterpreter
+
+        if not interpreter:
+            python_interpreter = PythonInterpreter.get()
+        elif isinstance(interpreter, PythonInterpreter):
+            python_interpreter = interpreter
+        else:
+            python_interpreter = PythonInterpreter.from_binary(interpreter)
+
         return cls(
             id=python_interpreter.binary.replace(os.sep, ".").lstrip("."),
             platform=python_interpreter.platform,