Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/macaron/build_spec_generator/build_spec_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ def gen_build_spec_for_purl(
case BuildSpecFormat.DOCKERFILE:
try:
build_spec_content = gen_dockerfile(build_spec)
except ValueError as error:
logger.error("Error while serializing the build spec: %s.", error)
except GenerateBuildSpecError as error:
logger.error("Error while generating the build spec: %s.", error)
return os.EX_DATAERR
build_spec_file_path = os.path.join(build_spec_dir_path, "dockerfile.buildspec")

Expand Down
5 changes: 4 additions & 1 deletion src/macaron/build_spec_generator/common_spec/base_spec.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

"""This module includes base build specification and helper classes."""
Expand Down Expand Up @@ -81,6 +81,9 @@ class BaseBuildSpecDict(TypedDict, total=False):
#: be a list of these that were used in building the wheel alongside their version.
build_backends: NotRequired[list[str]]

#: Flag to indicate if the artifact includes binaries.
has_binaries: NotRequired[bool]


class BaseBuildSpec(ABC):
"""Abstract base class for build specification behavior and field resolution."""
Expand Down
2 changes: 1 addition & 1 deletion src/macaron/build_spec_generator/common_spec/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ def gen_generic_build_spec(
"purl": str(purl),
"language": target_language,
"build_tools": build_tool_names,
"build_commands": [selected_build_command],
"build_commands": [selected_build_command] if selected_build_command else [],
}
)
ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl)
Expand Down
115 changes: 79 additions & 36 deletions src/macaron/build_spec_generator/common_spec/pypi_spec.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

"""This module includes build specification and helper classes for PyPI packages."""
Expand All @@ -17,7 +17,7 @@
from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands
from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict
from macaron.config.defaults import defaults
from macaron.errors import GenerateBuildSpecError, SourceCodeError
from macaron.errors import GenerateBuildSpecError, SourceCodeError, WheelTagError
from macaron.json_tools import json_extract
from macaron.slsa_analyzer.package_registry import pypi_registry
from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo
Expand Down Expand Up @@ -114,9 +114,9 @@ def resolve_fields(self, purl: PackageURL) -> None:

pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info)
patched_build_commands: list[list[str]] = []
build_requires_set: set[str] = set()
build_backends_set: set[str] = set()
parsed_build_requires: dict[str, str] = {}
sdist_build_requires: dict[str, str] = {}
python_version_set: set[str] = set()
wheel_name_python_version_list: list[str] = []
wheel_name_platforms: set[str] = set()
Expand All @@ -134,8 +134,16 @@ def resolve_fields(self, purl: PackageURL) -> None:
if py_version := json_extract(release, ["requires_python"], str):
python_version_set.add(py_version.replace(" ", ""))

self.data["has_binaries"] = not pypi_package_json.has_pure_wheel()

if self.data["has_binaries"]:
logger.debug("Can not find a pure wheel")
else:
logger.debug("Found pure wheel matching this PURL")

try:
with pypi_package_json.wheel():
# Argument called download_binaries
with pypi_package_json.wheel(download_binaries=self.data["has_binaries"]):
logger.debug("Wheel at %s", pypi_package_json.wheel_path)
# Should only have .dist-info directory.
logger.debug("It has directories %s", ",".join(os.listdir(pypi_package_json.wheel_path)))
Expand All @@ -155,8 +163,20 @@ def resolve_fields(self, purl: PackageURL) -> None:
chronologically_likeliest_version = (
pypi_package_json.get_chronologically_suitable_setuptools_version()
)
try:
# Get information from the wheel file name.
logger.debug(pypi_package_json.wheel_filename)
_, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename)
for tag in tags:
wheel_name_python_version_list.append(tag.interpreter)
wheel_name_platforms.add(tag.platform)
logger.debug(python_version_set)
except InvalidWheelFilename:
logger.debug("Could not parse wheel file name to extract version")
except WheelTagError:
logger.debug("Can not analyze non-pure wheels")
except SourceCodeError:
logger.debug("Could not find pure wheel matching this PURL")
logger.debug("Could not download wheel matching this PURL")

logger.debug("From .dist_info:")
logger.debug(parsed_build_requires)
Expand All @@ -169,27 +189,32 @@ def resolve_fields(self, purl: PackageURL) -> None:
content = tomli.loads(pyproject_content.decode("utf-8"))
requires = json_extract(content, ["build-system", "requires"], list)
if requires:
build_requires_set.update(elem.replace(" ", "") for elem in requires)
for requirement in requires:
self.add_parsed_requirement(sdist_build_requires, requirement)
# If we cannot find `requires` in `[build-system]`, we lean on the fact that setuptools
# was the de-facto build tool, and infer a setuptools version to include.
else:
build_requires_set.add(f"setuptools=={chronologically_likeliest_version}")
self.add_parsed_requirement(
sdist_build_requires, f"setuptools=={chronologically_likeliest_version}"
)
backend = json_extract(content, ["build-system", "build-backend"], str)
if backend:
build_backends_set.add(backend.replace(" ", ""))
python_version_constraint = json_extract(content, ["project", "requires-python"], str)
if python_version_constraint:
python_version_set.add(python_version_constraint.replace(" ", ""))
self.apply_tool_specific_inferences(build_requires_set, python_version_set, content)
self.apply_tool_specific_inferences(sdist_build_requires, python_version_set, content)
logger.debug(
"After analyzing pyproject.toml from the sdist: build-requires: %s, build_backend: %s",
build_requires_set,
sdist_build_requires,
build_backends_set,
)
# Here we have successfully analyzed the pyproject.toml file. Now, if we have a setup.py/cfg,
# we also need to infer a setuptools version to infer.
if pypi_package_json.file_exists("setup.py") or pypi_package_json.file_exists("setup.cfg"):
build_requires_set.add(f"setuptools=={chronologically_likeliest_version}")
self.add_parsed_requirement(
sdist_build_requires, f"setuptools=={chronologically_likeliest_version}"
)
except TypeError as error:
logger.debug(
"Found a type error while reading the pyproject.toml file from the sdist: %s", error
Expand All @@ -200,30 +225,20 @@ def resolve_fields(self, purl: PackageURL) -> None:
logger.debug("No pyproject.toml found: %s", error)
# Here we do not have a pyproject.toml file. Instead, we lean on the fact that setuptools
# was the de-facto build tool, and infer a setuptools version to include.
build_requires_set.add(f"setuptools=={chronologically_likeliest_version}")
self.add_parsed_requirement(
sdist_build_requires, f"setuptools=={chronologically_likeliest_version}"
)
except SourceCodeError as error:
logger.debug("No source distribution found: %s", error)

logger.debug("After complete analysis of the sdist:")
logger.debug(sdist_build_requires)

# Merge in pyproject.toml information only when the wheel dist_info does not contain the same.
# Hatch is an interesting example of this merge being required.
for requirement in build_requires_set:
try:
parsed_requirement = Requirement(requirement)
if parsed_requirement.name not in parsed_build_requires:
parsed_build_requires[parsed_requirement.name] = str(parsed_requirement.specifier)
except (InvalidRequirement, InvalidSpecifier) as error:
logger.debug("Malformed requirement encountered %s : %s", requirement, error)

try:
# Get information from the wheel file name.
logger.debug(pypi_package_json.wheel_filename)
_, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename)
for tag in tags:
wheel_name_python_version_list.append(tag.interpreter)
wheel_name_platforms.add(tag.platform)
logger.debug(python_version_set)
except InvalidWheelFilename:
logger.debug("Could not parse wheel file name to extract version")
for requirement_name, specifier in sdist_build_requires.items():
if requirement_name not in parsed_build_requires:
parsed_build_requires[requirement_name] = specifier

self.data["language_version"] = list(python_version_set) or wheel_name_python_version_list

Expand All @@ -243,9 +258,18 @@ def resolve_fields(self, purl: PackageURL) -> None:

if not patched_build_commands:
# Resolve and patch build commands.
selected_build_commands = self.data["build_commands"] or self.get_default_build_commands(
self.data["build_tools"]
)

# To ensure that selected_build_commands is never empty, we seed with the fallback
# command of python -m build --wheel -n
if self.data["build_commands"]:
selected_build_commands = self.data["build_commands"]
else:
self.data["build_commands"] = ["python -m build --wheel -n".split()]
selected_build_commands = (
self.get_default_build_commands(self.data["build_tools"]) or self.data["build_commands"]
)

logger.debug(selected_build_commands)

patched_build_commands = (
patch_commands(
Expand All @@ -259,16 +283,34 @@ def resolve_fields(self, purl: PackageURL) -> None:

self.data["build_commands"] = patched_build_commands

def add_parsed_requirement(self, build_requirements: dict[str, str], requirement: str) -> None:
"""
Parse a requirement string and add it to build_requirements, doing appropriate error handling.

Parameters
----------
build_requirements: dict[str,str]
Dictionary of build requirements to populate.
requirement: str
Requirement string to parse.
"""
try:
parsed_requirement = Requirement(requirement)
if parsed_requirement.name not in build_requirements:
build_requirements[parsed_requirement.name] = str(parsed_requirement.specifier)
except (InvalidRequirement, InvalidSpecifier) as error:
logger.debug("Malformed requirement encountered %s : %s", requirement, error)

def apply_tool_specific_inferences(
self, build_requires_set: set[str], python_version_set: set[str], pyproject_contents: dict[str, Any]
self, build_requirements: dict[str, str], python_version_set: set[str], pyproject_contents: dict[str, Any]
) -> None:
"""
Based on build tools inferred, look into the pyproject.toml for related additional dependencies.

Parameters
----------
build_requires_set: set[str]
Set of build requirements to populate.
build_requirements: dict[str,str]
Dictionary of build requirements to populate.
python_version_set: set[str]
Set of compatible interpreter versions to populate.
pyproject_contents: dict[str, Any]
Expand All @@ -283,7 +325,8 @@ def apply_tool_specific_inferences(
for _, section in hatch_build_hooks.items():
dependencies = section.get("dependencies")
if dependencies:
build_requires_set.update(elem.replace(" ", "") for elem in dependencies)
for requirement in dependencies:
self.add_parsed_requirement(build_requirements, requirement)
# If we have flit as a build_tool, we will check if the legacy header [tool.flit.metadata] exists,
# and if so, check to see if we can use its "requires-python".
if "flit" in self.data["build_tools"]:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

"""This module implements the logic to generate a dockerfile from a Python buildspec."""

import logging
import re
from textwrap import dedent

from packaging.specifiers import InvalidSpecifier, SpecifierSet
Expand Down Expand Up @@ -33,10 +34,11 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
GenerateBuildSpecError
Raised if dockerfile cannot be generated.
"""
if buildspec["has_binaries"]:
raise GenerateBuildSpecError("We currently do not support generating a dockerfile for non-pure Python packages")
language_version: str | None = pick_specific_version(buildspec)
if language_version is None:
logger.debug("Could not derive a specific interpreter version.")
raise GenerateBuildSpecError("Could not derive specific interpreter version.")
raise GenerateBuildSpecError("Could not derive specific interpreter version")
backend_install_commands: str = " && ".join(build_backend_commands(buildspec))
build_tool_install: str = ""
if (
Expand Down Expand Up @@ -124,8 +126,18 @@ def pick_specific_version(buildspec: BaseBuildSpecDict) -> str | None:
try:
version_set &= SpecifierSet(version)
except InvalidSpecifier as error:
logger.debug("Malformed interpreter version encountered: %s (%s)", version, error)
return None
logger.debug("Non-standard interpreter version encountered: %s (%s)", version, error)
# Whilst the Python tags specify interpreter implementation
# as well as version, with no standard way to parse out the
# implementation, we can attempt to heuristically:
try_parse_version = infer_interpreter_version(version)
if try_parse_version:
try:
version_set &= SpecifierSet(f">={try_parse_version}")
except InvalidSpecifier as error_for_retry:
logger.debug("Could not parse interpreter version from: %s (%s)", version, error_for_retry)

logger.debug(version_set)

# Now to get the latest acceptable one, we can step through all interpreter
# versions. For the most accurate result, we can query python.org for a
Expand All @@ -141,6 +153,31 @@ def pick_specific_version(buildspec: BaseBuildSpecDict) -> str | None:
return None


def infer_interpreter_version(tag: str) -> str | None:
"""Infer interpreter version from Python-tag.

Parameters
----------
tag: Python-tag, likely inferred from wheel name.


Returns
-------
str: interpreter version inferred from Python-tag
"""
# We will parse the interpreter version of CPython or just
# whatever generic Python version is specified.
pattern = re.compile(r"^(py|cp)(\d{1,3})$")
parsed_tag = pattern.match(tag)
if parsed_tag:
digits = parsed_tag.group(2)
# As match succeeded len(digits) \in {1,2,3}
if len(digits) == 1:
return parsed_tag.group(2)
return f"{digits[0]}.{digits[1:]}"
return None


def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]:
"""Generate the installation commands for each inferred build backend.

Expand Down
6 changes: 5 additions & 1 deletion src/macaron/errors.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2023 - 2025, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2023 - 2026, Oracle and/or its affiliates. All rights reserved.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

"""This module contains error classes for Macaron."""
Expand Down Expand Up @@ -129,3 +129,7 @@ class QueryMacaronDatabaseError(Exception):

class GenerateBuildSpecError(Exception):
"""Happens when there is an unexpected error while generating the build spec file."""


class WheelTagError(MacaronError):
"""Happens when a Python wheel with unsupported tags is requested for analysis."""
Loading