feat: improve buildspec and dockerfile generation

src/macaron/build_spec_generator/build_spec_generator.py

@@ -98,8 +98,8 @@ def gen_build_spec_for_purl(
             case BuildSpecFormat.DOCKERFILE:
                 try:
                     build_spec_content = gen_dockerfile(build_spec)
-                except ValueError as error:
-                    logger.error("Error while serializing the build spec: %s.", error)
+                except GenerateBuildSpecError as error:
+                    logger.error("Error while generating the build spec: %s.", error)
                     return os.EX_DATAERR
                 build_spec_file_path = os.path.join(build_spec_dir_path, "dockerfile.buildspec")
 

src/macaron/build_spec_generator/common_spec/core.py

@@ -378,7 +378,7 @@ def gen_generic_build_spec(
             "purl": str(purl),
             "language": target_language,
             "build_tools": build_tool_names,
-            "build_commands": [selected_build_command],
+            "build_commands": [selected_build_command] if selected_build_command else [],
         }
     )
     ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl)

src/macaron/build_spec_generator/common_spec/pypi_spec.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module includes build specification and helper classes for PyPI packages."""
@@ -120,6 +120,7 @@ def resolve_fields(self, purl: PackageURL) -> None:
         python_version_set: set[str] = set()
         wheel_name_python_version_list: list[str] = []
         wheel_name_platforms: set[str] = set()
+        version_constraint_set: set[str] = set()
         # Precautionary fallback to default version
         chronologically_likeliest_version: str = defaults.get("heuristic.pypi", "default_setuptools")
 
@@ -155,6 +156,16 @@ def resolve_fields(self, purl: PackageURL) -> None:
                         chronologically_likeliest_version = (
                             pypi_package_json.get_chronologically_suitable_setuptools_version()
                         )
+                        try:
+                            # Get information from the wheel file name.
+                            logger.debug(pypi_package_json.wheel_filename)
+                            _, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename)
+                            for tag in tags:
+                                wheel_name_python_version_list.append(tag.interpreter)
+                                wheel_name_platforms.add(tag.platform)
+                            logger.debug(python_version_set)
+                        except InvalidWheelFilename:
+                            logger.debug("Could not parse wheel file name to extract version")
                 except SourceCodeError:
                     logger.debug("Could not find pure wheel matching this PURL")
 
@@ -214,17 +225,6 @@ def resolve_fields(self, purl: PackageURL) -> None:
                     except (InvalidRequirement, InvalidSpecifier) as error:
                         logger.debug("Malformed requirement encountered %s : %s", requirement, error)
 
-                try:
-                    # Get information from the wheel file name.
-                    logger.debug(pypi_package_json.wheel_filename)
-                    _, _, _, tags = parse_wheel_filename(pypi_package_json.wheel_filename)
-                    for tag in tags:
-                        wheel_name_python_version_list.append(tag.interpreter)
-                        wheel_name_platforms.add(tag.platform)
-                    logger.debug(python_version_set)
-                except InvalidWheelFilename:
-                    logger.debug("Could not parse wheel file name to extract version")
-
                 self.data["language_version"] = list(python_version_set) or wheel_name_python_version_list
 
                 # Use the default build command for pure Python packages.
@@ -238,14 +238,32 @@ def resolve_fields(self, purl: PackageURL) -> None:
             build_backends_set.add("setuptools.build_meta")
 
         logger.debug("Combined build-requires: %s", parsed_build_requires)
+
+        for package, constraint in parsed_build_requires.items():
+            package_requirement = package + constraint
+            python_version_constraints = registry.get_python_requires_for_package_requirement(package_requirement)
+            if python_version_constraints:
+                version_constraint_set.add(python_version_constraints)
+
+        self.data["language_version"] = sorted(version_constraint_set)
+
         self.data["build_requires"] = parsed_build_requires
         self.data["build_backends"] = list(build_backends_set)
 
         if not patched_build_commands:
             # Resolve and patch build commands.
-            selected_build_commands = self.data["build_commands"] or self.get_default_build_commands(
-                self.data["build_tools"]
-            )
+
+            # To ensure that selected_build_commands is never empty, we seed with the fallback
+            # command of python -m build --wheel -n
+            if self.data["build_commands"]:
+                selected_build_commands = self.data["build_commands"]
+            else:
+                self.data["build_commands"] = ["python -m build --wheel -n".split()]
+                selected_build_commands = (
+                    self.get_default_build_commands(self.data["build_tools"]) or self.data["build_commands"]
+                )
+
+            logger.debug(selected_build_commands)
 
             patched_build_commands = (
                 patch_commands(

src/macaron/build_spec_generator/dockerfile/pypi_dockerfile_output.py

@@ -1,16 +1,19 @@
-# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2025 - 2026, Oracle and/or its affiliates. All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
 
 """This module implements the logic to generate a dockerfile from a Python buildspec."""
 
 import logging
+import re
 from textwrap import dedent
 
+from bs4 import BeautifulSoup
 from packaging.specifiers import InvalidSpecifier, SpecifierSet
 from packaging.version import InvalidVersion, Version
 
 from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict
 from macaron.errors import GenerateBuildSpecError
+from macaron.util import send_get_http_raw
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -35,8 +38,12 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
     """
     language_version: str | None = pick_specific_version(buildspec)
     if language_version is None:
-        logger.debug("Could not derive a specific interpreter version.")
-        raise GenerateBuildSpecError("Could not derive specific interpreter version.")
+        raise GenerateBuildSpecError("Could not derive specific interpreter version")
+    try:
+        version = Version(language_version)
+    except InvalidVersion as error:
+        logger.debug("Ran into issue converting %s to a version: %s", language_version, error)
+        raise GenerateBuildSpecError("Derived interpreter version could not be parsed") from error
     backend_install_commands: str = " && ".join(build_backend_commands(buildspec))
     build_tool_install: str = ""
     if (
@@ -49,6 +56,12 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
         build_tool_install = (
             f"pip install {buildspec['build_tools'][0]} && if test -f \"flit.ini\"; then python -m flit.tomlify; fi && "
         )
+    modern_build_command = build_tool_install + " ".join(x for x in buildspec["build_commands"][0])
+    legacy_build_command = (
+        'if test -f "setup.py"; then pip install wheel && python setup.py bdist_wheel; '
+        "else python -m build --wheel -n; fi"
+    )
+
     dockerfile_content = f"""
     #syntax=docker/dockerfile:1.10
     FROM oraclelinux:9
@@ -71,13 +84,22 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
       gcc-c++ gdb lzma glibc-devel libstdc++-devel openssl-devel \\
       readline-devel zlib-devel libzstd-devel libffi-devel bzip2-devel \\
       xz-devel sqlite sqlite-devel sqlite-libs libuuid-devel gdbm-libs \\
-      perf expat expat-devel mpdecimal python3-pip
+      perf expat expat-devel mpdecimal python3-pip \\
+      perl perl-File-Compare
+
+    {openssl_install_commands(version)}
+
+    ENV LD_LIBRARY_PATH=/opt/openssl/lib
+    ENV CPPFLAGS=-I/opt/openssl/include
+    ENV LDFLAGS=-L/opt/openssl/lib
 
     # Build interpreter and create venv
     RUN <<EOF
         cd Python-{language_version}
         ./configure --with-pydebug
         make -s -j $(nproc)
+        make install
+        ./python -m ensurepip
         ./python -m venv /deps
     EOF
 
@@ -98,12 +120,47 @@ def gen_dockerfile(buildspec: BaseBuildSpecDict) -> str:
     EOF
 
     # Run the build
-    RUN {"source /deps/bin/activate && " + build_tool_install + " ".join(x for x in buildspec["build_commands"][0])}
+    RUN source /deps/bin/activate &&  {modern_build_command if version in SpecifierSet(">=3.6") else legacy_build_command}
     """
 
     return dedent(dockerfile_content)
 
 
+def openssl_install_commands(version: Version) -> str:
+    """Appropriate openssl install commands for a given CPython version.
+
+    Parameters
+    ----------
+    version: Version
+        CPython version we are trying to build
+
+    Returns
+    -------
+    str
+       Install commands for the corresponding openssl version
+    """
+    # As per https://peps.python.org/pep-0644, all Python >= 3.10 requires at least OpenSSL 1.1.1,
+    # and 3.6 to 3.9 can be compiled with OpenSSL 1.1.1. Therefore, we compile as below:
+    if version in SpecifierSet(">=3.6"):
+        openssl_version = "1.1.1w"
+        source_url = "https://www.openssl.org/source/old/1.1.1/openssl-1.1.1w.tar.gz"
+    # From the same document, "Python versions 3.6 to 3.9 are compatible with OpenSSL 1.0.2,
+    # 1.1.0, and 1.1.1". As an attempt to generalize for any >= 3.3, we use OpenSSL 1.0.2.
+    else:
+        openssl_version = "1.0.2u"
+        source_url = "https://www.openssl.org/source/old/1.0.2/openssl-1.0.2u.tar.gz"
+
+    return f"""# Build OpenSSL {openssl_version}
+    RUN <<EOF
+        wget {source_url}
+        tar xzf openssl-{openssl_version}.tar.gz
+        cd openssl-{openssl_version}
+        ./config --prefix=/opt/openssl --openssldir=/opt/openssl shared zlib
+        make -j"$(nproc)"
+        make install_sw
+    EOF"""
+
+
 def pick_specific_version(buildspec: BaseBuildSpecDict) -> str | None:
     """Find the latest python interpreter version satisfying inferred constraints.
 
@@ -118,29 +175,111 @@ def pick_specific_version(buildspec: BaseBuildSpecDict) -> str | None:
         String in format major.minor.patch for the latest valid Python
         interpreter version, or None if no such version can be found.
     """
-    # We can most smoothly rebuild Python 3.0.0 and above on OL
-    version_set = SpecifierSet(">=3.0.0")
+    # We cannot create virtual environments for Python versions <= 3.3.0, as
+    # it did not exist back then
+    version_set = SpecifierSet(">=3.4.0")
     for version in buildspec["language_version"]:
         try:
             version_set &= SpecifierSet(version)
         except InvalidSpecifier as error:
-            logger.debug("Malformed interpreter version encountered: %s (%s)", version, error)
-            return None
+            logger.debug("Non-standard interpreter version encountered: %s (%s)", version, error)
+            # Whilst the Python tags specify interpreter implementation
+            # as well as version, with no standard way to parse out the
+            # implementation, we can attempt to heuristically:
+            try_parse_version = infer_interpreter_version(version)
+            if try_parse_version:
+                try:
+                    version_set &= SpecifierSet(f">={try_parse_version}")
+                except InvalidSpecifier as error_for_retry:
+                    logger.debug("Could not parse interpreter version from: %s (%s)", version, error_for_retry)
 
-    # Now to get the latest acceptable one, we can step through all interpreter
+    logger.debug(version_set)
+
+    # Now to get the earliest acceptable one, we can step through all interpreter
     # versions. For the most accurate result, we can query python.org for a
-    # list of all versions, but for now we can approximate by stepping down
-    # through every minor version from 3.14.0 to 3.0.0
-    for minor in range(14, -1, -1):
+    # list of all versions, but for now we can approximate by stepping up
+    # through every minor version from 3.3.0 to 3.14.0
+    for minor in range(3, 15, 1):
         try:
             if Version(f"3.{minor}.0") in version_set:
-                return f"3.{minor}.0"
+                return get_latest_patch(3, minor)
         except InvalidVersion as error:
             logger.debug("Ran into issue converting %s to a version: %s", minor, error)
             return None
     return None
 
 
+def infer_interpreter_version(tag: str) -> str | None:
+    """Infer interpreter version from Python-tag.
+
+    Parameters
+    ----------
+    tag: Python-tag, likely inferred from wheel name.
+
+
+    Returns
+    -------
+    str: interpreter version inferred from Python-tag
+    """
+    # We will parse the interpreter version of CPython or just
+    # whatever generic Python version is specified.
+    pattern = re.compile(r"^(py|cp)(\d{1,3})$")
+    parsed_tag = pattern.match(tag)
+    if parsed_tag:
+        digits = parsed_tag.group(2)
+        # As match succeeded len(digits) \in {1,2,3}
+        if len(digits) == 1:
+            return parsed_tag.group(2)
+        return f"{digits[0]}.{digits[1:]}"
+    return None
+
+
+def get_latest_patch(major: int, minor: int) -> str:
+    """Given major and minor interpreter version, return latest CPython patched version.
+
+    Parameters
+    ----------
+    major: int
+        Major component of version
+    minor: int
+        Minor component of version
+
+    Returns
+    -------
+    str
+        Full major.minor.patch version string corresponding to latest
+        patch for input major and minor.
+    """
+    # We install CPython source
+    response = send_get_http_raw("https://www.python.org/ftp/python/")
+    if not response:
+        raise GenerateBuildSpecError("Failed to fetch index of CPython versions.")
+    html = response.content.decode("utf-8")
+    soup = BeautifulSoup(html, "html.parser")
+    latest_patch: Version | None = None
+
+    # Versions can most reliably be found in anchor tags like:
+    # <a href="{Version}/"> {Version}/ </a>
+    for anchor in soup.find_all("a", href=True):
+        # Get text enclosed in the anchor tag stripping spaces.
+        text = anchor.get_text(strip=True)
+        sanitized_text = text.rstrip("/")
+        # Try to convert to a version.
+        try:
+            parsed_version = Version(sanitized_text)
+            if parsed_version.major == major and parsed_version.minor == minor:
+                if latest_patch is None or parsed_version > latest_patch:
+                    latest_patch = parsed_version
+        except InvalidVersion:
+            # Try the next tag
+            continue
+
+    if not latest_patch:
+        raise GenerateBuildSpecError(f"Failed to infer latest patch for CPython {major}.{minor}")
+
+    return str(latest_patch)
+
+
 def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]:
     """Generate the installation commands for each inferred build backend.
 
@@ -158,7 +297,10 @@ def build_backend_commands(buildspec: BaseBuildSpecDict) -> list[str]:
         return []
     commands: list[str] = []
     for backend, version_constraint in buildspec["build_requires"].items():
-        commands.append(f'/deps/bin/pip install "{backend}{version_constraint}"')
+        if backend == "setuptools":
+            commands.append("/deps/bin/pip install --upgrade setuptools")
+        else:
+            commands.append(f'/deps/bin/pip install "{backend}{version_constraint}"')
     # For a stable order on the install commands
     commands.sort()
     return commands