Merge pull request #725 from tiran/update_extra_environ

feat: update_extra_environ hook

Author: mergify[bot]

docs/hooks.rst

@@ -26,6 +26,25 @@ functions with default implementations, so it is only necessary to
 implement the functions needed to make it possible to build the
 package.
 
+Package settings hooks
+----------------------
+
+.. py:currentmodule:: fromager.packagesettings
+
+.. autofromagerhook:: default_update_extra_environ
+
+    The ``update_extra_environ`` can modify the extra environment variables
+    from settings file with dynamic values. The hook must update the
+    ``extra_environ`` dict in-place.
+
+    The hook is called multiple times during a build. The ``version``
+    argument is *None* for ``get_build_backend_dependencies`` and
+    ``get_build_sdist_dependencies``. For ``get_install_dependencies_of_sdist``,
+    ``build_sdist``, and ``build_wheel``, the ``version`` argument
+    contains the resolved version.
+
+    .. versionadded:: 0.60
+
 Dependency hooks
 ----------------
 

e2e/stevedore_override/build/lib/package_plugins/stevedore.py

@@ -9,6 +9,23 @@
 logger = logging.getLogger(__name__)
 
 
+def update_extra_environ(
+    *,
+    ctx: context.WorkContext,
+    req: Requirement,
+    version: Version | None,
+    sdist_root_dir: pathlib.Path,
+    extra_environ: dict[str, str],
+    build_env: build_environment.BuildEnvironment,
+) -> None:
+    """Update extra_environ in-place"""
+    logger.info("update_extra_environ resolved_version=%s", version)
+    marker = ctx.work_dir / "update_extra_environ.txt"
+    with marker.open(encoding="utf-8", mode="a") as f:
+        f.write(f"{version}\n")
+    return None
+
+
 def build_sdist(
     ctx: context.WorkContext,
     extra_environ: dict,

e2e/stevedore_override/src/package_plugins/stevedore.py

@@ -49,6 +49,7 @@ fromager \
 
 EXPECTED_FILES="
 $OUTDIR/sdists-repo/builds/stevedore-*.tar.gz
+$OUTDIR/work-dir/update_extra_environ.txt
 "
 
 pass=true

e2e/test_pep517_build_sdist.sh

@@ -324,6 +324,7 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version:
             install_dependencies = dependencies.get_install_dependencies_of_sdist(
                 ctx=self.ctx,
                 req=req,
+                version=resolved_version,
                 sdist_root_dir=sdist_root_dir,
                 build_env=build_env,
             )

src/fromager/bootstrapper.py

@@ -12,8 +12,15 @@
 import tomlkit
 from packaging.metadata import Metadata
 from packaging.requirements import Requirement
+from packaging.version import Version
 
-from . import build_environment, external_commands, overrides, requirements_file
+from . import (
+    build_environment,
+    external_commands,
+    overrides,
+    packagesettings,
+    requirements_file,
+)
 
 if typing.TYPE_CHECKING:
     from . import context
@@ -110,7 +117,13 @@ def get_build_backend_dependencies(
     logger.debug(
         f"file {build_backend_req_file} does not exist, getting dependencies from hook"
     )
-    extra_environ = pbi.get_extra_environ(build_env=build_env)
+    extra_environ = packagesettings.get_extra_environ(
+        ctx=ctx,
+        req=req,
+        version=None,
+        sdist_root_dir=sdist_root_dir,
+        build_env=build_env,
+    )
     orig_deps = overrides.find_and_invoke(
         req.name,
         "get_build_backend_dependencies",
@@ -175,7 +188,13 @@ def get_build_sdist_dependencies(
     logger.debug(
         f"file {build_sdist_req_file} does not exist, getting dependencies from hook"
     )
-    extra_environ = pbi.get_extra_environ(build_env=build_env)
+    extra_environ = packagesettings.get_extra_environ(
+        ctx=ctx,
+        req=req,
+        version=None,
+        sdist_root_dir=sdist_root_dir,
+        build_env=build_env,
+    )
     orig_deps = overrides.find_and_invoke(
         req.name,
         "get_build_sdist_dependencies",
@@ -227,6 +246,7 @@ def get_install_dependencies_of_sdist(
     *,
     ctx: context.WorkContext,
     req: Requirement,
+    version: Version,
     sdist_root_dir: pathlib.Path,
     build_env: build_environment.BuildEnvironment,
 ) -> set[Requirement]:
@@ -238,14 +258,21 @@ def get_install_dependencies_of_sdist(
         f"getting install requirements for {req} from sdist in {sdist_root_dir}"
     )
     pbi = ctx.package_build_info(req)
+    extra_environ = packagesettings.get_extra_environ(
+        ctx=ctx,
+        req=req,
+        version=version,
+        sdist_root_dir=sdist_root_dir,
+        build_env=build_env,
+    )
     build_dir = pbi.build_dir(sdist_root_dir)
-    extra_environ = pbi.get_extra_environ(build_env=build_env)
     orig_deps = overrides.find_and_invoke(
         req.name,
         "get_install_dependencies_of_sdist",
         default_get_install_dependencies_of_sdist,
         ctx=ctx,
         req=req,
+        version=version,
         sdist_root_dir=sdist_root_dir,
         build_env=build_env,
         extra_environ=extra_environ,
@@ -264,6 +291,7 @@ def default_get_install_dependencies_of_sdist(
     *,
     ctx: context.WorkContext,
     req: Requirement,
+    version: Version,
     sdist_root_dir: pathlib.Path,
     build_env: build_environment.BuildEnvironment,
     extra_environ: dict[str, str],

src/fromager/dependencies.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 import os
 import pathlib
@@ -19,7 +21,7 @@
 from . import overrides
 
 if typing.TYPE_CHECKING:
-    from . import build_environment
+    from . import build_environment, context
 
 logger = logging.getLogger(__name__)
 
@@ -419,7 +421,7 @@ def from_mapping(
         *,
         source: pathlib.Path | str | None,
         has_config: bool,
-    ) -> "PackageSettings":
+    ) -> PackageSettings:
         """Load from a dict"""
         package = Package(canonicalize_name(package, validate=True))
         try:
@@ -436,7 +438,7 @@ def from_string(
         raw_yaml: str,
         *,
         source: pathlib.Path | str | None = None,
-    ) -> "PackageSettings":
+    ) -> PackageSettings:
         """Load from raw yaml string"""
         parsed: typing.Any = yaml.safe_load(raw_yaml)
         if parsed is None:
@@ -449,7 +451,7 @@ def from_string(
         return cls.from_mapping(package, parsed, source=source, has_config=True)
 
     @classmethod
-    def from_file(cls, filename: pathlib.Path) -> "PackageSettings":
+    def from_file(cls, filename: pathlib.Path) -> PackageSettings:
         """Load from file
 
         Raises :exc:`FileNotFound` when the file is not found.
@@ -461,7 +463,7 @@ def from_file(cls, filename: pathlib.Path) -> "PackageSettings":
         return cls.from_string(filename.stem, raw_yaml, source=filename)
 
     @classmethod
-    def from_default(cls, package: str | Package) -> "PackageSettings":
+    def from_default(cls, package: str | Package) -> PackageSettings:
         """Create a default package setting"""
         return cls.from_mapping(package, {}, source="default", has_config=False)
 
@@ -558,7 +560,7 @@ class PackageBuildInfo:
     Public API for PackageSettings with i
     """
 
-    def __init__(self, settings: "Settings", ps: PackageSettings) -> None:
+    def __init__(self, settings: Settings, ps: PackageSettings) -> None:
         self._variant = typing.cast(Variant, settings.variant)
         self._patches_dir = settings.patches_dir
         self._variant_changelog = settings.variant_changelog()
@@ -763,7 +765,7 @@ def get_extra_environ(
         self,
         *,
         template_env: dict[str, str] | None = None,
-        build_env: "build_environment.BuildEnvironment | None" = None,
+        build_env: build_environment.BuildEnvironment | None = None,
     ) -> dict[str, str]:
         """Get extra environment variables for a variant
 
@@ -894,7 +896,7 @@ def from_string(
         raw_yaml: str,
         *,
         source: pathlib.Path | str | None = None,
-    ) -> "SettingsFile":
+    ) -> SettingsFile:
         """Load from raw yaml string"""
         parsed: typing.Any = yaml.safe_load(raw_yaml)
         if parsed is None:
@@ -916,7 +918,7 @@ def from_string(
             ) from err
 
     @classmethod
-    def from_file(cls, filename: pathlib.Path) -> "SettingsFile":
+    def from_file(cls, filename: pathlib.Path) -> SettingsFile:
         """Load from file
 
         Raises :exc:`FileNotFound` when the file is not found.
@@ -958,7 +960,7 @@ def from_files(
         variant: Variant | str,
         patches_dir: pathlib.Path,
         max_jobs: int | None,
-    ) -> "Settings":
+    ) -> Settings:
         """Create Settings from settings.yaml and directory"""
         if settings_file.is_file():
             settings = SettingsFile.from_file(settings_file)
@@ -1082,3 +1084,41 @@ def all_variants(self) -> set[Variant]:
         for ps in self._package_settings.values():
             variants.update(ps.variants.keys())
         return variants
+
+
+def default_update_extra_environ(
+    *,
+    ctx: context.WorkContext,
+    req: Requirement,
+    version: Version | None,
+    sdist_root_dir: pathlib.Path,
+    extra_environ: dict[str, str],
+    build_env: build_environment.BuildEnvironment,
+) -> None:
+    """Update extra_environ in-place"""
+    return None
+
+
+def get_extra_environ(
+    *,
+    ctx: context.WorkContext,
+    req: Requirement,
+    version: Version | None,
+    sdist_root_dir: pathlib.Path,
+    build_env: build_environment.BuildEnvironment,
+) -> dict[str, str]:
+    """Get extra environment variables from settings and update hook"""
+    pbi = ctx.package_build_info(req)
+    extra_environ = pbi.get_extra_environ(build_env=build_env)
+    overrides.find_and_invoke(
+        req.name,
+        "update_extra_environ",
+        default_update_extra_environ,
+        ctx=ctx,
+        req=req,
+        version=version,
+        sdist_root_dir=sdist_root_dir,
+        extra_environ=extra_environ,
+        build_env=build_env,
+    )
+    return extra_environ

src/fromager/packagesettings.py

@@ -22,6 +22,7 @@
     gitutils,
     metrics,
     overrides,
+    packagesettings,
     pyproject,
     requirements_file,
     resolver,
@@ -568,7 +569,13 @@ def build_sdist(
     build_dir = pbi.build_dir(sdist_root_dir)
 
     logger.info(f"building {ctx.variant} source distribution for {req} in {build_dir}")
-    extra_environ = pbi.get_extra_environ(build_env=build_env)
+    extra_environ = packagesettings.get_extra_environ(
+        ctx=ctx,
+        req=req,
+        version=version,
+        sdist_root_dir=sdist_root_dir,
+        build_env=build_env,
+    )
     if req.url:
         # The default approach to making an sdist is to make a tarball from the
         # source directory, since most of the time we got the source directory

src/fromager/sources.py

@@ -24,6 +24,7 @@
     external_commands,
     metrics,
     overrides,
+    packagesettings,
     requirements_file,
     resolver,
     sources,
@@ -270,7 +271,13 @@ def build_wheel(
 
     # add package and variant env vars, package's parallel job vars, and
     # build_env's virtual env vars.
-    extra_environ = pbi.get_extra_environ(build_env=build_env)
+    extra_environ = packagesettings.get_extra_environ(
+        ctx=ctx,
+        req=req,
+        version=version,
+        sdist_root_dir=sdist_root_dir,
+        build_env=build_env,
+    )
 
     if (
         pbi.build_ext_parallel