From ab19657af1ad2697441a7a3b27f6931ca3fd30f9 Mon Sep 17 00:00:00 2001 From: konstin Date: Thu, 26 Dec 2024 22:22:56 +0100 Subject: [PATCH 1/7] Support adding custom env vars --- setuptools_rust/_utils.py | 24 ++++++++++++++++++ setuptools_rust/build.py | 42 +++++++++++++++---------------- setuptools_rust/clean.py | 2 +- setuptools_rust/command.py | 10 +++++++- setuptools_rust/extension.py | 11 ++++++-- setuptools_rust/rustc_info.py | 32 ++++++++++++----------- setuptools_rust/setuptools_ext.py | 11 +++++++- tests/test_build.py | 18 +++++++------ 8 files changed, 102 insertions(+), 48 deletions(-) diff --git a/setuptools_rust/_utils.py b/setuptools_rust/_utils.py index b6d9400b..3bec8d23 100644 --- a/setuptools_rust/_utils.py +++ b/setuptools_rust/_utils.py @@ -1,4 +1,28 @@ import subprocess +from typing import Optional + + +class Env: + """Allow using ``functools.lru_cache`` with an environment variable dictionary. + + Dictionaries are unhashable, but ``functools.lru_cache`` needs all parameters to + be hashable, which we solve which a custom ``__hash__``.""" + + env: Optional[dict[str, str]] + + def __init__(self, env: Optional[dict[str, str]]): + self.env = env + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Env): + return False + return self.env == other.env + + def __hash__(self) -> int: + if self.env is not None: + return hash(tuple(sorted(self.env.items()))) + else: + return hash(None) def format_called_process_error( diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index feb93198..8c9da67e 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -24,7 +24,7 @@ from setuptools.command.build_ext import get_abi3_suffix from setuptools.command.install_scripts import install_scripts as CommandInstallScripts -from ._utils import format_called_process_error +from ._utils import format_called_process_error, Env from .command import RustCommand from .extension import Binding, RustBin, RustExtension, Strip from .rustc_info import ( @@ -45,8 +45,8 @@ from setuptools import Command as CommandBdistWheel # type: ignore[assignment] -def _check_cargo_supports_crate_type_option() -> bool: - version = get_rust_version() +def _check_cargo_supports_crate_type_option(env: Optional[Env]) -> bool: + version = get_rust_version(env) if version is None: return False @@ -144,10 +144,10 @@ def run_for_extension(self, ext: RustExtension) -> None: def build_extension( self, ext: RustExtension, forced_target_triple: Optional[str] = None ) -> List["_BuiltModule"]: - target_triple = self._detect_rust_target(forced_target_triple) - rustc_cfgs = get_rustc_cfgs(target_triple) + target_triple = self._detect_rust_target(forced_target_triple, ext.env) + rustc_cfgs = get_rustc_cfgs(target_triple, ext.env) - env = _prepare_build_environment() + env = _prepare_build_environment(ext.env) if not os.path.exists(ext.path): raise FileError( @@ -156,7 +156,7 @@ def build_extension( quiet = self.qbuild or ext.quiet debug = self._is_debug_build(ext) - use_cargo_crate_type = _check_cargo_supports_crate_type_option() + use_cargo_crate_type = _check_cargo_supports_crate_type_option(ext.env) package_id = ext.metadata(quiet=quiet)["resolve"]["root"] if package_id is None: @@ -477,7 +477,7 @@ def _py_limited_api(self) -> _PyLimitedApi: return cast(_PyLimitedApi, bdist_wheel.py_limited_api) def _detect_rust_target( - self, forced_target_triple: Optional[str] = None + self, forced_target_triple: Optional[str], env: Env ) -> Optional[str]: assert self.plat_name is not None if forced_target_triple is not None: @@ -486,14 +486,14 @@ def _detect_rust_target( return forced_target_triple # Determine local rust target which needs to be "forced" if necessary - local_rust_target = _adjusted_local_rust_target(self.plat_name) + local_rust_target = _adjusted_local_rust_target(self.plat_name, env) # Match cargo's behaviour of not using an explicit target if the # target we're compiling for is the host if ( local_rust_target is not None # check for None first to avoid calling to rustc if not needed - and local_rust_target != get_rust_host() + and local_rust_target != get_rust_host(env) ): return local_rust_target @@ -609,7 +609,7 @@ def _replace_vendor_with_unknown(target: str) -> Optional[str]: return "-".join(components) -def _prepare_build_environment() -> Dict[str, str]: +def _prepare_build_environment(env: Env) -> Dict[str, str]: """Prepares environment variables to use when executing cargo build.""" base_executable = None @@ -625,20 +625,18 @@ def _prepare_build_environment() -> Dict[str, str]: # executing python interpreter. bindir = os.path.dirname(executable) - env = os.environ.copy() - env.update( + env_vars = (env.env or os.environ).copy() + env_vars.update( { # disables rust's pkg-config seeking for specified packages, # which causes pythonXX-sys to fall back to detecting the # interpreter from the path. - "PATH": os.path.join(bindir, os.environ.get("PATH", "")), - "PYTHON_SYS_EXECUTABLE": os.environ.get( - "PYTHON_SYS_EXECUTABLE", executable - ), - "PYO3_PYTHON": os.environ.get("PYO3_PYTHON", executable), + "PATH": os.path.join(bindir, env_vars.get("PATH", "")), + "PYTHON_SYS_EXECUTABLE": env_vars.get("PYTHON_SYS_EXECUTABLE", executable), + "PYO3_PYTHON": env_vars.get("PYO3_PYTHON", executable), } ) - return env + return env_vars def _is_py_limited_api( @@ -692,19 +690,19 @@ def _binding_features( _PyLimitedApi = Literal["cp37", "cp38", "cp39", "cp310", "cp311", "cp312", True, False] -def _adjusted_local_rust_target(plat_name: str) -> Optional[str]: +def _adjusted_local_rust_target(plat_name: str, env: Env) -> Optional[str]: """Returns the local rust target for the given `plat_name`, if it is necessary to 'force' a specific target for correctness.""" # If we are on a 64-bit machine, but running a 32-bit Python, then # we'll target a 32-bit Rust build. if plat_name == "win32": - if get_rustc_cfgs(None).get("target_env") == "gnu": + if get_rustc_cfgs(None, env).get("target_env") == "gnu": return "i686-pc-windows-gnu" else: return "i686-pc-windows-msvc" elif plat_name == "win-amd64": - if get_rustc_cfgs(None).get("target_env") == "gnu": + if get_rustc_cfgs(None, env).get("target_env") == "gnu": return "x86_64-pc-windows-gnu" else: return "x86_64-pc-windows-msvc" diff --git a/setuptools_rust/clean.py b/setuptools_rust/clean.py index b8e9ed3a..1fd99e9f 100644 --- a/setuptools_rust/clean.py +++ b/setuptools_rust/clean.py @@ -25,6 +25,6 @@ def run_for_extension(self, ext: RustExtension) -> None: # Execute cargo command try: - subprocess.check_output(args) + subprocess.check_output(args, env=ext.env.env) except Exception: pass diff --git a/setuptools_rust/command.py b/setuptools_rust/command.py index e9f5c2be..1aa5c08d 100644 --- a/setuptools_rust/command.py +++ b/setuptools_rust/command.py @@ -52,8 +52,16 @@ def run(self) -> None: return all_optional = all(ext.optional for ext in self.extensions) + # Use the environment of the first non-optional extension, or the first optional + # extension if there is no non-optional extension. + env = None + for ext in self.extensions: + if ext.env: + env = ext.env + if not ext.optional: + break try: - version = get_rust_version() + version = get_rust_version(env) if version is None: min_version = max( # type: ignore[type-var] filter( diff --git a/setuptools_rust/extension.py b/setuptools_rust/extension.py index e305960e..7bf7bffe 100644 --- a/setuptools_rust/extension.py +++ b/setuptools_rust/extension.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from semantic_version import SimpleSpec -from ._utils import format_called_process_error +from ._utils import format_called_process_error, Env class Binding(IntEnum): @@ -112,6 +112,9 @@ class RustExtension: abort the build process, and instead simply not install the failing extension. py_limited_api: Deprecated. + env: Environment variables to use when calling cargo or rustc (``env=`` + in ``subprocess.Popen``). setuptools-rust may add additional + variables or modify ``PATH``. """ def __init__( @@ -131,6 +134,7 @@ def __init__( native: bool = False, optional: bool = False, py_limited_api: Literal["auto", True, False] = "auto", + env: Optional[Dict[str, str]] = None, ): if isinstance(target, dict): name = "; ".join("%s=%s" % (key, val) for key, val in target.items()) @@ -153,6 +157,7 @@ def __init__( self.script = script self.optional = optional self.py_limited_api = py_limited_api + self.env = Env(env) if native: warnings.warn( @@ -261,7 +266,7 @@ def _metadata(self, cargo: str, quiet: bool) -> "CargoMetadata": # If not quiet, let stderr be inherited stderr = subprocess.PIPE if quiet else None payload = subprocess.check_output( - metadata_command, stderr=stderr, encoding="latin-1" + metadata_command, stderr=stderr, encoding="latin-1", env=self.env.env ) except subprocess.CalledProcessError as e: raise SetupError(format_called_process_error(e)) @@ -319,6 +324,7 @@ def __init__( debug: Optional[bool] = None, strip: Strip = Strip.No, optional: bool = False, + env: Optional[dict[str, str]] = None, ): super().__init__( target=target, @@ -333,6 +339,7 @@ def __init__( optional=optional, strip=strip, py_limited_api=False, + env=env, ) def entry_points(self) -> List[str]: diff --git a/setuptools_rust/rustc_info.py b/setuptools_rust/rustc_info.py index 58dfd42a..732c2919 100644 --- a/setuptools_rust/rustc_info.py +++ b/setuptools_rust/rustc_info.py @@ -5,17 +5,19 @@ from functools import lru_cache from typing import Dict, List, NewType, Optional, TYPE_CHECKING +from ._utils import Env + if TYPE_CHECKING: from semantic_version import Version -def get_rust_version() -> Optional[Version]: # type: ignore[no-any-unimported] +def get_rust_version(env: Optional[Env]) -> Optional[Version]: # type: ignore[no-any-unimported] try: # first line of rustc -Vv is something like # rustc 1.61.0 (fe5b13d68 2022-05-18) from semantic_version import Version - return Version(_rust_version().split(" ")[1]) + return Version(_rust_version(env).split(" ")[1]) except (subprocess.CalledProcessError, OSError): return None @@ -23,11 +25,11 @@ def get_rust_version() -> Optional[Version]: # type: ignore[no-any-unimported] _HOST_LINE_START = "host: " -def get_rust_host() -> str: +def get_rust_host(env: Optional[Env]) -> str: # rustc -Vv has a line denoting the host which cargo uses to decide the # default target, e.g. # host: aarch64-apple-darwin - for line in _rust_version_verbose().splitlines(): + for line in _rust_version_verbose(env).splitlines(): if line.startswith(_HOST_LINE_START): return line[len(_HOST_LINE_START) :].strip() raise PlatformError("Could not determine rust host") @@ -36,9 +38,9 @@ def get_rust_host() -> str: RustCfgs = NewType("RustCfgs", Dict[str, Optional[str]]) -def get_rustc_cfgs(target_triple: Optional[str]) -> RustCfgs: +def get_rustc_cfgs(target_triple: Optional[str], env: Env) -> RustCfgs: cfgs = RustCfgs({}) - for entry in get_rust_target_info(target_triple): + for entry in get_rust_target_info(target_triple, env): maybe_split = entry.split("=", maxsplit=1) if len(maybe_split) == 2: cfgs[maybe_split[0]] = maybe_split[1].strip('"') @@ -49,25 +51,27 @@ def get_rustc_cfgs(target_triple: Optional[str]) -> RustCfgs: @lru_cache() -def get_rust_target_info(target_triple: Optional[str] = None) -> List[str]: +def get_rust_target_info(target_triple: Optional[str], env: Env) -> List[str]: cmd = ["rustc", "--print", "cfg"] if target_triple: cmd.extend(["--target", target_triple]) - output = subprocess.check_output(cmd, text=True) + output = subprocess.check_output(cmd, text=True, env=env.env) return output.splitlines() @lru_cache() -def get_rust_target_list() -> List[str]: - output = subprocess.check_output(["rustc", "--print", "target-list"], text=True) +def get_rust_target_list(env: Env) -> List[str]: + output = subprocess.check_output( + ["rustc", "--print", "target-list"], text=True, env=env.env + ) return output.splitlines() @lru_cache() -def _rust_version() -> str: - return subprocess.check_output(["rustc", "-V"], text=True) +def _rust_version(env: Env) -> str: + return subprocess.check_output(["rustc", "-V"], text=True, env=env.env) @lru_cache() -def _rust_version_verbose() -> str: - return subprocess.check_output(["rustc", "-Vv"], text=True) +def _rust_version_verbose(env: Env) -> str: + return subprocess.check_output(["rustc", "-Vv"], text=True, env=env.env) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index d637b8c3..4cee0e3d 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -15,6 +15,7 @@ from setuptools.command.sdist import sdist from setuptools.dist import Distribution +from ._utils import Env from .build import _get_bdist_wheel_cmd from .extension import Binding, RustBin, RustExtension, Strip @@ -96,7 +97,15 @@ def make_distribution(self) -> None: # # https://doc.rust-lang.org/cargo/commands/cargo-build.html#manifest-options cargo_manifest_args: Set[str] = set() + env: Optional[Env] = None for ext in self.distribution.rust_extensions: + if env is not None: + if ext.env != env: + raise ValueError( + "For vendoring, all extensions must have the same environment variables" + ) + else: + env = ext.env manifest_paths.append(ext.path) if ext.cargo_manifest_args: cargo_manifest_args.update(ext.cargo_manifest_args) @@ -120,7 +129,7 @@ def make_distribution(self) -> None: # set --manifest-path before vendor_path and after --sync to workaround that # See https://docs.rs/clap/latest/clap/struct.Arg.html#method.multiple for detail command.extend(["--manifest-path", manifest_paths[0], vendor_path]) - subprocess.run(command, check=True) + subprocess.run(command, check=True, env=env.env if env else None) cargo_config = _CARGO_VENDOR_CONFIG diff --git a/tests/test_build.py b/tests/test_build.py index 102433d2..eda3b6b6 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -5,20 +5,24 @@ def test_adjusted_local_rust_target_windows_msvc(): with mock.patch( - "setuptools_rust.rustc_info.get_rust_target_info", lambda _: ["target_env=msvc"] + "setuptools_rust.rustc_info.get_rust_target_info", + lambda _plat_name, _env: ["target_env=msvc"], ): - assert _adjusted_local_rust_target("win32") == "i686-pc-windows-msvc" - assert _adjusted_local_rust_target("win-amd64") == "x86_64-pc-windows-msvc" + assert _adjusted_local_rust_target("win32", None) == "i686-pc-windows-msvc" + assert ( + _adjusted_local_rust_target("win-amd64", None) == "x86_64-pc-windows-msvc" + ) def test_adjusted_local_rust_target_windows_gnu(): with mock.patch( - "setuptools_rust.rustc_info.get_rust_target_info", lambda _: ["target_env=gnu"] + "setuptools_rust.rustc_info.get_rust_target_info", + lambda _plat_name, _env: ["target_env=gnu"], ): - assert _adjusted_local_rust_target("win32") == "i686-pc-windows-gnu" - assert _adjusted_local_rust_target("win-amd64") == "x86_64-pc-windows-gnu" + assert _adjusted_local_rust_target("win32", None) == "i686-pc-windows-gnu" + assert _adjusted_local_rust_target("win-amd64", None) == "x86_64-pc-windows-gnu" def test_adjusted_local_rust_target_macos(): with mock.patch("platform.machine", lambda: "x86_64"): - assert _adjusted_local_rust_target("macosx-") == "x86_64-apple-darwin" + assert _adjusted_local_rust_target("macosx-", None) == "x86_64-apple-darwin" From d00026c9a69576152e86cebbb012fda2d1a5fa24 Mon Sep 17 00:00:00 2001 From: konstin Date: Fri, 10 Jan 2025 21:32:58 +0100 Subject: [PATCH 2/7] Review --- setuptools_rust/setuptools_ext.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index 4cee0e3d..2016e35f 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -98,14 +98,19 @@ def make_distribution(self) -> None: # https://doc.rust-lang.org/cargo/commands/cargo-build.html#manifest-options cargo_manifest_args: Set[str] = set() env: Optional[Env] = None + env_source: Optional[str] = None for ext in self.distribution.rust_extensions: if env is not None: if ext.env != env: raise ValueError( - "For vendoring, all extensions must have the same environment variables" + f"For vendoring, all extensions must have the same environment variables, " + f"but {env_source} and {ext.name} differ:\n" + f"{env_source}: {env}\n" + f"{ext.name}: {ext.env}" ) else: env = ext.env + env_source = ext.name manifest_paths.append(ext.path) if ext.cargo_manifest_args: cargo_manifest_args.update(ext.cargo_manifest_args) From 59bf58d5d1ecfd2f27bd1c159979d7428131dd17 Mon Sep 17 00:00:00 2001 From: konstin Date: Fri, 10 Jan 2025 21:55:56 +0100 Subject: [PATCH 3/7] Show ruff format diff on CI failure --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 3565dbff..cc93b46b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -139,7 +139,7 @@ def test_crossenv(session: nox.Session): @nox.session() def ruff(session: nox.Session): session.install("ruff") - session.run("ruff", "format", "--check", ".") + session.run("ruff", "format", "--diff", ".") session.run("ruff", "check", ".") From 1f3073ada9884a4595204866f75bd315dd35c073 Mon Sep 17 00:00:00 2001 From: konstin Date: Fri, 10 Jan 2025 21:57:13 +0100 Subject: [PATCH 4/7] Ruff --- examples/html-py-ever/tests/run_all.py | 8 ++++---- setuptools_rust/setuptools_ext.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/html-py-ever/tests/run_all.py b/examples/html-py-ever/tests/run_all.py index 74ddac19..89faa27a 100755 --- a/examples/html-py-ever/tests/run_all.py +++ b/examples/html-py-ever/tests/run_all.py @@ -46,14 +46,14 @@ def main(): count_py, parse_py, select_py = python(filename, "html.parser") assert count_rs == count_py print(f"{filename} {count_rs} {parse_rs:6f}s") - print(f"Parse py {parse_py:6f}s {parse_py/parse_rs:6.3f}x") - print(f"Select py {select_py:6f}s {select_py/select_rs:6.3f}x") + print(f"Parse py {parse_py:6f}s {parse_py / parse_rs:6.3f}x") + print(f"Select py {select_py:6f}s {select_py / select_rs:6.3f}x") if lxml is not None: count_lxml, parse_lxml, select_lxml = python(filename, "lxml") assert count_rs == count_lxml - print(f"Parse lxml {parse_lxml:6f}s {parse_lxml/parse_rs:6.3f}x") - print(f"Select lxml {select_lxml:6f}s {select_lxml/select_rs:6.3f}x") + print(f"Parse lxml {parse_lxml:6f}s {parse_lxml / parse_rs:6.3f}x") + print(f"Select lxml {select_lxml:6f}s {select_lxml / select_rs:6.3f}x") if __name__ == "__main__": diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index 2016e35f..49e86747 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -52,7 +52,7 @@ def add_rust_extension(dist: Distribution) -> None: ( "no-vendor-crates", None, - "don't vendor Rust crates." "[default; enable with --vendor-crates]", + "don't vendor Rust crates.[default; enable with --vendor-crates]", ), ] ) From 0b31586f9d2bd1bd99dcfc9ad8a9baac333a30f9 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 14 Mar 2025 11:34:51 +0000 Subject: [PATCH 5/7] add ruff lint for subprocess calls --- emscripten/.ruff.toml | 2 ++ pyproject.toml | 7 +++++++ setuptools_rust/_utils.py | 18 +++++++++++++++++- setuptools_rust/build.py | 8 ++++---- setuptools_rust/clean.py | 5 +++-- setuptools_rust/extension.py | 4 ++-- setuptools_rust/rustc_info.py | 12 ++++++------ setuptools_rust/setuptools_ext.py | 5 ++--- 8 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 emscripten/.ruff.toml diff --git a/emscripten/.ruff.toml b/emscripten/.ruff.toml new file mode 100644 index 00000000..abf8eeea --- /dev/null +++ b/emscripten/.ruff.toml @@ -0,0 +1,2 @@ +[lint] +extend-ignore = ["TID251"] diff --git a/pyproject.toml b/pyproject.toml index 09c2409c..1f08d6d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,16 @@ Changelog = "https://github.com/PyO3/setuptools-rust/blob/main/CHANGELOG.md" requires = ["setuptools>=62.4", "setuptools_scm"] build-backend = "setuptools.build_meta" +[tool.ruff.lint] +extend-select = ["TID251"] + [tool.ruff.lint.extend-per-file-ignores] "__init__.py" = ["F403"] +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"subprocess.run".msg = "Use `_utils.run_subprocess` to ensure `env` is passed" +"subprocess.check_output".msg = "Use `_utils.check_subprocess_output` to ensure `env` is passed" + [tool.pytest.ini_options] minversion = "6.0" addopts = "--doctest-modules" diff --git a/setuptools_rust/_utils.py b/setuptools_rust/_utils.py index 3bec8d23..671c0801 100644 --- a/setuptools_rust/_utils.py +++ b/setuptools_rust/_utils.py @@ -1,5 +1,5 @@ import subprocess -from typing import Optional +from typing import Any, Optional class Env: @@ -25,6 +25,22 @@ def __hash__(self) -> int: return hash(None) +def run_subprocess( + *args: Any, env: Optional[Env], **kwargs: Any +) -> subprocess.CompletedProcess: + """Wrapper around subprocess.run that requires a decision to pass an Env object.""" + if env is not None: + kwargs["env"] = env.env + return subprocess.run(*args, **kwargs) # noqa: TID251 # this is a wrapper to implement the rule + + +def check_subprocess_output(*args: Any, env: Optional[Env], **kwargs: Any) -> str: + """Wrapper around subprocess.run that requires a decision to pass an Env object.""" + if env is not None: + kwargs["env"] = env.env + return subprocess.check_output(*args, **kwargs) # noqa: TID251 # this is a wrapper to implement the rule + + def format_called_process_error( e: subprocess.CalledProcessError, *, diff --git a/setuptools_rust/build.py b/setuptools_rust/build.py index 8c9da67e..98b20e41 100644 --- a/setuptools_rust/build.py +++ b/setuptools_rust/build.py @@ -24,7 +24,7 @@ from setuptools.command.build_ext import get_abi3_suffix from setuptools.command.install_scripts import install_scripts as CommandInstallScripts -from ._utils import format_called_process_error, Env +from ._utils import check_subprocess_output, format_called_process_error, Env from .command import RustCommand from .extension import Binding, RustBin, RustExtension, Strip from .rustc_info import ( @@ -252,7 +252,7 @@ def build_extension( # If quiet, capture all output and only show it in the exception # If not quiet, forward all cargo output to stderr stderr = subprocess.PIPE if quiet else None - cargo_messages = subprocess.check_output( + cargo_messages = check_subprocess_output( command, env=env, stderr=stderr, @@ -417,7 +417,7 @@ def install_extension( args.insert(0, "strip") args.append(ext_path) try: - subprocess.check_output(args) + check_subprocess_output(args, env=None) except subprocess.CalledProcessError: pass @@ -566,7 +566,7 @@ def create_universal2_binary(output_path: str, input_paths: List[str]) -> None: # Try lipo first command = ["lipo", "-create", "-output", output_path, *input_paths] try: - subprocess.check_output(command, text=True) + check_subprocess_output(command, env=None, text=True) except subprocess.CalledProcessError as e: output = e.output raise CompileError("lipo failed with code: %d\n%s" % (e.returncode, output)) diff --git a/setuptools_rust/clean.py b/setuptools_rust/clean.py index 1fd99e9f..e1a132c2 100644 --- a/setuptools_rust/clean.py +++ b/setuptools_rust/clean.py @@ -1,6 +1,7 @@ -import subprocess import sys +from setuptools_rust._utils import check_subprocess_output + from .command import RustCommand from .extension import RustExtension @@ -25,6 +26,6 @@ def run_for_extension(self, ext: RustExtension) -> None: # Execute cargo command try: - subprocess.check_output(args, env=ext.env.env) + check_subprocess_output(args, env=ext.env) except Exception: pass diff --git a/setuptools_rust/extension.py b/setuptools_rust/extension.py index 7bf7bffe..65e8b067 100644 --- a/setuptools_rust/extension.py +++ b/setuptools_rust/extension.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from semantic_version import SimpleSpec -from ._utils import format_called_process_error, Env +from ._utils import check_subprocess_output, format_called_process_error, Env class Binding(IntEnum): @@ -265,7 +265,7 @@ def _metadata(self, cargo: str, quiet: bool) -> "CargoMetadata": # If quiet, capture stderr and only show it on exceptions # If not quiet, let stderr be inherited stderr = subprocess.PIPE if quiet else None - payload = subprocess.check_output( + payload = check_subprocess_output( metadata_command, stderr=stderr, encoding="latin-1", env=self.env.env ) except subprocess.CalledProcessError as e: diff --git a/setuptools_rust/rustc_info.py b/setuptools_rust/rustc_info.py index 732c2919..a46f34a7 100644 --- a/setuptools_rust/rustc_info.py +++ b/setuptools_rust/rustc_info.py @@ -5,7 +5,7 @@ from functools import lru_cache from typing import Dict, List, NewType, Optional, TYPE_CHECKING -from ._utils import Env +from ._utils import Env, check_subprocess_output if TYPE_CHECKING: from semantic_version import Version @@ -55,23 +55,23 @@ def get_rust_target_info(target_triple: Optional[str], env: Env) -> List[str]: cmd = ["rustc", "--print", "cfg"] if target_triple: cmd.extend(["--target", target_triple]) - output = subprocess.check_output(cmd, text=True, env=env.env) + output = check_subprocess_output(cmd, env=env, text=True) return output.splitlines() @lru_cache() def get_rust_target_list(env: Env) -> List[str]: - output = subprocess.check_output( - ["rustc", "--print", "target-list"], text=True, env=env.env + output = check_subprocess_output( + ["rustc", "--print", "target-list"], env=env, text=True ) return output.splitlines() @lru_cache() def _rust_version(env: Env) -> str: - return subprocess.check_output(["rustc", "-V"], text=True, env=env.env) + return check_subprocess_output(["rustc", "-V"], env=env, text=True) @lru_cache() def _rust_version_verbose(env: Env) -> str: - return subprocess.check_output(["rustc", "-Vv"], text=True, env=env.env) + return check_subprocess_output(["rustc", "-Vv"], env=env, text=True) diff --git a/setuptools_rust/setuptools_ext.py b/setuptools_rust/setuptools_ext.py index 49e86747..42b155c9 100644 --- a/setuptools_rust/setuptools_ext.py +++ b/setuptools_rust/setuptools_ext.py @@ -1,5 +1,4 @@ import os -import subprocess import sys import sysconfig import logging @@ -15,7 +14,7 @@ from setuptools.command.sdist import sdist from setuptools.dist import Distribution -from ._utils import Env +from ._utils import Env, run_subprocess from .build import _get_bdist_wheel_cmd from .extension import Binding, RustBin, RustExtension, Strip @@ -134,7 +133,7 @@ def make_distribution(self) -> None: # set --manifest-path before vendor_path and after --sync to workaround that # See https://docs.rs/clap/latest/clap/struct.Arg.html#method.multiple for detail command.extend(["--manifest-path", manifest_paths[0], vendor_path]) - subprocess.run(command, check=True, env=env.env if env else None) + run_subprocess(command, env=env, check=True) cargo_config = _CARGO_VENDOR_CONFIG From 0859f234afb087e257501be33d795c4faff013fb Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 14 Mar 2025 11:52:21 +0000 Subject: [PATCH 6/7] add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a5b52e..b6ae06b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Packaging - Drop support for Python 3.8. [#479](https://github.com/PyO3/setuptools-rust/pull/479) - Support free-threaded Python. [#502](https://github.com/PyO3/setuptools-rust/pull/502) +- Support adding custom env vars. [#504](https://github.com/PyO3/setuptools-rust/pull/504) ## 1.10.2 (2024-10-02) ### Fixed From 4faa70502927a55cd76f4c42e73c71dbe772ba6e Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Fri, 14 Mar 2025 11:57:39 +0000 Subject: [PATCH 7/7] allow dict as env in subprocess wrappers --- setuptools_rust/_utils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/setuptools_rust/_utils.py b/setuptools_rust/_utils.py index 671c0801..62c7d72a 100644 --- a/setuptools_rust/_utils.py +++ b/setuptools_rust/_utils.py @@ -1,5 +1,5 @@ import subprocess -from typing import Any, Optional +from typing import Any, Optional, Union, cast class Env: @@ -26,19 +26,23 @@ def __hash__(self) -> int: def run_subprocess( - *args: Any, env: Optional[Env], **kwargs: Any + *args: Any, env: Union[Env, dict[str, str], None], **kwargs: Any ) -> subprocess.CompletedProcess: - """Wrapper around subprocess.run that requires a decision to pass an Env object.""" - if env is not None: - kwargs["env"] = env.env + """Wrapper around subprocess.run that requires a decision to pass env.""" + if isinstance(env, Env): + env = env.env + kwargs["env"] = env return subprocess.run(*args, **kwargs) # noqa: TID251 # this is a wrapper to implement the rule -def check_subprocess_output(*args: Any, env: Optional[Env], **kwargs: Any) -> str: - """Wrapper around subprocess.run that requires a decision to pass an Env object.""" - if env is not None: - kwargs["env"] = env.env - return subprocess.check_output(*args, **kwargs) # noqa: TID251 # this is a wrapper to implement the rule +def check_subprocess_output( + *args: Any, env: Union[Env, dict[str, str], None], **kwargs: Any +) -> str: + """Wrapper around subprocess.run that requires a decision to pass env.""" + if isinstance(env, Env): + env = env.env + kwargs["env"] = env + return cast(str, subprocess.check_output(*args, **kwargs)) # noqa: TID251 # this is a wrapper to implement the rule def format_called_process_error(