From 9c9cd624a820bdb717928d332d699e3fde8a1860 Mon Sep 17 00:00:00 2001 From: Clay Rosenthal Date: Mon, 25 Nov 2024 14:20:04 -0800 Subject: [PATCH 1/2] Adding pep-723 support --- src/shiv/bootstrap/__init__.py | 14 +++++++++-- src/shiv/bootstrap/environment.py | 2 ++ src/shiv/cli.py | 29 ++++++++++++++++++++--- src/shiv/constants.py | 3 ++- src/shiv/inline_script.py | 31 ++++++++++++++++++++++++ test/script/deps.py | 17 ++++++++++++++ test/script/deps_and_python.py | 29 +++++++++++++++++++++++ test/script/min_python.py | 17 ++++++++++++++ test/test_cli.py | 32 +++++++++++++++++++++++-- test/test_inline_script.py | 39 +++++++++++++++++++++++++++++++ tox.ini | 4 +++- 11 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 src/shiv/inline_script.py create mode 100644 test/script/deps.py create mode 100644 test/script/deps_and_python.py create mode 100644 test/script/min_python.py create mode 100644 test/test_inline_script.py diff --git a/src/shiv/bootstrap/__init__.py b/src/shiv/bootstrap/__init__.py index b6ad546..056bd05 100644 --- a/src/shiv/bootstrap/__init__.py +++ b/src/shiv/bootstrap/__init__.py @@ -18,13 +18,14 @@ from .interpreter import execute_interpreter -def run(module): # pragma: no cover +def run(module, inline_script=False): # pragma: no cover """Run a module in a scrubbed environment. If a single pyz has multiple callers, we want to remove these vars as we no longer need them and they can cause subprocesses to fail with a ModuleNotFoundError. :param Callable module: The entry point to invoke the pyz with. + :param bool inline_script: Whether the script is annotated with inline metadata. """ with suppress(KeyError): del os.environ[Environment.MODULE] @@ -35,7 +36,12 @@ def run(module): # pragma: no cover with suppress(KeyError): del os.environ[Environment.CONSOLE_SCRIPT] - sys.exit(module()) + if inline_script: + # inline script will just return a globals dict from the module + module() + sys.exit() + else: + sys.exit(module()) @contextmanager @@ -261,6 +267,10 @@ def bootstrap(): # pragma: no cover if env.entry_point is not None and not env.script: run(import_string(env.entry_point)) + elif env.inline_script is not None: + run(partial(runpy.run_path, str(site_packages / "bin" / env.script), + run_name="__main__"), inline_script=True) + elif env.script is not None: run(partial(runpy.run_path, str(site_packages / "bin" / env.script), run_name="__main__")) diff --git a/src/shiv/bootstrap/environment.py b/src/shiv/bootstrap/environment.py index a8a1404..cb7ea2e 100644 --- a/src/shiv/bootstrap/environment.py +++ b/src/shiv/bootstrap/environment.py @@ -39,6 +39,7 @@ def __init__( no_modify: bool = False, reproducible: bool = False, script: Optional[str] = None, + inline_script: Optional[str] = None, preamble: Optional[str] = None, root: Optional[str] = None, ) -> None: @@ -50,6 +51,7 @@ def __init__( self.no_modify: bool = no_modify self.reproducible: bool = reproducible self.preamble: Optional[str] = preamble + self.inline_script: Optional[str] = inline_script # properties self._entry_point: Optional[str] = entry_point diff --git a/src/shiv/cli.py b/src/shiv/cli.py index 8f5c93e..905d47e 100644 --- a/src/shiv/cli.py +++ b/src/shiv/cli.py @@ -13,6 +13,7 @@ import click from . import __version__ +from .inline_script import parse_script_metadata from . import builder, pip from .bootstrap.environment import Environment from .constants import ( @@ -22,7 +23,8 @@ DISALLOWED_PIP_ARGS, NO_ENTRY_POINT, NO_OUTFILE, - NO_PIP_ARGS_OR_SITE_PACKAGES, + NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES, + SCRIPT_NOT_ANNOTATED, SOURCE_DATE_EPOCH_DEFAULT, SOURCE_DATE_EPOCH_ENV, ) @@ -102,6 +104,12 @@ def copytree(src: Path, dst: Path) -> None: "(default is '/usr/bin/env python3')" ), ) +@click.option( + "--inline-script", + "-s", + help="The path to a PEP-723 inline metadata annotated script to make into the zipapp.", + type=click.Path(exists=True), +) @click.option( "--site-packages", help="The path to an existing site-packages directory to copy into the zipapp.", @@ -161,6 +169,7 @@ def main( entry_point: Optional[str], console_script: Optional[str], python: Optional[str], + inline_script: Optional[str], site_packages: Optional[str], build_id: Optional[str], compressed: bool, @@ -177,8 +186,8 @@ def main( as outlined in PEP 441, but with all their dependencies included! """ - if not pip_args and not site_packages: - sys.exit(NO_PIP_ARGS_OR_SITE_PACKAGES) + if not pip_args and not site_packages and not inline_script: + sys.exit(NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES) if output_file is None: sys.exit(NO_OUTFILE) @@ -213,6 +222,19 @@ def main( # Install dependencies into staged site-packages. pip.install(["--target", tmp_site_packages] + list(pip_args)) + if inline_script: + # Parse the script and add the dependencies to sources + metadata = parse_script_metadata(Path(inline_script).read_text()) + if "script" not in metadata: + sys.exit(SCRIPT_NOT_ANNOTATED) + script_dependencies = metadata["script"].get("dependencies", []) + if script_dependencies: + pip.install(["--target", tmp_site_packages] + list(script_dependencies)) + console_script = Path(inline_script).name + bin_dir = Path(tmp_site_packages, "bin") + bin_dir.mkdir(exist_ok=True) + shutil.copy(Path(inline_script).absolute(), bin_dir / console_script) + if preamble: bin_dir = Path(tmp_site_packages, "bin") bin_dir.mkdir(exist_ok=True) @@ -252,6 +274,7 @@ def main( build_id=build_id, entry_point=entry_point, script=console_script, + inline_script=inline_script, compile_pyc=compile_pyc, extend_pythonpath=extend_pythonpath, shiv_version=__version__, diff --git a/src/shiv/constants.py b/src/shiv/constants.py index 6a7c7ca..599da0b 100644 --- a/src/shiv/constants.py +++ b/src/shiv/constants.py @@ -3,7 +3,8 @@ # errors: DISALLOWED_PIP_ARGS = "\nYou supplied a disallowed pip argument! '{arg}'\n\n{reason}\n" -NO_PIP_ARGS_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS or --site-packages!\n" +NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS, --script, or --site-packages!\n" +SCRIPT_NOT_ANNOTATED = "\nThe provided script is not annotated with PEP-723 metadata!\n" NO_OUTFILE = "\nYou must provide an output file option! (--output-file/-o)\n" NO_ENTRY_POINT = "\nNo entry point '{entry_point}' found in console_scripts or the bin dir!\n" BINPRM_ERROR = "\nShebang is too long, it would exceed BINPRM_BUF_SIZE! Consider /usr/bin/env\n" diff --git a/src/shiv/inline_script.py b/src/shiv/inline_script.py new file mode 100644 index 0000000..c783494 --- /dev/null +++ b/src/shiv/inline_script.py @@ -0,0 +1,31 @@ +import re +try: + # python 3.11+ has tomllib in stdlib + import tomllib # type: ignore +except (ModuleNotFoundError, ImportError): + # python 3.8-3.10, use pip vendored tomli + import pip._vendor.tomli as tomllib # type: ignore + +REGEX = r'(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$' + + +def parse_script_metadata(script: str) -> dict: + """Parses the metadata from a PEP-723 annotated script. + + The metadata is stored in a nested dictionary structure. + The only PEP defined metadata type is "script", which contains the + dependencies of the script and minimum Python version. + + :param script: The text of the script to parse. + """ + + metadata = {} + + for match in re.finditer(REGEX, script): + md_type, content = match.group('type'), ''.join( + line[2:] if line.startswith('# ') else line[1:] + for line in match.group('content').splitlines(keepends=True) + ) + metadata[md_type] = tomllib.loads(content) + + return metadata diff --git a/test/script/deps.py b/test/script/deps.py new file mode 100644 index 0000000..92386ed --- /dev/null +++ b/test/script/deps.py @@ -0,0 +1,17 @@ +# /// script +# dependencies = [ +# "pyyaml<7", +# "rich", +# ] +# /// + +import yaml +from rich.pretty import pprint + +document = """ + hello: world + foo: + bar: 1 + baz: 2 +""" +pprint(yaml.safe_load(document)) diff --git a/test/script/deps_and_python.py b/test/script/deps_and_python.py new file mode 100644 index 0000000..0875926 --- /dev/null +++ b/test/script/deps_and_python.py @@ -0,0 +1,29 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "python-dateutil", +# "rich", +# ] +# /// + +# Code from https://pypi.org/project/python-dateutil/ +# Description: +# Suppose you want to know how much time is left, in years/months/days/etc, +# before the next easter happening on a year with a Friday 13th in August, +# and you want to get today’s date out of the “date” unix system command. + +from dateutil.relativedelta import relativedelta, FR +from dateutil.easter import easter +from dateutil.rrule import rrule, YEARLY +from dateutil.parser import parse +from rich.pretty import pprint + +now = parse("Sat Oct 11 17:13:46 UTC 2003") +today = now.date() +year = rrule(YEARLY, dtstart=now, bymonth=8, bymonthday=13, byweekday=FR)[0].year +rdelta = relativedelta(easter(year), today) + +pprint(f"Today is: {today}") +pprint(f"Year with next Aug 13th on a Friday is: {year}") +pprint(f"How far is the Easter of that year: {rdelta}") +pprint(f"And the Easter of that year is: {today+rdelta}") diff --git a/test/script/min_python.py b/test/script/min_python.py new file mode 100644 index 0000000..c197fd4 --- /dev/null +++ b/test/script/min_python.py @@ -0,0 +1,17 @@ +# /// script +# requires-python = ">=3.11" +# /// + +# No external dependencies + +import json + +document = { + "hello": "world", + "foo": { + "bar": 1, + "baz": 2, + }, +} + +print(json.dumps(document)) diff --git a/test/test_cli.py b/test/test_cli.py index bd1af23..f4f992b 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -12,7 +12,7 @@ from click.testing import CliRunner from shiv.cli import console_script_exists, find_entry_point, main -from shiv.constants import DISALLOWED_ARGS, DISALLOWED_PIP_ARGS, NO_OUTFILE, NO_PIP_ARGS_OR_SITE_PACKAGES +from shiv.constants import DISALLOWED_ARGS, DISALLOWED_PIP_ARGS, NO_OUTFILE, NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES from shiv.info import main as info_main from shiv.pip import install @@ -76,7 +76,7 @@ def test_no_args(self, runner): result = runner([]) assert result.exit_code == 1 - assert NO_PIP_ARGS_OR_SITE_PACKAGES in result.output + assert NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES in result.output def test_no_outfile(self, runner): """This should fail with a warning about not providing an outfile""" @@ -384,3 +384,31 @@ def test_alternate_root_environment_variable(self, runner, package_location, tmp assert proc.returncode == 0 assert "hello" in proc.stdout.decode() assert shiv_root_path.exists() + + @pytest.mark.parametrize( + "script_location, expected_output", + [ + ("test/script/deps_and_python.py", ["2003-10-11", "2004-04-11"]), + ("test/script/min_python.py", ["hello", "world"]), + ("test/script/deps.py", ["foo", "bar"]), + ], + ) + def test_inline_script(self, script_location, expected_output, runner, tmp_path): + """Test that the --inline-script argument works.""" + + output_file = tmp_path / "test.pyz" + result = runner(["--inline-script", script_location, "-o", str(output_file)]) + + # check that the command successfully completed + assert result.exit_code == 0 + + # ensure the created file actually exists + assert output_file.exists() + + # now run the produced zipapp + proc = subprocess.run( + [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ, + ) + + assert proc.returncode == 0 + assert all(expected in proc.stdout.decode() for expected in expected_output) diff --git a/test/test_inline_script.py b/test/test_inline_script.py new file mode 100644 index 0000000..99a97fe --- /dev/null +++ b/test/test_inline_script.py @@ -0,0 +1,39 @@ +from pathlib import Path + +import pytest + +from shiv.inline_script import parse_script_metadata + + +class TestInlineScript: + @pytest.mark.parametrize( + "script_location,expected_metadata", + [ + ("test/script/deps.py", { + "script": { + "dependencies": [ + "pyyaml<7", + "rich" + ], + } + }), + ("test/script/min_python.py", { + "script": { + "requires-python": ">=3.11", + } + }), + ("test/script/deps_and_python.py", { + "script": { + "requires-python": ">=3.11", + "dependencies": [ + "python-dateutil", + "rich", + ], + } + }), + ("test/package/hello/__init__.py", {}), + ], + ) + def test_parse_script_metadata(self, script_location, expected_metadata): + script_text = Path(script_location).read_text() + assert parse_script_metadata(script_text) == expected_metadata diff --git a/tox.ini b/tox.ini index 448554c..15ac2cc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,7 @@ [tox] isolated_build = True -envlist = py38, py39, py310, py311 +envlist = py38, py39, py310, py311, py312 + [gh-actions] python = @@ -8,6 +9,7 @@ python = 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 [testenv] commands= From cea339df256db50cf87dc63fe7680eac53879803 Mon Sep 17 00:00:00 2001 From: Clay Rosenthal Date: Mon, 2 Dec 2024 16:20:07 -0800 Subject: [PATCH 2/2] Adding packaging and python version check --- setup.cfg | 1 + src/shiv/cli.py | 7 +++++++ src/shiv/constants.py | 1 + test/script/deps_and_python.py | 2 +- test/script/min_python.py | 2 +- test/test_inline_script.py | 4 ++-- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2542d75..e33e55b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,7 @@ install_requires = click>=6.7,!=7.0 pip>=9.0.3 setuptools + packaging python_requires = >=3.8 include_package_data = True diff --git a/src/shiv/cli.py b/src/shiv/cli.py index 905d47e..a61ca55 100644 --- a/src/shiv/cli.py +++ b/src/shiv/cli.py @@ -6,7 +6,10 @@ from configparser import ConfigParser from datetime import datetime +from packaging.version import Version +from packaging.specifiers import SpecifierSet from pathlib import Path +from platform import python_version from tempfile import TemporaryDirectory from typing import List, Optional @@ -27,6 +30,7 @@ SCRIPT_NOT_ANNOTATED, SOURCE_DATE_EPOCH_DEFAULT, SOURCE_DATE_EPOCH_ENV, + MIN_PYTHON_VERSION_ERROR, ) @@ -228,6 +232,9 @@ def main( if "script" not in metadata: sys.exit(SCRIPT_NOT_ANNOTATED) script_dependencies = metadata["script"].get("dependencies", []) + min_python = metadata["script"].get("requires-python", None) + if min_python and Version(python_version()) not in SpecifierSet(min_python): + sys.exit(MIN_PYTHON_VERSION_ERROR.format(min_python=min_python, python_version=python_version())) if script_dependencies: pip.install(["--target", tmp_site_packages] + list(script_dependencies)) console_script = Path(inline_script).name diff --git a/src/shiv/constants.py b/src/shiv/constants.py index 599da0b..304267d 100644 --- a/src/shiv/constants.py +++ b/src/shiv/constants.py @@ -5,6 +5,7 @@ DISALLOWED_PIP_ARGS = "\nYou supplied a disallowed pip argument! '{arg}'\n\n{reason}\n" NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS, --script, or --site-packages!\n" SCRIPT_NOT_ANNOTATED = "\nThe provided script is not annotated with PEP-723 metadata!\n" +MIN_PYTHON_VERSION_ERROR = "\nThe provided script requires Python {min_python}, but you are using {python_version}!\n" NO_OUTFILE = "\nYou must provide an output file option! (--output-file/-o)\n" NO_ENTRY_POINT = "\nNo entry point '{entry_point}' found in console_scripts or the bin dir!\n" BINPRM_ERROR = "\nShebang is too long, it would exceed BINPRM_BUF_SIZE! Consider /usr/bin/env\n" diff --git a/test/script/deps_and_python.py b/test/script/deps_and_python.py index 0875926..2db0401 100644 --- a/test/script/deps_and_python.py +++ b/test/script/deps_and_python.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.8" # dependencies = [ # "python-dateutil", # "rich", diff --git a/test/script/min_python.py b/test/script/min_python.py index c197fd4..1f34cbd 100644 --- a/test/script/min_python.py +++ b/test/script/min_python.py @@ -1,5 +1,5 @@ # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.8" # /// # No external dependencies diff --git a/test/test_inline_script.py b/test/test_inline_script.py index 99a97fe..bd4dd99 100644 --- a/test/test_inline_script.py +++ b/test/test_inline_script.py @@ -19,12 +19,12 @@ class TestInlineScript: }), ("test/script/min_python.py", { "script": { - "requires-python": ">=3.11", + "requires-python": ">=3.8", } }), ("test/script/deps_and_python.py", { "script": { - "requires-python": ">=3.11", + "requires-python": ">=3.8", "dependencies": [ "python-dateutil", "rich",