From 7c807207816c21e705f5919ad25cd826ea52514c Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:35:56 +0900 Subject: [PATCH 01/14] tox.ini: Test against tristan/sboms buildstream branch --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3ecd216..330c0cb 100644 --- a/tox.ini +++ b/tox.ini @@ -58,7 +58,7 @@ setenv = py{39,310,311,312,313}: XDG_CACHE_HOME = {envtmpdir}/cache py{39,310,311,312,313}: XDG_CONFIG_HOME = {envtmpdir}/config py{39,310,311,312,313}: XDG_DATA_HOME = {envtmpdir}/share - !master: BST_VERSION = 2.4.0 + !master: BST_VERSION = ad25c4ba0e6947013cc59c9e6e88a235329292d2 master: BST_VERSION = master allowlist_externals = From b7672faba5cff61ccb8cfd626cba629f1ef6a979 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:06 +0900 Subject: [PATCH 02/14] sources/patch.py: Implement collect_source_info() --- src/buildstream_plugins/sources/patch.py | 33 ++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/buildstream_plugins/sources/patch.py b/src/buildstream_plugins/sources/patch.py index 1eece2b..1f2b1b4 100644 --- a/src/buildstream_plugins/sources/patch.py +++ b/src/buildstream_plugins/sources/patch.py @@ -45,17 +45,41 @@ See `built-in functionality doumentation `_ for details on common configuration options for sources. + + +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The patch source reports the project relative path of the patch file as the *url*. + +Further, the patch source reports the `SourceInfoMedium.LOCAL +`_ +*medium* and the `SourceVersionType.SHA256 +`_ +*version_type*, for which it reports the sha256 checksum of the patch file as the *version*. + +The *guess_version* of a patch source is meaningless, as it is tied instead to +the BuildStream project in which it is contained. """ import os from buildstream import Source, SourceError from buildstream import utils +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except ImportError: + pass + class PatchSource(Source): # pylint: disable=attribute-defined-outside-init - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" BST_REQUIRES_PREVIOUS_SOURCES_STAGE = True @@ -64,13 +88,15 @@ def configure(self, node): self.path = self.node_get_project_path(node.get_scalar("path"), check_is_file=True) self.strip_level = node.get_int("strip-level", default=1) self.fullpath = os.path.join(self.get_project_directory(), self.path) + self.sha256 = None def preflight(self): # Check if patch is installed, get the binary at the same time self.host_patch = utils.get_host_tool("patch") def get_unique_key(self): - return [self.path, utils.sha256sum(self.fullpath), self.strip_level] + self.sha256 = utils.sha256sum(self.fullpath) + return [self.path, self.sha256, self.strip_level] def is_resolved(self): return True @@ -114,6 +140,9 @@ def stage(self, directory): fail="Failed to apply patch {}".format(self.path), ) + def collect_source_info(self): + return [self.create_source_info(self.path, SourceInfoMedium.LOCAL, SourceVersionType.SHA256, self.sha256)] + # Plugin entry point def setup(): From d3289d9d98d977e38595a35f6183fe571f866137 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:13 +0900 Subject: [PATCH 03/14] sources/cargo.py: Implement collect_source_info() --- src/buildstream_plugins/sources/cargo.py | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/buildstream_plugins/sources/cargo.py b/src/buildstream_plugins/sources/cargo.py index ff49cd9..03dd742 100644 --- a/src/buildstream_plugins/sources/cargo.py +++ b/src/buildstream_plugins/sources/cargo.py @@ -61,6 +61,19 @@ See `built-in functionality doumentation `_ for details on common configuration options for sources. + + +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The cargo source reports the URL of the archive crates as the *url* for each crate. + +Further, the cargo source reports the `SourceInfoMedium.REMOTE_FILE +`_ +*medium* and the `SourceVersionType.SHA256 +`_ +*version_type*, for which it reports the checksum of the archive as the *version*. + +The versions extracted from the Cargo.lock at tracking time are used to report the *guess_version*. """ import json @@ -79,6 +92,16 @@ from ._utils import download_file +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except: + pass + # This automatically goes into .cargo/config # @@ -133,6 +156,12 @@ def fetch(self, alias_override=None, **kwargs): "File downloaded from {} has sha256sum '{}', not '{}'!".format(crate_url, sha256, self.sha) ) + def get_source_info(self): + url, _ = self._get_url() + return self.cargo.create_source_info( + url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.sha, version_guess=self.version + ) + ######################################################## # Helper APIs for the Cargo Source to use # ######################################################## @@ -334,7 +363,7 @@ def _get_mirror_file(self, sha=None): class CargoSource(Source): - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" # We need the Cargo.lock file to construct our ref at track time BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True From 8596250d4076287adbd99444b2fa44c3b67eedfd Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:16 +0900 Subject: [PATCH 04/14] sources/git.py: Implement collect_source_info() --- src/buildstream_plugins/sources/git.py | 138 ++++++++++++++++++++++++- 1 file changed, 133 insertions(+), 5 deletions(-) diff --git a/src/buildstream_plugins/sources/git.py b/src/buildstream_plugins/sources/git.py index 63aee45..0643947 100644 --- a/src/buildstream_plugins/sources/git.py +++ b/src/buildstream_plugins/sources/git.py @@ -69,7 +69,8 @@ # url from which they are to be fetched allows you to easily # rebuild the same sources from a different location. This is # especially handy when used with project defined aliases which - # can be redefined at a later time. + # can be redefined at a later time, or overridden with mirrors. + # # You may also explicitly specify whether to check out this # submodule. If 'checkout' is set, it will control whether to # checkout that submodule and recurse into it. It defaults to the @@ -84,6 +85,18 @@ url: upstream:baz.git checkout: False + # Modify the default version guessing pattern + # + # Since 2.5 + # + version-guess-pattern: \'(\\d+)\\.(\\d+)(?:\\.(\\d+))?\' + + # Override the version guessing with an explicit version + # + # Since 2.5 + # + version: 5.9 + # Enable tag tracking. # # This causes the `tags` metadata to be populated automatically @@ -139,8 +152,8 @@ details on common configuration options for sources. -**Configurable Warnings:** - +Configurable Warnings: +~~~~~~~~~~~~~~~~~~~~~~ This plugin provides the following `configurable warnings `_: @@ -158,9 +171,42 @@ - `ref-not-in-track `_ - The provided ref was not found in the provided track in the element's git repository. -""" +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The git source reports the URL of the git repository as the *url*. + +Further, the git source reports the `SourceInfoMedium.GIT +`_ +*medium* and the `SourceVersionType.COMMIT +`_ +*version_type*, for which it reports the git commit sha as the *version*. + +If the ref is found to be in ``git-describe`` format, an attempt to guess the version based on the +git tag portion of the ref will be made for the reporting of the *guess_version*. Control over how +the guess is made or overridden is controlled based on the ``version-guess-pattern`` and ``version`` +configuration attributes described above. + +In order to understand how the ``version-guess-pattern`` works, please refer to the documentation +for `utils.guess_version() `_ + +In the case that a git describe string represents a commit that is beyond the tag portion +of the git describe reference (i.e. the version is not exact), then the number of commits +found beyond the tag will be reported in the ``commit-offset`` field of the *extra_data*. + +.. attention:: + + SourceInfo is **not** reported for submodules. + + This is due to the limitation that the git plugin requires to have the toplevel + module fetched first in order to have knowledge of the commits of the submodules. + + While this does not provide *complete* information on the provenance of sources, + at least one can consider the module to be a comprehensive whole, and the versions + of submodules are deterministically controlled by the version of the main repository. +""" + import os import re import shutil @@ -174,6 +220,16 @@ from buildstream import utils from buildstream.utils import DirectoryExistsError +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except ImportError: + pass + GIT_MODULES = ".gitmodules" EXACT_TAG_PATTERN = r"(?P.*)-0-g(?P.*)" @@ -697,7 +753,7 @@ def _rebuild_git(self, fullpath): class GitSource(Source): # pylint: disable=attribute-defined-outside-init - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" def configure(self, node): ref = node.get_str("ref", None) @@ -711,6 +767,8 @@ def configure(self, node): "ref-format", "track-tags", "tags", + "version", + "version-guess-pattern", ] node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS) @@ -727,6 +785,14 @@ def configure(self, node): self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1) + self.guess_pattern_string = node.get_str("version-guess-pattern", None) + self.guess_pattern = None + if self.guess_pattern_string is not None: + self.guess_pattern = re.compile(self.guess_pattern_string) + + # Get the explicitly set guess_version for the toplevel git repo + self.version = node.get_str("version", None) + # At this point we now know if the source has a ref and/or a track. # If it is missing both then we will be unable to track or build. if self.mirror.ref is None and self.tracking is None: @@ -796,6 +862,13 @@ def get_unique_key(self): if self.submodule_checkout_overrides: key.append({"submodule_checkout_overrides": self.submodule_checkout_overrides}) + # Backwards compatible method of supporting configuration + # attributes which affect SourceInfo generation. + if self.version is not None: + key.append(self.version) + elif self.guess_pattern_string is not None: + key.append(self.guess_pattern_string) + return key def is_resolved(self): @@ -986,6 +1059,60 @@ def validate_cache(self): warning_token=CoreWarnings.REF_NOT_IN_TRACK, ) + # _guess_version() + # + # Guess the version, in the case the ref is recorded in git-describe format + # + # Returns: a three-tuple composed of the following three values: + # - the version (a git sha) + # - the version_guess (as guessed from the tag portion of a git describe string) + # - the number of commits beyond the tag (indicating that the guessed version is in this case inexact) + # + # All returns are either None or strings, only the "sha" is guaranteed to be discovered. + # + def _guess_version(self): + version_guess = self.version + commits = None + commit_sha = None + + string = self.mirror.ref + + main_split = string.rsplit("-g", 1) + if len(main_split) < 2: + # Didn't find the `-g`, assume its a raw git sha as input + commit_sha = string + else: + # Split out the commits and the tag portion (which may also contain '-') + commit_sha = main_split[1] + sub_split = main_split[0].rsplit("-", 1) + + commits = sub_split[1] + if commits == "0": + commits = None + + # Run the guess logic on the tag portion of the git describe string + if version_guess is None: + version_guess = utils.guess_version(sub_split[0], pattern=self.guess_pattern) + + return (commit_sha, version_guess, commits) + + def collect_source_info(self): + extra_data = None + commit_sha, version_guess, commits = self._guess_version() + if commits: + extra_data = {"commit-offset": commits} + + return [ + self.create_source_info( + self.mirror.url, + SourceInfoMedium.GIT, + SourceVersionType.COMMIT, + commit_sha, + version_guess=version_guess, + extra_data=extra_data, + ) + ] + ########################################################### # Local Functions # ########################################################### @@ -1010,6 +1137,7 @@ def _configure_submodules(self, submodules): continue # Allow configuration to override the upstream location of the submodules. submodule.url = self.submodule_overrides.get(submodule.path, submodule.url) + yield submodule # _recurse_submodules(): From 1d55406c427a45bdae9d0ef0baa80f15658ae02b Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:20 +0900 Subject: [PATCH 05/14] sources/docker.py: Implement collect_source_info() --- src/buildstream_plugins/sources/docker.py | 82 +++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/src/buildstream_plugins/sources/docker.py b/src/buildstream_plugins/sources/docker.py index c495915..62c4d76 100644 --- a/src/buildstream_plugins/sources/docker.py +++ b/src/buildstream_plugins/sources/docker.py @@ -57,6 +57,13 @@ # # **Since**: 2.0.1 + # Specify the version to be reported as the *guess_version* when reporting + # SourceInfo + # + # Since: 2.5 + # + version: 1.2 + Note that Docker images may contain device nodes. BuildStream elements cannot contain device nodes so those will be dropped. Any regular files in the /dev directory will also be dropped. @@ -64,6 +71,32 @@ See `built-in functionality doumentation `_ for details on common configuration options for sources. + + +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The docker source reports the URL of the docker registry as the *url*. + +Further, the docker source reports the `SourceInfoMedium.OCI_IMAGE +`_ +*medium* and the `SourceVersionType.OCI_DIGEST +`_ +*version_type*, for which it reports the content digest of the docker image as the *version*. + +Additionally, the docker source reports the docker image name through +the ``image-name`` key of the *extra_data*. + +As such, after removing the scheme from the URL (i.e. remove ``https://``) the same docker +image can be obtained by calling: + +.. code:: bash + + docker pull /@ + +Since the docker source does not have any way to guess what tag is associated +to the digest, or what release version that would mean; the docker source exposes +the ``version`` configuration attribute to allow explicit specification of the +*guess_version*. """ import hashlib @@ -84,6 +117,16 @@ move_atomic, ) +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except ImportError: + pass + _DOCKER_HUB_URL = "https://registry.hub.docker.com" @@ -342,7 +385,7 @@ def mode(self, permission): class DockerSource(Source): # pylint: disable=too-many-instance-attributes - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" # Docker identifies images by a content digest calculated from the image's # manifest. This corresponds well with the concept of a 'ref' in @@ -365,7 +408,8 @@ def configure(self, node): # url is deprecated, but accept it as a valid key so that we can raise # a nicer warning. node.validate_keys( - Source.COMMON_CONFIG_KEYS + ["architecture", "registry-url", "image", "os", "ref", "track", "url"] + Source.COMMON_CONFIG_KEYS + + ["architecture", "registry-url", "image", "os", "ref", "track", "url", "version"] ) if "url" in node and ("image" in node or "registry-url" in node): @@ -391,6 +435,7 @@ def configure(self, node): self.architecture = node.get_str("architecture", "") or default_architecture() self.os = node.get_str("os", "") or default_os() + self.version = node.get_str("version", None) self.digest = None self.load_ref(node) @@ -409,9 +454,16 @@ def preflight(self): def get_unique_key(self): if self.url is not None: - return [self.url, self.digest] + unique_key = [self.url, self.digest] else: - return [self.original_registry_url, self.image, self.digest] + unique_key = [self.original_registry_url, self.image, self.digest] + + # Backwards compatible method of supporting configuration + # attributes which affect SourceInfo generation. + if self.version is not None: + unique_key.append(self.version) + + return unique_key def get_ref(self): return None if self.digest is None else self._digest_to_ref(self.digest) @@ -583,6 +635,28 @@ def stage(self, directory): except (OSError, SourceError, tarfile.TarError) as e: raise SourceError("{}: Error staging source: {}".format(self, e)) from e + def collect_source_info(self): + + # If the image was configured with "url" only rather + # than "registry-url" and "image", then self.image + # at this point will have a leading forward slash "/". + # + # Lets just normalize that problem here, as changing + # the self.image itself can break cache keys. + # + image_name = self.image.lstrip("/") + + return [ + self.create_source_info( + self.registry_url, + SourceInfoMedium.OCI_IMAGE, + SourceVersionType.OCI_DIGEST, + self.digest, + version_guess=self.version, + extra_data={"image-name": image_name}, + ) + ] + @staticmethod def _get_extract_and_remove_files(layer_tar_path): """Return the set of files to remove and extract for a given layer From 0d688cc42df678139dbd6d6ad99385e153d034a8 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:22 +0900 Subject: [PATCH 06/14] sources/bzr.py: Implement collect_source_info() --- src/buildstream_plugins/sources/bzr.py | 53 ++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/src/buildstream_plugins/sources/bzr.py b/src/buildstream_plugins/sources/bzr.py index bceb88b..1eb3b01 100644 --- a/src/buildstream_plugins/sources/bzr.py +++ b/src/buildstream_plugins/sources/bzr.py @@ -53,9 +53,31 @@ # revision number to the one on the tip of the branch specified in 'track'. ref: 6622 + # Specify the version to be reported as the *guess_version* when reporting + # SourceInfo + # + # Since 2.5 + # + version: 1.2 + See `built-in functionality doumentation `_ for details on common configuration options for sources. + + +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The bzr source reports the URL of the bzr repository as the *url*. + +Further, the bzr source reports the `SourceInfoMedium.BZR +`_ +*medium* and the` `SourceVersionType.COMMIT +`_ +*version_type*, for which it reports the bzr revision number as the *version*. + +Since the bzr source does not have a way to know what the release version +corresponds to the revision number, the bzr source exposes the ``version`` configuration +attribute to allow explicit specification of the *guess_version*. """ import os @@ -66,26 +88,44 @@ from buildstream import Source, SourceError from buildstream import utils +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except ImportError: + pass + class BzrSource(Source): # pylint: disable=attribute-defined-outside-init - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" def configure(self, node): - node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS]) + node.validate_keys(["url", "track", "ref", "version", *Source.COMMON_CONFIG_KEYS]) self.original_url = node.get_str("url") self.tracking = node.get_str("track") self.ref = node.get_str("ref", None) self.url = self.translate_url(self.original_url) + self.version = node.get_str("version", None) def preflight(self): # Check if bzr is installed, get the binary at the same time. self.host_bzr = utils.get_host_tool("bzr") def get_unique_key(self): - return [self.original_url, self.tracking, self.ref] + unique_key = [self.original_url, self.tracking, self.ref] + + # Backwards compatible method of supporting configuration + # attributes which affect SourceInfo generation. + if self.version is not None: + unique_key.append(self.version) + + return unique_key def is_cached(self): with self._locked(): @@ -167,6 +207,13 @@ def init_workspace(self, directory): fail="Failed to switch workspace's parent branch to {}".format(url), ) + def collect_source_info(self): + return [ + self.create_source_info( + self.url, SourceInfoMedium.BAZAAR, SourceVersionType.COMMIT, self.ref, version_guess=self.version + ) + ] + # _locked() # # This context manager ensures exclusive access to the From 32945ad406114bb22151d540627f18bfd6273082 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:25 +0900 Subject: [PATCH 07/14] sources/pip.py: Implement collect_source_info() --- src/buildstream_plugins/sources/pip.py | 50 ++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/src/buildstream_plugins/sources/pip.py b/src/buildstream_plugins/sources/pip.py index 1c8546e..95fc6bc 100644 --- a/src/buildstream_plugins/sources/pip.py +++ b/src/buildstream_plugins/sources/pip.py @@ -65,13 +65,40 @@ See `built-in functionality doumentation `_ for details on common configuration options for sources. + + +Reporting `SourceInfo `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The pip source reports the python package index (PyPI) instance as the *url*. + +Further, the pip source reports the `SourceInfoMedium.PYTHON_PACKAGE_INDEX +`_ +*medium* and the `SourceVersionType.INDEXED_VERSION +`_ +*version_type*, for which it reports the tracked package version as the *version* and the *version_guess*. + +Additionally, the pip source reports the package name through +the ``package-name`` key of the *extra_data*. + +The pip source will report one SourceInfo instance for each of the packages discovered in tracking. """ import hashlib import os import re -from buildstream import Source, SourceError, utils +from buildstream import Source, SourceError +from buildstream import utils + +# +# Soft import of buildstream symbols only available in newer versions +# +# The BST_MIN_VERSION will provide a better user experience. +# +try: + from buildstream import SourceInfoMedium, SourceVersionType +except ImportError: + pass _OUTPUT_DIRNAME = ".bst_pip_downloads" _PYPI_INDEX_URL = "https://pypi.org/simple/" @@ -108,7 +135,7 @@ class PipSource(Source): # pylint: disable=attribute-defined-outside-init - BST_MIN_VERSION = "2.0" + BST_MIN_VERSION = "2.5" # We need access to previous sources at track time to use requirements.txt # but not at fetch time as self.ref should contain sufficient information @@ -224,6 +251,25 @@ def stage(self, directory): with self.timed_activity("Staging Python packages", silent_nested=True): utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME)) + def collect_source_info(self): + infos = [] + packages_versions = self.ref.splitlines() + for package_version in packages_versions: + split = package_version.split("==") + package = split[0] + version = split[1] + infos.append( + self.create_source_info( + self.index_url, + SourceInfoMedium.PYTHON_PACKAGE_INDEX, + SourceVersionType.INDEXED_VERSION, + version, + version_guess=version, + extra_data={"package-name": package}, + ) + ) + return infos + # Directory where this source should stage its files # @property From c498d6213dd3bc485bb239e1e9855c2744d772b3 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:28 +0900 Subject: [PATCH 08/14] tests/sources/patch.py: Test patch's collect_source_info() implementation --- tests/sources/patch.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/sources/patch.py b/tests/sources/patch.py index ba6e151..f4f2098 100644 --- a/tests/sources/patch.py +++ b/tests/sources/patch.py @@ -19,6 +19,8 @@ import socket import pytest +from buildstream import _yaml + from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import @@ -212,3 +214,23 @@ def test_patch_strip_level(cli, tmpdir, datafiles): # Test the file.txt was patched and changed with open(os.path.join(checkoutdir, "file.txt"), encoding="utf-8") as f: assert f.read() == "This is text file with superpowers\n" + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) +def test_show_source_info(cli, tmpdir, datafiles): + # Get the source info + project = str(datafiles) + result = cli.run(project=project, args=["show", "--format", "%{name}:\n%{source-info}", "target.bst"]) + result.assert_success() + + # Check the results of the patch source, which will be second in the list after the local file + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("target.bst") + source_info = sources.mapping_at(1) + + assert source_info.get_str("kind") == "patch" + assert source_info.get_str("url") == "file_1.patch" + assert source_info.get_str("medium") == "local" + assert source_info.get_str("version-type") == "sha256" + assert source_info.get_str("version") == "063fbc138d30a2f2ea8f3cd53810824520ff3971d752235c70f17c67fefca0f1" + assert source_info.get_str("version-guess", None) is None From 4fd134e7ef7613b80ad050f49c0e0d13024d5db9 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:31 +0900 Subject: [PATCH 09/14] tests/sources/cargo.py: Test cargo's collect_source_info() implementation --- tests/sources/cargo.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/sources/cargo.py b/tests/sources/cargo.py index 6d68089..7fadab1 100644 --- a/tests/sources/cargo.py +++ b/tests/sources/cargo.py @@ -50,8 +50,27 @@ def generate_project(project_dir): def test_cargo_track_fetch_build(cli, datafiles): project = str(datafiles) generate_project(project) + + # First track result = cli.run(project=project, args=["source", "track", "base64.bst"]) result.assert_success() + + # Now we should be able to get the source info + result = cli.run(project=project, args=["show", "--format", "%{name}:\n%{source-info}", "base64.bst"]) + result.assert_success() + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("base64.bst") + + # Assert the cargo source, which is in the second position after the local source + source_info = sources.mapping_at(1) + assert source_info.get_str("kind") == "cargo" + assert source_info.get_str("url") == "https://static.crates.io/crates/base64/base64-0.22.1.crate" + assert source_info.get_str("medium") == "remote-file" + assert source_info.get_str("version-type") == "sha256" + assert source_info.get_str("version") == "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + assert source_info.get_str("version-guess") == "0.22.1" + + # Now fetch and build result = cli.run(project=project, args=["source", "fetch", "base64.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "base64.bst"]) From 88ba5683826879c04297b25ff98de334d34dd2b5 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:33 +0900 Subject: [PATCH 10/14] tests/sources/git.py: Testing git's collect_source_info() implementation --- tests/sources/git.py | 97 +++++++++++++++++++ .../source-info/elements/describe-custom.bst | 7 ++ .../source-info/elements/describe-offset.bst | 6 ++ .../git/source-info/elements/describe.bst | 6 ++ .../git/source-info/elements/no-describe.bst | 6 ++ .../git/source-info/elements/override.bst | 7 ++ tests/sources/git/source-info/project.conf | 10 ++ 7 files changed, 139 insertions(+) create mode 100644 tests/sources/git/source-info/elements/describe-custom.bst create mode 100644 tests/sources/git/source-info/elements/describe-offset.bst create mode 100644 tests/sources/git/source-info/elements/describe.bst create mode 100644 tests/sources/git/source-info/elements/no-describe.bst create mode 100644 tests/sources/git/source-info/elements/override.bst create mode 100644 tests/sources/git/source-info/project.conf diff --git a/tests/sources/git.py b/tests/sources/git.py index f292a69..5b766a3 100644 --- a/tests/sources/git.py +++ b/tests/sources/git.py @@ -31,6 +31,7 @@ from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import generate_project, generate_element, load_yaml from buildstream._testing import create_repo +from buildstream import _yaml from tests.testutils.site import HAVE_GIT, HAVE_OLD_GIT @@ -1146,3 +1147,99 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +@pytest.mark.parametrize( + "target, expected_kind, expected_url, expected_medium, expected_version_type, expected_version, expected_guess_version, expected_offset", + [ + ( + "no-describe.bst", + "git", + "https://flying-ponies.com/pony.git", + "git", + "commit", + "40af18f96af0c7f1b56f67339ef53e000e738754", + None, + None, + ), + ( + "describe.bst", + "git", + "https://flying-ponies.com/pony.git", + "git", + "commit", + "40af18f96af0c7f1b56f67339ef53e000e738754", + "3.4", + None, + ), + ( + "describe-offset.bst", + "git", + "https://flying-ponies.com/pony.git", + "git", + "commit", + "40af18f96af0c7f1b56f67339ef53e000e738754", + "3.4", + "7", + ), + ( + "describe-custom.bst", + "git", + "https://flying-ponies.com/pony.git", + "git", + "commit", + "40af18f96af0c7f1b56f67339ef53e000e738754", + "3.4.6", + None, + ), + ( + "override.bst", + "git", + "https://flying-ponies.com/pony.git", + "git", + "commit", + "40af18f96af0c7f1b56f67339ef53e000e738754", + "5.5", + None, + ), + ], + ids=["no-describe", "describe", "commit-offset", "custom-pattern", "override"], +) +def test_source_info( + cli, + datafiles, + target, + expected_url, + expected_kind, + expected_medium, + expected_version_type, + expected_version, + expected_guess_version, + expected_offset, +): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", target]) + result.assert_success() + + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence(target) + source_info = sources.mapping_at(0) + + assert source_info.get_str("kind") == expected_kind + assert source_info.get_str("url") == expected_url + assert source_info.get_str("medium") == expected_medium + assert source_info.get_str("version-type") == expected_version_type + assert source_info.get_str("version") == expected_version + + guess_version = source_info.get_str("version-guess", None) + if guess_version or expected_guess_version: + assert guess_version == expected_guess_version + + extra_data = source_info.get_mapping("extra-data", None) + if extra_data: + commit_offset = extra_data.get_str("commit-offset", None) + else: + commit_offset = None + + assert commit_offset == expected_offset diff --git a/tests/sources/git/source-info/elements/describe-custom.bst b/tests/sources/git/source-info/elements/describe-custom.bst new file mode 100644 index 0000000..5b3e29c --- /dev/null +++ b/tests/sources/git/source-info/elements/describe-custom.bst @@ -0,0 +1,7 @@ +kind: import + +sources: +- kind: git + url: https://flying-ponies.com/pony.git + ref: pony-3_4_r6-0-g40af18f96af0c7f1b56f67339ef53e000e738754 + version-guess-pattern: '(\d+)_(\d+)_r(\d+)?' diff --git a/tests/sources/git/source-info/elements/describe-offset.bst b/tests/sources/git/source-info/elements/describe-offset.bst new file mode 100644 index 0000000..e182df5 --- /dev/null +++ b/tests/sources/git/source-info/elements/describe-offset.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: git + url: https://flying-ponies.com/pony.git + ref: 3.4-7-g40af18f96af0c7f1b56f67339ef53e000e738754 diff --git a/tests/sources/git/source-info/elements/describe.bst b/tests/sources/git/source-info/elements/describe.bst new file mode 100644 index 0000000..44655d6 --- /dev/null +++ b/tests/sources/git/source-info/elements/describe.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: git + url: https://flying-ponies.com/pony.git + ref: pony-3.4-0-g40af18f96af0c7f1b56f67339ef53e000e738754 diff --git a/tests/sources/git/source-info/elements/no-describe.bst b/tests/sources/git/source-info/elements/no-describe.bst new file mode 100644 index 0000000..1dfc000 --- /dev/null +++ b/tests/sources/git/source-info/elements/no-describe.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: git + url: https://flying-ponies.com/pony.git + ref: 40af18f96af0c7f1b56f67339ef53e000e738754 diff --git a/tests/sources/git/source-info/elements/override.bst b/tests/sources/git/source-info/elements/override.bst new file mode 100644 index 0000000..abebd28 --- /dev/null +++ b/tests/sources/git/source-info/elements/override.bst @@ -0,0 +1,7 @@ +kind: import + +sources: +- kind: git + url: https://flying-ponies.com/pony.git + ref: pony-3.4-0-g40af18f96af0c7f1b56f67339ef53e000e738754 + version: 5.5 diff --git a/tests/sources/git/source-info/project.conf b/tests/sources/git/source-info/project.conf new file mode 100644 index 0000000..4f1e7d3 --- /dev/null +++ b/tests/sources/git/source-info/project.conf @@ -0,0 +1,10 @@ +# Project config for bst show source-info test +name: test +min-version: 2.5 +element-path: elements + +plugins: +- origin: pip + package-name: buildstream-plugins + sources: + - git From f9e0c15843f88cda62ffe193e0e4bd4257d5a430 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:37:25 +0900 Subject: [PATCH 11/14] tests/sources/docker.py: Test docker's collect_source_info() implementation --- tests/sources/docker.py | 26 +++++++++++++++++++ .../docker/elements/dockerhub-alpine.bst | 1 + 2 files changed, 27 insertions(+) diff --git a/tests/sources/docker.py b/tests/sources/docker.py index ce4881e..9761111 100644 --- a/tests/sources/docker.py +++ b/tests/sources/docker.py @@ -23,6 +23,8 @@ import responses from ruamel.yaml import YAML +from buildstream import _yaml + from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import @@ -89,6 +91,30 @@ def test_handle_network_error(cli, datafiles): result.assert_success() +@pytest.mark.datafiles(DATA_DIR) +def test_show_source_info(cli, datafiles): + + # Get the source info + project = str(datafiles) + result = cli.run(project=project, args=["show", "--format", "%{name}:\n%{source-info}", "dockerhub-alpine.bst"]) + result.assert_success() + + # Check the results + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("dockerhub-alpine.bst") + source_info = sources.mapping_at(0) + + assert source_info.get_str("kind") == "docker" + assert source_info.get_str("url") == "https://registry.hub.docker.com" + assert source_info.get_str("medium") == "oci-image" + assert source_info.get_str("version-type") == "oci-digest" + assert source_info.get_str("version") == "sha256:4b8ffaaa896d40622ac10dc6662204f429f1c8c5714be62a6493a7895f664098" + assert source_info.get_str("version-guess") == "1.2.3" + extra_data = source_info.get_mapping("extra-data", None) + assert extra_data is not None + assert extra_data.get_str("image-name", "library/alpine") + + @pytest.mark.datafiles(DATA_DIR) def test_fetch_duplicate_layers(cli, datafiles): # test that fetching a layer twice does not break the mirror directory diff --git a/tests/sources/docker/elements/dockerhub-alpine.bst b/tests/sources/docker/elements/dockerhub-alpine.bst index 3513087..40a59ce 100644 --- a/tests/sources/docker/elements/dockerhub-alpine.bst +++ b/tests/sources/docker/elements/dockerhub-alpine.bst @@ -6,3 +6,4 @@ sources: image: library/alpine track: latest ref: 4b8ffaaa896d40622ac10dc6662204f429f1c8c5714be62a6493a7895f664098 + version: 1.2.3 From da15f39caeac6a1a22370a87b1c8de760009e53b Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:51 +0900 Subject: [PATCH 12/14] tests/sources/bzr.py: Test bzr's collect_source_info() implementation --- tests/sources/bzr.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py index 180fe15..bbc1741 100644 --- a/tests/sources/bzr.py +++ b/tests/sources/bzr.py @@ -81,3 +81,32 @@ def test_open_bzr_customize(cli, tmpdir, datafiles): stripped_url = source_config.get_str("url").lstrip("file:///") expected_output_str = "checkout of branch: /{}/{}".format(stripped_url, source_config.get_str("track")) assert expected_output_str in str(output) + + +@pytest.mark.datafiles(os.path.join(DATA_DIR)) +def test_show_source_info(cli, tmpdir, datafiles): + project = str(datafiles) + repo = create_repo("bzr", str(tmpdir)) + ref = repo.create(os.path.join(project, "basic")) + + # Write out our test target + source_config = repo.source_config(ref=ref) + source_config["version"] = "1.2.3" + element = {"kind": "import", "sources": [source_config]} + generate_element(project, "target.bst", element) + + # Get the source info + result = cli.run(project=project, args=["show", "--format", "%{name}:\n%{source-info}", "target.bst"]) + result.assert_success() + + # Check the results + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("target.bst") + source_info = sources.mapping_at(0) + + assert source_info.get_str("kind") == "bzr" + assert source_info.get_str("url") == "file://" + repo.repo + assert source_info.get_str("medium") == "bzr" + assert source_info.get_str("version-type") == "commit" + assert source_info.get_str("version") == "1" + assert source_info.get_str("version-guess") == "1.2.3" From c3fe3e5dad6a8a06e17e591fee8a5dd585027ccd Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:36:59 +0900 Subject: [PATCH 13/14] tests/sources/pip_build.py: Test pip's collect_source_info() implementation --- tests/sources/pip_build.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/sources/pip_build.py b/tests/sources/pip_build.py index 363da5b..72fda59 100644 --- a/tests/sources/pip_build.py +++ b/tests/sources/pip_build.py @@ -198,6 +198,7 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo): pypi_repo = os.path.join(project, "files", "pypi-repo") os.makedirs(pypi_repo, exist_ok=True) setup_pypi_repo(mock_packages, pypi_repo) + realpath_repo = os.path.realpath(pypi_repo) element = { "kind": "manual", @@ -206,7 +207,7 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo): {"kind": "local", "path": "files/pip-source"}, { "kind": "pip", - "url": "file://{}".format(os.path.realpath(pypi_repo)), + "url": "file://{}".format(realpath_repo), "requirements-files": ["myreqs.txt"], "packages": dependencies, }, @@ -227,9 +228,39 @@ def test_pip_source_build(cli, datafiles, setup_pypi_repo): result = cli.run(project=project, args=["source", "track", element_name]) assert result.exit_code == 0 + # + # Lets sneak in here and test out that Source.collect_source_info() works as expected + # + # The ref for this generated pip source is: + # + # "app2==0.1\napp_3==0.1\napp_4==0.1\napp_5==0.1\napp_no_6==0.1\napp_no_7==0.1\napp_no_8==0.1\nhellolib==0.1" + # + # Lets just make an assertion on the first dependency app2, which is the second in the list after the local source for this element + # + result = cli.run( + project=project, + silent=True, + args=["show", "--deps", "none", "--format", "element:\n%{source-info}", element_name], + ) + result.assert_success() + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("element") + source_info = sources.mapping_at(1) + assert source_info.get_str("kind") == "pip" + assert source_info.get_str("url") == f"file://{realpath_repo}" + assert source_info.get_str("medium") == "pypi" + assert source_info.get_str("version-type") == "indexed-version" + assert source_info.get_str("version") == "0.1" + assert source_info.get_str("version-guess") == "0.1" + extra_data = source_info.get_mapping("extra-data", None) + assert extra_data is not None + assert extra_data.get_str("package-name", "app2") + + # Go ahead and build result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 + # Use a build shell to assert the output of something we installed result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"]) assert result.exit_code == 0 assert result.output == "Hello App1! This is hellolib\n" From ad3710c10b7a8749b90188493e167cf2819e4146 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 7 May 2025 12:37:05 +0900 Subject: [PATCH 14/14] project.conf: Update min-version to 2.5 --- project.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.conf b/project.conf index bad78fe..07d7996 100644 --- a/project.conf +++ b/project.conf @@ -17,7 +17,7 @@ # that plugins can be loaded via junctions. # name: buildstream-plugins -min-version: 2.0 +min-version: 2.5 plugins: - origin: local