From 9c0b554c5e2dece09005f0214f4378e59fbe195d Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 14 May 2025 16:25:04 +0900 Subject: [PATCH 1/2] Implement loading of user provided provenance information With the recent addition of SourceInfo etc, we are closer to generating automated SBoMs, and to a reasonable degree, everything that should be delegated to plugins has been delegated via the Source.collect_source_info() and SourceFetcher.get_source_info() abstract methods, and the utility function utils.guess_version() for version guessing. This leaves some information which only users can explicitly provide in their buildstream projects. This patch adds some source level provenance information which users can use to contribute to the SourceInfo. Explanation of the changes in this patch: * types.py: Add internal _SourceProvenance object This validates and parses user provided source provenance information. * _loader/metasource.py: Now carry the _SourceProvenance * _loader/types.py: Add the Symbol.PROVENANCE for consistency, as the element and loader are using this to parse common format related symbols. * element.py: Load the _SourceProvenance from source nodes and pass those through to the MetaSource constructor * source.py: - Add the "homepage" and "issue-tracker" user provided source provenance attributes to SourceInfo, and include those in the SourceInfo serialization. - Automatically pass these attributes on to SourceInfo objects constructed with Source.create_source_info(), so that there is no plugin participation required - Update documentation about the addition of the `provenance` dictionary since 2.5 - Adhere to the new MetaSource constructor in `Source.__clone_for_uri()` --- src/buildstream/_loader/metasource.py | 7 ++- src/buildstream/_loader/types.py | 1 + src/buildstream/element.py | 22 ++++++-- src/buildstream/source.py | 75 ++++++++++++++++++++++++--- src/buildstream/types.py | 36 +++++++++++++ 5 files changed, 129 insertions(+), 12 deletions(-) diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py index 8df034227..7ab069175 100644 --- a/src/buildstream/_loader/metasource.py +++ b/src/buildstream/_loader/metasource.py @@ -26,14 +26,17 @@ class MetaSource: # element_index: The index of the source in the owning element's source list # element_kind: The kind of the owning element # kind: The kind of the source + # directory: A subdirectory where to stage the source + # provenance: The user provided provenance information (e.g. homepage, issue tracking, etc). # config: The configuration data for the source # first_pass: This source will be used with first project pass configuration (used for junctions). # - def __init__(self, element_name, element_index, element_kind, kind, config, directory, first_pass): + def __init__(self, element_name, element_index, element_kind, kind, directory, provenance, config, first_pass): self.element_name = element_name self.element_index = element_index self.element_kind = element_kind self.kind = kind - self.config = config self.directory = directory + self.provenance = provenance + self.config = config self.first_pass = first_pass diff --git a/src/buildstream/_loader/types.py b/src/buildstream/_loader/types.py index 6f6829bd2..a8989d577 100644 --- a/src/buildstream/_loader/types.py +++ b/src/buildstream/_loader/types.py @@ -40,3 +40,4 @@ class Symbol: JUNCTION = "junction" SANDBOX = "sandbox" STRICT = "strict" + PROVENANCE = "provenance" diff --git a/src/buildstream/element.py b/src/buildstream/element.py index ca8080314..156a3fead 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -90,7 +90,7 @@ from .sandbox import _SandboxFlags, SandboxCommandError from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote -from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey +from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance from ._artifact import Artifact from ._elementproxy import ElementProxy from ._elementsources import ElementSources @@ -2584,8 +2584,9 @@ def __load_sources(self, load_element): 0, self.get_kind(), "workspace", - Node.from_dict(workspace_node), None, + None, + Node.from_dict(workspace_node), load_element.first_pass, ) meta_sources.append(meta) @@ -2606,8 +2607,23 @@ def __load_sources(self, load_element): directory = source.get_str(Symbol.DIRECTORY, default=None) if directory: del source[Symbol.DIRECTORY] + + # Provenance is optional + provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None) + provenance = None + if provenance_node: + del source[Symbol.PROVENANCE] + provenance = _SourceProvenance.new_from_node(provenance_node) + meta_source = MetaSource( - self.name, index, self.get_kind(), kind.as_str(), source, directory, load_element.first_pass + self.name, + index, + self.get_kind(), + kind.as_str(), + directory, + provenance, + source, + load_element.first_pass, ) meta_sources.append(meta_source) diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 23b8a9444..ab4ffc2d3 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -32,6 +32,25 @@ This sets the location within the build root that the content of the source will be loaded in to. If the location does not exist, it will be created. +* Provenance + + The ``provenance`` attribute depicts a dictionary which is used for users + to provide additional source provenance related metadata which will later + be reported in :class:`.SourceInfo` objects. + + The ``provenance`` dictionary supports the following fields: + + * Homepage + + The ``homepage`` attribute can be used to specify the project homepage URL + + * Issue Tracker + + The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL + + *Since: 2.5* + + .. _core_source_abstract_methods: Abstract Methods @@ -359,7 +378,7 @@ from .node import MappingNode from .plugin import Plugin from .sourcemirror import SourceMirror -from .types import SourceRef, CoreWarnings, FastEnum +from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance from ._exceptions import BstError, ImplError, PluginError from .exceptions import ErrorDomain from ._loader.metasource import MetaSource @@ -537,10 +556,18 @@ class SourceInfo: *Since: 2.5* """ + # + # NOTE: The constructor is not public API, and plugins must + # call Source.create_source_info(), the docstring above + # starting with `SourceInfo()` ensures that documentation + # does not show constructor arguments. + # def __init__( self, kind: str, url: str, + homepage: Optional[str], + issue_tracker: Optional[str], medium: Union[SourceInfoMedium, str], version_type: Union[SourceVersionType, str], version: str, @@ -558,6 +585,16 @@ def __init__( The url of the source input """ + self.homepage: Optional[str] = homepage + """ + The project homepage URL + """ + + self.issue_tracker: Optional[str] = issue_tracker + """ + The project issue tracking URL + """ + self.medium: Union[SourceInfoMedium, str] = medium """ The :class:`.SourceInfoMedium` of the source input, or in the case @@ -618,11 +655,17 @@ def _serialize(self) -> Dict[str, Any]: version_info = { "kind": self.kind, "url": self.url, - "medium": medium_str, - "version-type": version_type_str, - "version": self.version, } + if self.homepage is not None: + version_info["homepage"] = self.homepage + if self.issue_tracker is not None: + version_info["issue-tracker"] = self.issue_tracker + + version_info["medium"] = medium_str + version_info["version-type"] = version_type_str + version_info["version"] = self.version + if self.version_guess is not None: version_info["version-guess"] = self.version_guess @@ -798,6 +841,9 @@ def __init__( self.__element_kind = meta.element_kind # The kind of the element owning this source self._directory = meta.directory # Staging relative directory self.__variables = variables # The variables used to resolve the source's config + self.__provenance: Optional[ + _SourceProvenance + ] = meta.provenance # The _SourceProvenance for general user provided SourceInfo self.__key = None # Cache key for source @@ -822,7 +868,7 @@ def __init__( self.__is_cached = None - COMMON_CONFIG_KEYS = ["kind", "directory"] + COMMON_CONFIG_KEYS = ["kind", "directory", "provenance"] """Common source config keys Source config keys that must not be accessed in configure(), and @@ -1357,8 +1403,22 @@ def create_source_info( *Since: 2.5* """ + homepage = None + issue_tracker = None + if self.__provenance is not None: + homepage = self.__provenance.homepage + issue_tracker = self.__provenance.issue_tracker + return SourceInfo( - self.get_kind(), url, medium, version_type, version, version_guess=version_guess, extra_data=extra_data + self.get_kind(), + url, + homepage, + issue_tracker, + medium, + version_type, + version, + version_guess=version_guess, + extra_data=extra_data, ) ############################################################# @@ -1850,8 +1910,9 @@ def __clone_for_uri(self, mirror): self.__element_index, self.__element_kind, self.get_kind(), - self.__config, self._directory, + self.__provenance, + self.__config, self.__first_pass, ) diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 778941768..26b12df84 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -384,6 +384,42 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror": return cls(name, aliases) +# _SourceProvenance() +# +# A simple object describing user provided source provenance information +# +# Args: +# homepage: The project homepage URL +# issue_tracker: The project issue reporting URL +# +class _SourceProvenance: + def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]): + self.homepage: Optional[str] = homepage + self.issue_tracker: Optional[str] = issue_tracker + + # new_from_node(): + # + # Creates a _SourceProvenance() from a YAML loaded node. + # + # Args: + # node: The configuration node describing the spec. + # + # Returns: + # The described _SourceProvenance instance. + # + # Raises: + # LoadError: If the node is malformed. + # + @classmethod + def new_from_node(cls, node: MappingNode) -> "_SourceProvenance": + node.validate_keys(["homepage", "issue-tracker"]) + + homepage: Optional[str] = node.get_str("homepage", None) + issue_tracker: Optional[str] = node.get_str("issue-tracker", None) + + return cls(homepage, issue_tracker) + + ######################################## # Type aliases # ######################################## From d6d5e738c8e3b455bb349542614dc6217a62a20b Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Wed, 14 May 2025 18:26:03 +0900 Subject: [PATCH 2/2] tests/frontend/show.py: Add SourceInfo test for user provided data --- tests/frontend/show.py | 44 ++++++++++++++++--- .../source-info/elements/user-provenance.bst | 9 ++++ 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 tests/frontend/source-info/elements/user-provenance.bst diff --git a/tests/frontend/show.py b/tests/frontend/show.py index f30b1cb88..842614b80 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -580,7 +580,7 @@ def test_invalid_alias(cli, tmpdir, datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) @pytest.mark.parametrize( - "target, expected_kind, expected_url, expected_medium, expected_version_type, expected_version, expected_guess_version", + "target, expected_kind, expected_url, expected_medium, expected_version_type, expected_version, expected_guess_version, expected_homepage, expected_issue_tracker", [ ( "local.bst", @@ -590,6 +590,8 @@ def test_invalid_alias(cli, tmpdir, datafiles): "cas-digest", "9391a5943daf287b46520c4289d41cab5f6b33e643f7661bcf620de7f02c1c9b/82", None, + None, + None, ), ( "tar.bst", @@ -599,6 +601,8 @@ def test_invalid_alias(cli, tmpdir, datafiles): "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "1.2.3", + None, + None, ), ( "tar-no-micro.bst", @@ -608,6 +612,8 @@ def test_invalid_alias(cli, tmpdir, datafiles): "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "1.2", + None, + None, ), ( "tar-custom-version.bst", @@ -617,6 +623,8 @@ def test_invalid_alias(cli, tmpdir, datafiles): "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "2.4.93", + None, + None, ), ( "tar-explicit.bst", @@ -626,6 +634,8 @@ def test_invalid_alias(cli, tmpdir, datafiles): "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "3.2.1", + None, + None, ), ( "testsource.bst", @@ -635,9 +645,30 @@ def test_invalid_alias(cli, tmpdir, datafiles): "pony-age", "1234567", "12", + None, + None, + ), + ( + "user-provenance.bst", + "tar", + "https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz", + "remote-file", + "sha256", + "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", + "1.2.3", + "https://flying-ponies.com/index.html", + "https://bugs.flying-ponies.com/issues", ), ], - ids=["local", "tar-full-version", "tar-no-micro", "tar-custom-version", "tar-explicit", "testsource"], + ids=[ + "local", + "tar-full-version", + "tar-no-micro", + "tar-custom-version", + "tar-explicit", + "testsource", + "user-provenance", + ], ) def test_source_info( cli, @@ -649,6 +680,8 @@ def test_source_info( expected_version_type, expected_version, expected_guess_version, + expected_homepage, + expected_issue_tracker, ): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", target]) @@ -664,9 +697,10 @@ def test_source_info( assert source_info.get_str("version-type") == expected_version_type assert source_info.get_str("version") == expected_version - guess_version = source_info.get_str("version-guess", None) - if guess_version or expected_guess_version: - assert guess_version == expected_guess_version + # Optional fields + assert source_info.get_str("version-guess", None) == expected_guess_version + assert source_info.get_str("homepage", None) == expected_homepage + assert source_info.get_str("issue-tracker", None) == expected_issue_tracker @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) diff --git a/tests/frontend/source-info/elements/user-provenance.bst b/tests/frontend/source-info/elements/user-provenance.bst new file mode 100644 index 000000000..50d15ae9e --- /dev/null +++ b/tests/frontend/source-info/elements/user-provenance.bst @@ -0,0 +1,9 @@ +kind: import + +sources: +- kind: tar + url: https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz + ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 + provenance: + homepage: https://flying-ponies.com/index.html + issue-tracker: https://bugs.flying-ponies.com/issues