Implement new collect_source_info() method

project.conf

@@ -17,7 +17,7 @@
 # that plugins can be loaded via junctions.
 #
 name: buildstream-plugins
-min-version: 2.0
+min-version: 2.5
 
 plugins:
 - origin: local

src/buildstream_plugins/sources/bzr.py

@@ -53,9 +53,31 @@
    # revision number to the one on the tip of the branch specified in 'track'.
    ref: 6622
 
+   # Specify the version to be reported as the *guess_version* when reporting
+   # SourceInfo
+   #
+   # Since 2.5
+   #
+   version: 1.2
+
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The bzr source reports the URL of the bzr repository as the *url*.
+
+Further, the bzr source reports the `SourceInfoMedium.BZR
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.BZR>`_
+*medium* and the` `SourceVersionType.COMMIT
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.COMMIT>`_
+*version_type*, for which it reports the bzr revision number as the *version*.
+
+Since the bzr source does not have a way to know what the release version
+corresponds to the revision number, the bzr source exposes the ``version`` configuration
+attribute to allow explicit specification of the *guess_version*.
 """
 
 import os
@@ -66,26 +88,44 @@
 from buildstream import Source, SourceError
 from buildstream import utils
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except ImportError:
+    pass
+
 
 class BzrSource(Source):
     # pylint: disable=attribute-defined-outside-init
 
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     def configure(self, node):
-        node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS])
+        node.validate_keys(["url", "track", "ref", "version", *Source.COMMON_CONFIG_KEYS])
 
         self.original_url = node.get_str("url")
         self.tracking = node.get_str("track")
         self.ref = node.get_str("ref", None)
         self.url = self.translate_url(self.original_url)
+        self.version = node.get_str("version", None)
 
     def preflight(self):
         # Check if bzr is installed, get the binary at the same time.
         self.host_bzr = utils.get_host_tool("bzr")
 
     def get_unique_key(self):
-        return [self.original_url, self.tracking, self.ref]
+        unique_key = [self.original_url, self.tracking, self.ref]
+
+        # Backwards compatible method of supporting configuration
+        # attributes which affect SourceInfo generation.
+        if self.version is not None:
+            unique_key.append(self.version)
+
+        return unique_key
 
     def is_cached(self):
         with self._locked():
@@ -167,6 +207,13 @@ def init_workspace(self, directory):
                 fail="Failed to switch workspace's parent branch to {}".format(url),
             )
 
+    def collect_source_info(self):
+        return [
+            self.create_source_info(
+                self.url, SourceInfoMedium.BAZAAR, SourceVersionType.COMMIT, self.ref, version_guess=self.version
+            )
+        ]
+
     # _locked()
     #
     # This context manager ensures exclusive access to the

src/buildstream_plugins/sources/cargo.py

@@ -61,6 +61,19 @@
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The cargo source reports the URL of the archive crates as the *url* for each crate.
+
+Further, the cargo source reports the `SourceInfoMedium.REMOTE_FILE
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.REMOTE_FILE>`_
+*medium* and the `SourceVersionType.SHA256
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.SHA256>`_
+*version_type*, for which it reports the checksum of the archive as the *version*.
+
+The versions extracted from the Cargo.lock at tracking time are used to report the *guess_version*.
 """
 
 import json
@@ -79,6 +92,16 @@
 
 from ._utils import download_file
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except:
+    pass
+
 
 # This automatically goes into .cargo/config
 #
@@ -133,6 +156,12 @@ def fetch(self, alias_override=None, **kwargs):
                     "File downloaded from {} has sha256sum '{}', not '{}'!".format(crate_url, sha256, self.sha)
                 )
 
+    def get_source_info(self):
+        url, _ = self._get_url()
+        return self.cargo.create_source_info(
+            url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.sha, version_guess=self.version
+        )
+
     ########################################################
     #        Helper APIs for the Cargo Source to use       #
     ########################################################
@@ -334,7 +363,7 @@ def _get_mirror_file(self, sha=None):
 
 
 class CargoSource(Source):
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     # We need the Cargo.lock file to construct our ref at track time
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True

src/buildstream_plugins/sources/docker.py

@@ -57,13 +57,46 @@
    #
    # **Since**: 2.0.1
 
+   # Specify the version to be reported as the *guess_version* when reporting
+   # SourceInfo
+   #
+   # Since: 2.5
+   #
+   version: 1.2
+
 Note that Docker images may contain device nodes. BuildStream elements cannot
 contain device nodes so those will be dropped. Any regular files in the /dev
 directory will also be dropped.
 
 See `built-in functionality doumentation
 <https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
 details on common configuration options for sources.
+
+
+Reporting `SourceInfo <https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfo>`_
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The docker source reports the URL of the docker registry as the *url*.
+
+Further, the docker source reports the `SourceInfoMedium.OCI_IMAGE
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceInfoMedium.OCI_IMAGE>`_
+*medium* and the `SourceVersionType.OCI_DIGEST
+<https://docs.buildstream.build/master/buildstream.source.html#buildstream.source.SourceVersionType.OCI_DIGEST>`_
+*version_type*, for which it reports the content digest of the docker image as the *version*.
+
+Additionally, the docker source reports the docker image name through
+the ``image-name`` key of the *extra_data*.
+
+As such, after removing the scheme from the URL (i.e. remove ``https://``) the same docker
+image can be obtained by calling:
+
+.. code:: bash
+
+   docker pull <url-without-scheme>/<image-name>@<version>
+
+Since the docker source does not have any way to guess what tag is associated
+to the digest, or what release version that would mean; the docker source exposes
+the ``version`` configuration attribute to allow explicit specification of the
+*guess_version*.
 """
 
 import hashlib
@@ -84,6 +117,16 @@
     move_atomic,
 )
 
+#
+# Soft import of buildstream symbols only available in newer versions
+#
+# The BST_MIN_VERSION will provide a better user experience.
+#
+try:
+    from buildstream import SourceInfoMedium, SourceVersionType
+except ImportError:
+    pass
+
 _DOCKER_HUB_URL = "https://registry.hub.docker.com"
 
 
@@ -342,7 +385,7 @@ def mode(self, permission):
 class DockerSource(Source):
     # pylint: disable=too-many-instance-attributes
 
-    BST_MIN_VERSION = "2.0"
+    BST_MIN_VERSION = "2.5"
 
     # Docker identifies images by a content digest calculated from the image's
     # manifest. This corresponds well with the concept of a 'ref' in
@@ -365,7 +408,8 @@ def configure(self, node):
         # url is deprecated, but accept it as a valid key so that we can raise
         # a nicer warning.
         node.validate_keys(
-            Source.COMMON_CONFIG_KEYS + ["architecture", "registry-url", "image", "os", "ref", "track", "url"]
+            Source.COMMON_CONFIG_KEYS
+            + ["architecture", "registry-url", "image", "os", "ref", "track", "url", "version"]
         )
 
         if "url" in node and ("image" in node or "registry-url" in node):
@@ -391,6 +435,7 @@ def configure(self, node):
 
         self.architecture = node.get_str("architecture", "") or default_architecture()
         self.os = node.get_str("os", "") or default_os()
+        self.version = node.get_str("version", None)
 
         self.digest = None
         self.load_ref(node)
@@ -409,9 +454,16 @@ def preflight(self):
 
     def get_unique_key(self):
         if self.url is not None:
-            return [self.url, self.digest]
+            unique_key = [self.url, self.digest]
         else:
-            return [self.original_registry_url, self.image, self.digest]
+            unique_key = [self.original_registry_url, self.image, self.digest]
+
+        # Backwards compatible method of supporting configuration
+        # attributes which affect SourceInfo generation.
+        if self.version is not None:
+            unique_key.append(self.version)
+
+        return unique_key
 
     def get_ref(self):
         return None if self.digest is None else self._digest_to_ref(self.digest)
@@ -583,6 +635,28 @@ def stage(self, directory):
         except (OSError, SourceError, tarfile.TarError) as e:
             raise SourceError("{}: Error staging source: {}".format(self, e)) from e
 
+    def collect_source_info(self):
+
+        # If the image was configured with "url" only rather
+        # than "registry-url" and "image", then self.image
+        # at this point will have a leading forward slash "/".
+        #
+        # Lets just normalize that problem here, as changing
+        # the self.image itself can break cache keys.
+        #
+        image_name = self.image.lstrip("/")
+
+        return [
+            self.create_source_info(
+                self.registry_url,
+                SourceInfoMedium.OCI_IMAGE,
+                SourceVersionType.OCI_DIGEST,
+                self.digest,
+                version_guess=self.version,
+                extra_data={"image-name": image_name},
+            )
+        ]
+
     @staticmethod
     def _get_extract_and_remove_files(layer_tar_path):
         """Return the set of files to remove and extract for a given layer