From b0dcf3925b8728e06cf6818b42c102dc6cc6ec0e Mon Sep 17 00:00:00 2001 From: Abderrahim Kitouni Date: Tue, 8 Jul 2025 08:21:04 +0100 Subject: [PATCH 1/2] _sandboxremote.py: stop trying to pull partial artifacts There used to be a time where buildstream could partially download artifacts, which could mean that not all artifact blobs are available locally. This is no longer the case: support for partial artifacts was dropped in favour of the cache storage-service and is only used for buildtrees, which aren't used in this case. Downloading missing blobs from the cache storage-service is handled in `CASCache._send_blobs()` called a few lines below. --- src/buildstream/sandbox/_sandboxremote.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index b087e053f..5b4bd0414 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -23,7 +23,7 @@ from .. import _signals from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.google.rpc import code_pb2 -from .._exceptions import BstError, SandboxError +from .._exceptions import SandboxError from .._protos.google.longrunning import operations_pb2 @@ -160,9 +160,7 @@ def _execute_action(self, action, flags): stdout, stderr = self._get_output() context = self._get_context() - project = self._get_project() cascache = context.get_cascache() - artifactcache = context.artifactcache action_digest = cascache.add_object(buffer=action.SerializeToString()) @@ -182,15 +180,6 @@ def _execute_action(self, action, flags): except grpc.RpcError as e: raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e - # Check if any blobs are also missing locally (partial artifact) - # and pull them from the artifact cache. - try: - local_missing_blobs = cascache.missing_blobs(missing_blobs) - if local_missing_blobs: - artifactcache.fetch_missing_blobs(project, local_missing_blobs) - except (grpc.RpcError, BstError) as e: - raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e - # Add command and action messages to blob list to push missing_blobs.append(action.command_digest) missing_blobs.append(action_digest) From e867d6f3a257391cc89195b10f947a70bc0f7eb5 Mon Sep 17 00:00:00 2001 From: Abderrahim Kitouni Date: Tue, 8 Jul 2025 08:27:00 +0100 Subject: [PATCH 2/2] _artifactcache.py: drop {fetch,find}_missing_blobs These were related to partial artifacts and are no longer used --- src/buildstream/_artifactcache.py | 52 ------------------------------- 1 file changed, 52 deletions(-) diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index a531c453c..a32d0ee62 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -238,58 +238,6 @@ def link_key(self, element, oldkey, newkey): utils.safe_link(os.path.join(self._basedir, oldref), os.path.join(self._basedir, newref)) - # fetch_missing_blobs(): - # - # Fetch missing blobs from configured remote repositories. - # - # Args: - # project (Project): The current project - # missing_blobs (list): The Digests of the blobs to fetch - # - def fetch_missing_blobs(self, project, missing_blobs): - - index_remotes, _ = self.get_remotes(project.name, False) - for remote in index_remotes: - if not missing_blobs: - break - - remote.init() - - # fetch_blobs() will return the blobs that are still missing - missing_blobs = self.cas.fetch_blobs(remote, missing_blobs, allow_partial=True) - - if missing_blobs: - raise ArtifactError("Blobs not found on configured artifact servers") - - # find_missing_blobs(): - # - # Find missing blobs from configured push remote repositories. - # - # Args: - # project (Project): The current project - # missing_blobs (list): The Digests of the blobs to check - # - # Returns: - # (list): The Digests of the blobs missing on at least one push remote - # - def find_missing_blobs(self, project, missing_blobs): - if not missing_blobs: - return [] - - _, push_remotes = self.get_remotes(project.name, True) - remote_missing_blobs_list = [] - - for remote in push_remotes: - remote.init() - - remote_missing_blobs = self.cas.missing_blobs(missing_blobs, remote=remote) - - for blob in remote_missing_blobs: - if blob not in remote_missing_blobs_list: - remote_missing_blobs_list.append(blob) - - return remote_missing_blobs_list - # check_remotes_for_element() # # Check if the element is available in any of the remotes