Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 0 additions & 52 deletions src/buildstream/_artifactcache.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,58 +238,6 @@ def link_key(self, element, oldkey, newkey):

utils.safe_link(os.path.join(self._basedir, oldref), os.path.join(self._basedir, newref))

# fetch_missing_blobs():
#
# Fetch missing blobs from configured remote repositories.
#
# Args:
# project (Project): The current project
# missing_blobs (list): The Digests of the blobs to fetch
#
def fetch_missing_blobs(self, project, missing_blobs):

index_remotes, _ = self.get_remotes(project.name, False)
for remote in index_remotes:
if not missing_blobs:
break

remote.init()

# fetch_blobs() will return the blobs that are still missing
missing_blobs = self.cas.fetch_blobs(remote, missing_blobs, allow_partial=True)

if missing_blobs:
raise ArtifactError("Blobs not found on configured artifact servers")

# find_missing_blobs():
#
# Find missing blobs from configured push remote repositories.
#
# Args:
# project (Project): The current project
# missing_blobs (list): The Digests of the blobs to check
#
# Returns:
# (list): The Digests of the blobs missing on at least one push remote
#
def find_missing_blobs(self, project, missing_blobs):
if not missing_blobs:
return []

_, push_remotes = self.get_remotes(project.name, True)
remote_missing_blobs_list = []

for remote in push_remotes:
remote.init()

remote_missing_blobs = self.cas.missing_blobs(missing_blobs, remote=remote)

for blob in remote_missing_blobs:
if blob not in remote_missing_blobs_list:
remote_missing_blobs_list.append(blob)

return remote_missing_blobs_list

# check_remotes_for_element()
#
# Check if the element is available in any of the remotes
Expand Down
13 changes: 1 addition & 12 deletions src/buildstream/sandbox/_sandboxremote.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from .. import _signals
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from .._protos.google.rpc import code_pb2
from .._exceptions import BstError, SandboxError
from .._exceptions import SandboxError
from .._protos.google.longrunning import operations_pb2


Expand Down Expand Up @@ -160,9 +160,7 @@ def _execute_action(self, action, flags):
stdout, stderr = self._get_output()

context = self._get_context()
project = self._get_project()
cascache = context.get_cascache()
artifactcache = context.artifactcache

action_digest = cascache.add_object(buffer=action.SerializeToString())

Expand All @@ -182,15 +180,6 @@ def _execute_action(self, action, flags):
except grpc.RpcError as e:
raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e

# Check if any blobs are also missing locally (partial artifact)
# and pull them from the artifact cache.
try:
local_missing_blobs = cascache.missing_blobs(missing_blobs)
if local_missing_blobs:
artifactcache.fetch_missing_blobs(project, local_missing_blobs)
except (grpc.RpcError, BstError) as e:
raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e

# Add command and action messages to blob list to push
missing_blobs.append(action.command_digest)
missing_blobs.append(action_digest)
Expand Down
Loading