From 1a669b5fc96d03e2d58dca0a6968d56779b67d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 14 Nov 2025 14:07:29 +0100 Subject: [PATCH 1/4] tests/integration/shellbuildtrees.py: Test working directory in artifact Adjust `test_shell_pull_artifact_cached_buildtree` now that the working directory is stored in the artifact. --- tests/integration/shellbuildtrees.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py index d042ea5fd..437668ac2 100644 --- a/tests/integration/shellbuildtrees.py +++ b/tests/integration/shellbuildtrees.py @@ -362,8 +362,7 @@ def test_shell_pull_artifact_cached_buildtree(share_with_buildtrees, datafiles, artifact_name, "--", "cat", - # We don't preserve the working directory in artifacts, so we will be executing at / - "/buildstream/test/build-shell/buildtree.bst/test", + "test", ], ) From b52382a333c9099594a76cae6e99163b2b38e2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 14 Nov 2025 13:34:55 +0100 Subject: [PATCH 2/4] artifact.proto: Add marked directories to the build sandbox state --- src/buildstream/_protos/buildstream/v2/artifact.proto | 1 + src/buildstream/_protos/buildstream/v2/artifact_pb2.py | 6 +++--- src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/buildstream/_protos/buildstream/v2/artifact.proto b/src/buildstream/_protos/buildstream/v2/artifact.proto index 1024ce860..28d006f0f 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact.proto +++ b/src/buildstream/_protos/buildstream/v2/artifact.proto @@ -90,6 +90,7 @@ message Artifact { repeated build.bazel.remote.execution.v2.Command.EnvironmentVariable environment = 1; string working_directory = 2; repeated build.bazel.remote.execution.v2.Digest subsandbox_digests = 3; + repeated string marked_directories = 4; }; SandboxState buildsandbox = 18; // optional } diff --git a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py index 7d86ea3b8..a81006af5 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact_pb2.py +++ b/src/buildstream/_protos/buildstream/v2/artifact_pb2.py @@ -26,7 +26,7 @@ from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\x8a\t\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\x06 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x07 \x01(\x08\x12\x36\n\x05\x66iles\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x37\n\nbuild_deps\x18\t \x03(\x0b\x32#.buildstream.v2.Artifact.Dependency\x12<\n\x0bpublic_data\x18\n \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12.\n\x04logs\x18\x0b \x03(\x0b\x32 .buildstream.v2.Artifact.LogFile\x12:\n\tbuildtree\x18\x0c \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x38\n\x07sources\x18\r \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x43\n\x12low_diversity_meta\x18\x0e \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x44\n\x13high_diversity_meta\x18\x0f \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x12\n\nstrict_key\x18\x10 \x01(\t\x12:\n\tbuildroot\x18\x11 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12;\n\x0c\x62uildsandbox\x18\x12 \x01(\x0b\x32%.buildstream.v2.Artifact.SandboxState\x1a\x63\n\nDependency\x12\x14\n\x0cproject_name\x18\x01 \x01(\t\x12\x14\n\x0c\x65lement_name\x18\x02 \x01(\t\x12\x11\n\tcache_key\x18\x03 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x04 \x01(\x08\x1aP\n\x07LogFile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x1a\xc1\x01\n\x0cSandboxState\x12Q\n\x0b\x65nvironment\x18\x01 \x03(\x0b\x32<.build.bazel.remote.execution.v2.Command.EnvironmentVariable\x12\x19\n\x11working_directory\x18\x02 \x01(\t\x12\x43\n\x12subsandbox_digests\x18\x03 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digestb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\xa6\t\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\x06 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x07 \x01(\x08\x12\x36\n\x05\x66iles\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x37\n\nbuild_deps\x18\t \x03(\x0b\x32#.buildstream.v2.Artifact.Dependency\x12<\n\x0bpublic_data\x18\n \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12.\n\x04logs\x18\x0b \x03(\x0b\x32 .buildstream.v2.Artifact.LogFile\x12:\n\tbuildtree\x18\x0c \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x38\n\x07sources\x18\r \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x43\n\x12low_diversity_meta\x18\x0e \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x44\n\x13high_diversity_meta\x18\x0f \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x12\n\nstrict_key\x18\x10 \x01(\t\x12:\n\tbuildroot\x18\x11 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12;\n\x0c\x62uildsandbox\x18\x12 \x01(\x0b\x32%.buildstream.v2.Artifact.SandboxState\x1a\x63\n\nDependency\x12\x14\n\x0cproject_name\x18\x01 \x01(\t\x12\x14\n\x0c\x65lement_name\x18\x02 \x01(\t\x12\x11\n\tcache_key\x18\x03 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x04 \x01(\x08\x1aP\n\x07LogFile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x1a\xdd\x01\n\x0cSandboxState\x12Q\n\x0b\x65nvironment\x18\x01 \x03(\x0b\x32<.build.bazel.remote.execution.v2.Command.EnvironmentVariable\x12\x19\n\x11working_directory\x18\x02 \x01(\t\x12\x43\n\x12subsandbox_digests\x18\x03 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x1a\n\x12marked_directories\x18\x04 \x03(\tb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,11 +34,11 @@ if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_ARTIFACT']._serialized_start=136 - _globals['_ARTIFACT']._serialized_end=1298 + _globals['_ARTIFACT']._serialized_end=1326 _globals['_ARTIFACT_DEPENDENCY']._serialized_start=921 _globals['_ARTIFACT_DEPENDENCY']._serialized_end=1020 _globals['_ARTIFACT_LOGFILE']._serialized_start=1022 _globals['_ARTIFACT_LOGFILE']._serialized_end=1102 _globals['_ARTIFACT_SANDBOXSTATE']._serialized_start=1105 - _globals['_ARTIFACT_SANDBOXSTATE']._serialized_end=1298 + _globals['_ARTIFACT_SANDBOXSTATE']._serialized_end=1326 # @@protoc_insertion_point(module_scope) diff --git a/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi b/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi index 08c6960c1..7eb4d7550 100644 --- a/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi +++ b/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi @@ -28,14 +28,16 @@ class Artifact(_message.Message): digest: _remote_execution_pb2.Digest def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... class SandboxState(_message.Message): - __slots__ = ("environment", "working_directory", "subsandbox_digests") + __slots__ = ("environment", "working_directory", "subsandbox_digests", "marked_directories") ENVIRONMENT_FIELD_NUMBER: _ClassVar[int] WORKING_DIRECTORY_FIELD_NUMBER: _ClassVar[int] SUBSANDBOX_DIGESTS_FIELD_NUMBER: _ClassVar[int] + MARKED_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] environment: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Command.EnvironmentVariable] working_directory: str subsandbox_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] - def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ...) -> None: ... + marked_directories: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., marked_directories: _Optional[_Iterable[str]] = ...) -> None: ... VERSION_FIELD_NUMBER: _ClassVar[int] BUILD_SUCCESS_FIELD_NUMBER: _ClassVar[int] BUILD_ERROR_FIELD_NUMBER: _ClassVar[int] From 3ed8f422e97d7bc381cce306d2f4b74ece99d927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 14 Nov 2025 13:39:09 +0100 Subject: [PATCH 3/4] _artifact.py: Store marked directories in Artifact proto Fixes #2092. --- src/buildstream/_artifact.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index c06603559..939b3018e 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -325,6 +325,9 @@ def cache( for key, value in sorted(sandbox_env.items()): artifact.buildsandbox.environment.add(name=key, value=value) + for directory in buildsandbox._get_marked_directories(): + artifact.buildsandbox.marked_directories.append(directory) + artifact.buildsandbox.working_directory = buildsandbox._get_work_directory() for subsandbox in buildsandbox._get_subsandboxes(): @@ -699,7 +702,7 @@ def pull(self, *, pull_buildtrees): def configure_sandbox(self, sandbox): artifact = self._get_proto() - if artifact.buildsandbox and artifact.buildsandbox.environment: + if artifact.HasField("buildsandbox") and artifact.buildsandbox.environment: env = {} for env_var in artifact.buildsandbox.environment: env[env_var.name] = env_var.value @@ -708,8 +711,12 @@ def configure_sandbox(self, sandbox): sandbox.set_environment(env) - if artifact.buildsandbox and artifact.buildsandbox.working_directory: - sandbox.set_work_directory(artifact.buildsandbox.working_directory) + if artifact.HasField("buildsandbox"): + for directory in artifact.buildsandbox.marked_directories: + sandbox.mark_directory(directory) + + if artifact.buildsandbox.working_directory: + sandbox.set_work_directory(artifact.buildsandbox.working_directory) # load_proto() # From 46c29eb3fab2042608e239b0e310b01f15473f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 14 Nov 2025 14:10:10 +0100 Subject: [PATCH 4/4] tests/integration/shellbuildtrees.py: Test that filesystem is writable --- tests/integration/shellbuildtrees.py | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py index 437668ac2..a366962e5 100644 --- a/tests/integration/shellbuildtrees.py +++ b/tests/integration/shellbuildtrees.py @@ -339,6 +339,21 @@ def test_shell_pull_cached_buildtree(share_with_buildtrees, datafiles, cli, pull result.assert_success() assert "Hi" in result.output + # Check that the working directory is writable + result = cli.run( + project=project, + args=[ + "shell", + "--build", + element_name, + "--use-buildtree", + "--", + "touch", + "foo", + ], + ) + result.assert_success() + # # Test behavior of shelling into a buildtree by its artifact name @@ -372,6 +387,21 @@ def test_shell_pull_artifact_cached_buildtree(share_with_buildtrees, datafiles, result.assert_success() assert "Hi" in result.output + # Check that the working directory is writable + result = cli.run( + project=project, + args=[ + "shell", + "--build", + "--use-buildtree", + artifact_name, + "--", + "touch", + "foo", + ], + ) + result.assert_success() + # # Test behavior of launching a shell and requesting to use a buildtree.