From ae3e0cd52e09187fb743d3ac024fd9b8082fc765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 23 May 2025 12:36:10 +0200 Subject: [PATCH 1/4] _context.py: Add `effective_build_max_jobs` helper --- src/buildstream/_context.py | 7 +++++++ src/buildstream/_project.py | 11 +---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 6b0cf4123..4d553b38b 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -531,6 +531,13 @@ def sourcecache(self) -> SourceCache: return self._sourcecache + @property + def effective_build_max_jobs(self) -> int: + # Based on some testing (mainly on AWS), maximum effective + # max-jobs value seems to be around 8-10 if we have enough cores + # users should set values based on workload and build infrastructure + return self.build_max_jobs or self.platform.get_cpu_count(8) + # add_project(): # # Add a project to the context. diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index b55f257fa..334462df0 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -1075,16 +1075,7 @@ def _load_pass(self, config, output, *, ignore_unknown=False): # Extend variables with automatic variables and option exports # Initialize it as a string as all variables are processed as strings. - # Based on some testing (mainly on AWS), maximum effective - # max-jobs value seems to be around 8-10 if we have enough cores - # users should set values based on workload and build infrastructure - if self._context.build_max_jobs == 0: - # User requested automatic max-jobs - platform = self._context.platform - output.base_variables["max-jobs"] = str(platform.get_cpu_count(8)) - else: - # User requested explicit max-jobs setting - output.base_variables["max-jobs"] = str(self._context.build_max_jobs) + output.base_variables["max-jobs"] = str(self._context.effective_build_max_jobs) # Export options into variables, if that was requested output.options.export_variables(output.base_variables) From 54cfb70ff79dc65a2eea3d7e65a229ae11dc7834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 11 Apr 2025 14:12:09 +0200 Subject: [PATCH 2/4] Enable local execution in buildbox-casd to support nested execution This bumps the minimum version of buildbox-casd to 1.2.0. --- src/buildstream/_cas/casdprocessmanager.py | 20 ++++++++++++++++---- src/buildstream/_context.py | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py index 899a5473d..fd1692d9f 100644 --- a/src/buildstream/_cas/casdprocessmanager.py +++ b/src/buildstream/_cas/casdprocessmanager.py @@ -45,9 +45,9 @@ # # Minimum required version of buildbox-casd # -_REQUIRED_CASD_MAJOR = 0 -_REQUIRED_CASD_MINOR = 0 -_REQUIRED_CASD_MICRO = 58 +_REQUIRED_CASD_MAJOR = 1 +_REQUIRED_CASD_MINOR = 2 +_REQUIRED_CASD_MICRO = 0 # CASDProcessManager @@ -76,7 +76,8 @@ def __init__( messenger, *, reserved=None, - low_watermark=None + low_watermark=None, + local_jobs=None ): os.makedirs(path, exist_ok=True) @@ -104,6 +105,17 @@ def __init__( if protect_session_blobs: casd_args.append("--protect-session-blobs") + if local_jobs is not None: + try: + buildbox_run = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox") + casd_args.append("--buildbox-run={}".format(buildbox_run)) + casd_args.append("--jobs={}".format(local_jobs)) + except utils.ProgramNotFoundError: + # Not fatal as buildbox-run is not needed for remote execution + # and buildbox-casd local execution will never be used if + # buildbox-run is not available. + pass + if remote_cache_spec: casd_args.append("--cas-remote={}".format(remote_cache_spec.url)) if remote_cache_spec.instance_name: diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 4d553b38b..02942c8b0 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -734,6 +734,7 @@ def get_casd(self) -> CASDProcessManager: assert self.logdir is not None, "log_directory is required for casd" log_dir = os.path.join(self.logdir, "_casd") + assert self.sched_builders is not None, "builders configuration is required" self._casd = CASDProcessManager( self.cachedir, log_dir, @@ -744,6 +745,7 @@ def get_casd(self) -> CASDProcessManager: messenger=self.messenger, reserved=self.config_cache_reserved, low_watermark=self.config_cache_low_watermark, + local_jobs=self.sched_builders * self.effective_build_max_jobs, ) return self._casd From e587aa21b2f7a565db9b13eeb836638643c015d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 2 May 2025 16:57:24 +0200 Subject: [PATCH 3/4] sandbox: Add support for `remote-apis-socket` Fixes #1985. --- doc/source/format_declaring.rst | 18 +++++++++++ src/buildstream/sandbox/_config.py | 30 +++++++++++++++++-- .../sandbox/_sandboxbuildboxrun.py | 7 +++++ src/buildstream/sandbox/_sandboxreapi.py | 3 ++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/doc/source/format_declaring.rst b/doc/source/format_declaring.rst index 7bad61f12..7a1f71854 100644 --- a/doc/source/format_declaring.rst +++ b/doc/source/format_declaring.rst @@ -389,6 +389,24 @@ field in the ``Command`` uploaded. Whether this actually results in a building the element for the desired OS and architecture is dependent on the server having implemented these options the same as buildstream. +.. code:: yaml + + # Specify UNIX socket path for access to REAPI for (nested) remote execution + sandbox: + remote-apis-socket: + path: /run/reapi.sock + +Setting a path will add a UNIX socket to the sandbox that allows the use of +`REAPI `_ clients such as +`recc `_. + +This enables more fine-grained caching of, e.g., individual compile commands +to speed up rebuilds of elements with only small changes. + +This is supported with and without :ref:`remote execution `. +With remote execution configured, this additionally enables scaling out of, +e.g., compile commands across a cluster of build machines. + .. _format_dependencies: diff --git a/src/buildstream/sandbox/_config.py b/src/buildstream/sandbox/_config.py index 654f16454..87e6b35fa 100644 --- a/src/buildstream/sandbox/_config.py +++ b/src/buildstream/sandbox/_config.py @@ -34,6 +34,7 @@ # build_arch: A canonical machine architecture name, as defined by Platform.canonicalize_arch() # build_uid: The UID for the sandbox process # build_gid: The GID for the sandbox process +# remote_apis_socket_path: The path to a UNIX socket providing REAPI access for nested remote execution # # If the build_uid or build_gid is unspecified, then the underlying sandbox implementation # does not guarantee what UID/GID will be used, but generally UID/GID 0 will be used in a @@ -45,12 +46,19 @@ # class SandboxConfig: def __init__( - self, *, build_os: str, build_arch: str, build_uid: Optional[int] = None, build_gid: Optional[int] = None + self, + *, + build_os: str, + build_arch: str, + build_uid: Optional[int] = None, + build_gid: Optional[int] = None, + remote_apis_socket_path: Optional[str] = None ): self.build_os = build_os self.build_arch = build_arch self.build_uid = build_uid self.build_gid = build_gid + self.remote_apis_socket_path = remote_apis_socket_path # to_dict(): # @@ -87,6 +95,9 @@ def to_dict(self) -> Dict[str, Union[str, int]]: if self.build_gid is not None: sandbox_dict["build-gid"] = self.build_gid + if self.remote_apis_socket_path is not None: + sandbox_dict["remote-apis-socket-path"] = self.remote_apis_socket_path + return sandbox_dict # new_from_node(): @@ -108,7 +119,7 @@ def to_dict(self) -> Dict[str, Union[str, int]]: # @classmethod def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platform] = None) -> "SandboxConfig": - config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"]) + config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch", "remote-apis-socket"]) build_os: str build_arch: str @@ -132,4 +143,17 @@ def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platfo build_uid = config.get_int("build-uid", None) build_gid = config.get_int("build-gid", None) - return cls(build_os=build_os, build_arch=build_arch, build_uid=build_uid, build_gid=build_gid) + remote_apis_socket = config.get_mapping("remote-apis-socket", default=None) + if remote_apis_socket: + remote_apis_socket.validate_keys(["path"]) + remote_apis_socket_path = remote_apis_socket.get_str("path") + else: + remote_apis_socket_path = None + + return cls( + build_os=build_os, + build_arch=build_arch, + build_uid=build_uid, + build_gid=build_gid, + remote_apis_socket_path=remote_apis_socket_path, + ) diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py index 25d200262..d4b49860a 100644 --- a/src/buildstream/sandbox/_sandboxbuildboxrun.py +++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py @@ -83,12 +83,19 @@ def check_sandbox_config(cls, config): if config.build_gid is not None and "platform:unixGID" not in cls._capabilities: raise SandboxUnavailableError("Configuring sandbox GID is not supported by buildbox-run.") + if config.remote_apis_socket_path is not None and "platform:remoteApisSocketPath" not in cls._capabilities: + raise SandboxUnavailableError("Configuring Remote APIs socket path is not supported by buildbox-run.") + def _execute_action(self, action, flags): stdout, stderr = self._get_output() context = self._get_context() cascache = context.get_cascache() casd = cascache.get_casd() + config = self._get_config() + + if config.remote_apis_socket_path and context.remote_cache_spec: + raise SandboxError("'remote-apis-socket' is not currently supported with 'storage-service'.") with utils._tempnamedfile() as action_file, utils._tempnamedfile() as result_file: action_file.write(action.SerializeToString()) diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index 17997b355..262cb3f04 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -131,6 +131,9 @@ def _create_platform(self, flags): if flags & _SandboxFlags.NETWORK_ENABLED: platform_dict["network"] = "on" + if config.remote_apis_socket_path: + platform_dict["remoteApisSocketPath"] = config.remote_apis_socket_path.lstrip(os.path.sep) + # Create Platform message with properties sorted by name in code point order platform = remote_execution_pb2.Platform() for key, value in sorted(platform_dict.items()): From c78a1847e599252c4b1144168b0e722d5b4900e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 13 Jun 2025 16:56:45 +0200 Subject: [PATCH 4/4] tests/integration/sandbox.py: Add test for `remote-apis-socket` --- .../project/elements/sandbox/remote-apis-socket.bst | 13 +++++++++++++ tests/integration/sandbox.py | 11 +++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/integration/project/elements/sandbox/remote-apis-socket.bst diff --git a/tests/integration/project/elements/sandbox/remote-apis-socket.bst b/tests/integration/project/elements/sandbox/remote-apis-socket.bst new file mode 100644 index 000000000..113cdcf65 --- /dev/null +++ b/tests/integration/project/elements/sandbox/remote-apis-socket.bst @@ -0,0 +1,13 @@ +kind: manual + +depends: + - filename: base.bst + type: build + +sandbox: + remote-apis-socket: + path: /tmp/reapi.sock + +config: + build-commands: + - test -S /tmp/reapi.sock diff --git a/tests/integration/sandbox.py b/tests/integration/sandbox.py index aed00c743..7d2d00123 100644 --- a/tests/integration/sandbox.py +++ b/tests/integration/sandbox.py @@ -48,3 +48,14 @@ def test_build_arch(cli, datafiles): result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 + + +# Test that the REAPI socket is created in the sandbox. +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.datafiles(DATA_DIR) +def test_remote_apis_socket(cli, datafiles): + project = str(datafiles) + element_name = "sandbox/remote-apis-socket.bst" + + result = cli.run(project=project, args=["build", element_name]) + assert result.exit_code == 0