From 878fa52512871c20ca08e13716ca80e6082953c6 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 9 Nov 2025 04:20:01 +0900 Subject: [PATCH 1/4] Support artifact remote arguments in bst artifact show command. This is useful for ensuring that the shallow artifacts have been downloaded from the remotes, without requiring heavy duty `bst artifact pull`, and is particularly useful with the advent of `%{artifact-cas-digest}` being available in `bst show` output. This updates the CLI to support the new arguments which we missed adding previously, and adds the glue code in Stream() object to load the pipeline with the artifact remote related arguments. --- src/buildstream/_frontend/cli.py | 23 ++++++++++++++++++++--- src/buildstream/_stream.py | 18 ++++++++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 964a03e04..081a7d46b 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1302,12 +1302,29 @@ def artifact(): ), help="The dependencies we also want to show", ) +@click.option( + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading artifacts (Since: 2.7)", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects (Since: 2.7)", +) @click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj -def artifact_show(app, deps, artifacts): - """show the cached state of artifacts""" +def artifact_show(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): + """Show the cached state of artifacts""" with app.initialized(): - targets = app.stream.artifact_show(artifacts, selection=deps) + targets = app.stream.artifact_show( + artifacts, + selection=deps, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) click.echo(app.logger.show_state_of_artifacts(targets)) sys.exit(0) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index c7ec4c660..312b9de55 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -788,11 +788,25 @@ def _export_artifact(self, tar, location, compression, target, hardlinks, virdir # # Args: # targets (str): Targets to show the cached state of + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # - def artifact_show(self, targets, *, selection=_PipelineSelection.NONE): + def artifact_show( + self, + targets, + *, + selection=_PipelineSelection.NONE, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ): # Obtain list of Element and/or ArtifactElement objects target_objects = self.load_selection( - targets, selection=selection, connect_artifact_cache=True, load_artifacts=True + targets, + selection=selection, + connect_artifact_cache=True, + load_artifacts=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) self.query_cache(target_objects) From 870cd66867de7783b18e45b6bc0acfb87ba8dc5a Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 9 Nov 2025 13:27:08 +0900 Subject: [PATCH 2/4] tests/frontend/artifact_show.py: Split out test files from common data dir This just follows the new, improved policy to not share data files across test cases, leading to the huge crazy shared mess that is found in tests/frontend/project, where any change here for a given test risks affecting many adjacent tests. --- .../frontend/artifact-show/elements/compose-all.bst | 12 ++++++++++++ tests/frontend/artifact-show/elements/import-bin.bst | 4 ++++ tests/frontend/artifact-show/elements/import-dev.bst | 4 ++++ tests/frontend/artifact-show/elements/manual.bst | 9 +++++++++ tests/frontend/artifact-show/elements/target.bst | 8 ++++++++ .../artifact-show/files/bin-files/usr/bin/hello | 3 +++ .../artifact-show/files/dev-files/usr/include/pony.h | 12 ++++++++++++ tests/frontend/artifact-show/project.conf | 10 ++++++++++ tests/frontend/artifact_show.py | 2 +- 9 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 tests/frontend/artifact-show/elements/compose-all.bst create mode 100644 tests/frontend/artifact-show/elements/import-bin.bst create mode 100644 tests/frontend/artifact-show/elements/import-dev.bst create mode 100644 tests/frontend/artifact-show/elements/manual.bst create mode 100644 tests/frontend/artifact-show/elements/target.bst create mode 100755 tests/frontend/artifact-show/files/bin-files/usr/bin/hello create mode 100644 tests/frontend/artifact-show/files/dev-files/usr/include/pony.h create mode 100644 tests/frontend/artifact-show/project.conf diff --git a/tests/frontend/artifact-show/elements/compose-all.bst b/tests/frontend/artifact-show/elements/compose-all.bst new file mode 100644 index 000000000..ba47081b3 --- /dev/null +++ b/tests/frontend/artifact-show/elements/compose-all.bst @@ -0,0 +1,12 @@ +kind: compose + +depends: +- filename: import-bin.bst + type: build +- filename: import-dev.bst + type: build + +config: + # Dont try running the sandbox, we dont have a + # runtime to run anything in this context. + integrate: False diff --git a/tests/frontend/artifact-show/elements/import-bin.bst b/tests/frontend/artifact-show/elements/import-bin.bst new file mode 100644 index 000000000..a847c0c23 --- /dev/null +++ b/tests/frontend/artifact-show/elements/import-bin.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/bin-files diff --git a/tests/frontend/artifact-show/elements/import-dev.bst b/tests/frontend/artifact-show/elements/import-dev.bst new file mode 100644 index 000000000..152a54667 --- /dev/null +++ b/tests/frontend/artifact-show/elements/import-dev.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/dev-files diff --git a/tests/frontend/artifact-show/elements/manual.bst b/tests/frontend/artifact-show/elements/manual.bst new file mode 100644 index 000000000..142409a08 --- /dev/null +++ b/tests/frontend/artifact-show/elements/manual.bst @@ -0,0 +1,9 @@ +kind: manual + +config: + build-commands: + - echo "hello" + +sources: + - kind: local + path: elements/manual.bst diff --git a/tests/frontend/artifact-show/elements/target.bst b/tests/frontend/artifact-show/elements/target.bst new file mode 100644 index 000000000..b9432fafa --- /dev/null +++ b/tests/frontend/artifact-show/elements/target.bst @@ -0,0 +1,8 @@ +kind: stack +description: | + + Main stack target for the bst build test + +depends: +- import-bin.bst +- compose-all.bst diff --git a/tests/frontend/artifact-show/files/bin-files/usr/bin/hello b/tests/frontend/artifact-show/files/bin-files/usr/bin/hello new file mode 100755 index 000000000..f534a4083 --- /dev/null +++ b/tests/frontend/artifact-show/files/bin-files/usr/bin/hello @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Hello !" diff --git a/tests/frontend/artifact-show/files/dev-files/usr/include/pony.h b/tests/frontend/artifact-show/files/dev-files/usr/include/pony.h new file mode 100644 index 000000000..40bd0c2e7 --- /dev/null +++ b/tests/frontend/artifact-show/files/dev-files/usr/include/pony.h @@ -0,0 +1,12 @@ +#ifndef __PONY_H__ +#define __PONY_H__ + +#define PONY_BEGIN "Once upon a time, there was a pony." +#define PONY_END "And they lived happily ever after, the end." + +#define MAKE_PONY(story) \ + PONY_BEGIN \ + story \ + PONY_END + +#endif /* __PONY_H__ */ diff --git a/tests/frontend/artifact-show/project.conf b/tests/frontend/artifact-show/project.conf new file mode 100644 index 000000000..7e690f56f --- /dev/null +++ b/tests/frontend/artifact-show/project.conf @@ -0,0 +1,10 @@ +# Project config for frontend build test +name: test +min-version: 2.0 +element-path: elements + +plugins: +- origin: pip + package-name: sample-plugins + sources: + - git diff --git a/tests/frontend/artifact_show.py b/tests/frontend/artifact_show.py index 32e6542c1..2b8e405c6 100644 --- a/tests/frontend/artifact_show.py +++ b/tests/frontend/artifact_show.py @@ -26,7 +26,7 @@ # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), - "project", + "artifact-show", ) SIMPLE_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), From 1c31c6ece8f90453b51c25e562f54e8d2a1ff913 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 9 Nov 2025 14:16:41 +0900 Subject: [PATCH 3/4] Fixup implementation --- src/buildstream/_stream.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 312b9de55..c6edf4a90 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -792,12 +792,12 @@ def _export_artifact(self, tar, location, compression, target, hardlinks, virdir # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # def artifact_show( - self, - targets, - *, - selection=_PipelineSelection.NONE, - artifact_remotes: Iterable[RemoteSpec] = (), - ignore_project_artifact_remotes: bool = False, + self, + targets, + *, + selection=_PipelineSelection.NONE, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, ): # Obtain list of Element and/or ArtifactElement objects target_objects = self.load_selection( From ccb6f5580b4f9211a1168c5e7f7fa33fde1067be Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Sun, 9 Nov 2025 14:39:37 +0900 Subject: [PATCH 4/4] tests/frontend/artifact_show.py: Add coverage for newly added options Added tests to cover --artifact-remote and --ignore-project-artifact-remotes. This also improves overall coverage of test_artifact_show_available_remotely to cover all three methods of setting the remotes (project, user config, and now CLI options), plus test this by providing the element name and the artifact name separately (except for when the remote is specified by the project, because project.conf is intentionally ignored when loading artifact names). --- tests/frontend/artifact_show.py | 95 +++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/tests/frontend/artifact_show.py b/tests/frontend/artifact_show.py index 2b8e405c6..1fd67144c 100644 --- a/tests/frontend/artifact_show.py +++ b/tests/frontend/artifact_show.py @@ -21,6 +21,7 @@ from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share +from . import configure_project # Project directory @@ -155,18 +156,96 @@ def test_artifact_show_glob(cli, tmpdir, datafiles, pattern, expected_prefixes): # Test artifact show artifact in remote @pytest.mark.datafiles(DATA_DIR) -def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles): +@pytest.mark.parametrize("config", ["config-project", "config-user", "config-cli"]) +@pytest.mark.parametrize("by_element_name", [True, False], ids=["by-element-name", "by-artifact-name"]) +def test_artifact_show_available_remotely(cli, tmpdir, datafiles, config, by_element_name): project = str(datafiles) element = "target.bst" + # + # Skip this configuration, BuildStream intentionally ignores the local project.conf + # if an artifact name is specified. + # + if config == "config-project" and not by_element_name: + pytest.skip("No project.conf in context") + # Set up remote and local shares local_cache = os.path.join(str(tmpdir), "artifacts") + cli.configure( + { + "cachedir": local_cache, + } + ) + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: - cli.configure( + extra_cli_args = [] + if config == "config-project": + configure_project( + project, + { + "artifacts": [ + { + "url": remote.repo, + "push": True, + } + ] + }, + ) + elif config == "config-user": + cli.configure( + { + "artifacts": {"servers": [{"url": remote.repo, "push": True}]}, + } + ) + else: + extra_cli_args = ["--artifact-remote", remote.repo] + + # Build the element + result = cli.run(project=project, args=["build"] + extra_cli_args + [element]) + result.assert_success() + + artifact_name = cli.get_artifact_name(project, "test", element) + + # Make sure it's in the share + assert remote.get_artifact(artifact_name) + + # Delete the artifact from the local cache + result = cli.run(project=project, args=["artifact", "delete", element]) + result.assert_success() + assert cli.get_element_state(project, element) != "cached" + + # Do the artifact show and assert + element_or_artifact = element if by_element_name else artifact_name + result = cli.run(project=project, args=["artifact", "show"] + extra_cli_args + [element_or_artifact]) + result.assert_success() + assert "available {}".format(element_or_artifact) in result.output + + +# Test out --ignore-project-artifact-remotes +@pytest.mark.datafiles(DATA_DIR) +def test_artifact_show_ignore_project_remotes(cli, tmpdir, datafiles): + project = str(datafiles) + element = "target.bst" + + # Set up remote and local shares + local_cache = os.path.join(str(tmpdir), "artifacts") + cli.configure( + { + "cachedir": local_cache, + } + ) + + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: + configure_project( + project, { - "artifacts": {"servers": [{"url": remote.repo, "push": True}]}, - "cachedir": local_cache, - } + "artifacts": [ + { + "url": remote.repo, + "push": True, + } + ] + }, ) # Build the element @@ -181,6 +260,12 @@ def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles): result.assert_success() assert cli.get_element_state(project, element) != "cached" + # It is available remotely with the project configured remote result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "available {}".format(element) in result.output + + # Ignoring project remotes, it is not found in any remote + result = cli.run(project=project, args=["artifact", "show", "--ignore-project-artifact-remotes", element]) + result.assert_success() + assert "not cached {}".format(element) in result.output