From af7c9a4fe237060529d8d349724996ac8d5efcde Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Mon, 14 Oct 2024 18:33:01 +0900 Subject: [PATCH 1/4] make plugin: Remove bogus since version from the doc string This makes reference to the format-version from BuildStream 1, but this plugin is for BuildStream 2 and has been available since the initial buildstream-plugins release. --- src/buildstream_plugins/elements/make.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/buildstream_plugins/elements/make.py b/src/buildstream_plugins/elements/make.py index c9d6375..7e64b3c 100644 --- a/src/buildstream_plugins/elements/make.py +++ b/src/buildstream_plugins/elements/make.py @@ -26,11 +26,6 @@ `_ implementation for using GNU make based build. -.. note:: - - The ``make`` element is available since `format version 9 - `_ - Here is the default configuration for the ``make`` element in full: .. literalinclude:: ../../../src/buildstream_plugins/elements/make.yaml From d74f0b075d18a84628629032fcd1b14d3bf4fbc3 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Mon, 14 Oct 2024 18:31:08 +0900 Subject: [PATCH 2/4] src/buildstream_plugins/__init__.py: Bump version Bumping version to 2.4.0, for the addition of the source mirror plugins. --- src/buildstream_plugins/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/buildstream_plugins/__init__.py b/src/buildstream_plugins/__init__.py index cfdef39..e7191e5 100644 --- a/src/buildstream_plugins/__init__.py +++ b/src/buildstream_plugins/__init__.py @@ -15,4 +15,4 @@ # # Remember to adjust this version number before tagging releases # -__version__ = "2.3.0" +__version__ = "2.4.0" From d0d7e82381bf2ce74273dce819b9db77fdce52c9 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Mon, 14 Oct 2024 18:24:15 +0900 Subject: [PATCH 3/4] Adding simple_mirror source mirror plugin Since we've added the SourceMirror feature in BuildStream 2.2, it makes sense to add this mirror plugin upstream. This is especially helpful as we can rely on loading stable buildstream-plugins via the `pip` plugin origin for loading source mirror plugins, avoiding a cyclic dependency issue if you want to use mirror plugins to load plugins via the `junction` plugin origin. --- doc/Makefile | 6 +- doc/source/index.rst | 6 ++ project.conf | 5 ++ setup.py | 3 + .../sourcemirrors/simple_mirror.py | 59 +++++++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 src/buildstream_plugins/sourcemirrors/simple_mirror.py diff --git a/doc/Makefile b/doc/Makefile index eae1340..f19537c 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -72,12 +72,15 @@ clean: templates-clean templates: mkdir -p source/elements mkdir -p source/sources + mkdir -p source/sourcemirrors $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,elements) $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sources) + $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sourcemirrors) templates-clean: rm -rf source/elements rm -rf source/sources + rm -rf source/sourcemirrors # Targets which generate docs with sphinx build # @@ -88,7 +91,8 @@ html devhelp: templates $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \ $(wildcard source/*.rst) \ $(wildcard source/elements/*.rst) \ - $(wildcard source/sources/*.rst) + $(wildcard source/sources/*.rst) \ + $(wildcard source/sourcemirrors/*.rst) @echo @echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@" diff --git a/doc/source/index.rst b/doc/source/index.rst index 7aec4ea..95b6f8c 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -42,3 +42,9 @@ To these plugins in your project, follow the sources/git sources/patch sources/pip + +.. toctree:: + :maxdepth: 1 + :caption: Source Mirror Plugins + + sourcemirrors/simple_mirror diff --git a/project.conf b/project.conf index a685d47..301bb08 100644 --- a/project.conf +++ b/project.conf @@ -39,3 +39,8 @@ plugins: - git - patch - pip + +- origin: local + path: src/buildstream_plugins/sourcemirrors + source-mirrors: + - simple_mirror diff --git a/setup.py b/setup.py index b5dad20..b7de25b 100755 --- a/setup.py +++ b/setup.py @@ -92,6 +92,9 @@ "pip = buildstream_plugins.sources.pip", "zip = buildstream_plugins.sources.zip", ], + "buildstream.plugins.sourcemirrors": [ + "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror", + ], }, extras_require={ "cargo": ['tomli; python_version < "3.11"'], diff --git a/src/buildstream_plugins/sourcemirrors/simple_mirror.py b/src/buildstream_plugins/sourcemirrors/simple_mirror.py new file mode 100644 index 0000000..031e632 --- /dev/null +++ b/src/buildstream_plugins/sourcemirrors/simple_mirror.py @@ -0,0 +1,59 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +simple_mirror - plugin for simplifying mirror definitions +========================================================= + +.. note:: + + The ``simple_mirror`` plugin is available *Since 2.4.0* + +**Usage:** + +.. code:: yaml + + - name: my-mirror + kind: simple_mirror + config: + url: https://example.com/mirrors/{alias}/ + aliases: + - my-alias + - another-alias + +This plugin simplifies defining mirrors for projects where the mirrors follow +a predictable URL format that only varies with the alias name. +""" + +from posixpath import join +from buildstream import SourceMirror + + +class SimpleMirror(SourceMirror): + BST_MIN_VERSION = "2.2" + + def configure(self, node): + node.validate_keys(["url", "aliases"]) + self.set_supported_aliases(node.get_str_list("aliases")) + + self.url = node.get_str("url") + + def translate_url(self, alias, alias_url, source_url, extra_data): + base_url = self.url.format(alias=alias) + translated_url = join(base_url, source_url) + + return translated_url + + +def setup(): + return SimpleMirror From 67739037d35f0f815362636f84e01ae9a567a6c1 Mon Sep 17 00:00:00 2001 From: Tristan van Berkom Date: Mon, 14 Oct 2024 18:28:56 +0900 Subject: [PATCH 4/4] Adding gitlab_lfs_mirror source mirror plugin Since we've added the SourceMirror feature in BuildStream 2.2, it makes sense to add this mirror plugin upstream. This is especially helpful as we can rely on loading stable buildstream-plugins via the `pip` plugin origin for loading source mirror plugins, avoiding a cyclic dependency issue if you want to use mirror plugins to load plugins via the `junction` plugin origin. --- doc/source/index.rst | 1 + project.conf | 1 + setup.py | 1 + .../sourcemirrors/gitlab_lfs_mirror.py | 89 +++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py diff --git a/doc/source/index.rst b/doc/source/index.rst index 95b6f8c..88c5bc5 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -48,3 +48,4 @@ To these plugins in your project, follow the :caption: Source Mirror Plugins sourcemirrors/simple_mirror + sourcemirrors/gitlab_lfs_mirror diff --git a/project.conf b/project.conf index 301bb08..bad78fe 100644 --- a/project.conf +++ b/project.conf @@ -43,4 +43,5 @@ plugins: - origin: local path: src/buildstream_plugins/sourcemirrors source-mirrors: + - gitlab_lfs_mirror - simple_mirror diff --git a/setup.py b/setup.py index b7de25b..5417244 100755 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ "zip = buildstream_plugins.sources.zip", ], "buildstream.plugins.sourcemirrors": [ + "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror", "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror", ], }, diff --git a/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py new file mode 100644 index 0000000..c258cd8 --- /dev/null +++ b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py @@ -0,0 +1,89 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab +========================================================================== + +.. note:: + + The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0* + +If you store files in a git-lfs repository on gitlab.com, you can access +them using a URL like + +https://gitlab.com/path/to/repo/-/raw/master/path/to/file + +which you can then use as a mirror for buildstream. However this only works for +public repositories. For internal (on self-hosted GitLab instances) and private +repositories, the above doesn't work since buildstream cannot authenticate with +the GitLab web UI. + +This plugin solves this by going through the GitLab REST API. You need an +access token that access the Repository Files API (i.e. with read_api or +read_repository scope). As of this writing, the GitLab CI/CD job token doesn't +allow access to this API endpoint, so you need a dedicated access token. + +**Usage:** + +.. code:: yaml + + - name: my-mirror + kind: gitlab_lfs_mirror + config: + url: https://gitlab.example.com/ + project: mirrors/{alias} + ref: main # optional, defaults to master + aliases: + - my-alias + - another-alias +""" + +from posixpath import join +from urllib.parse import quote + +from buildstream import SourceMirror + + +class GitlabLFSMirror(SourceMirror): + BST_MIN_VERSION = "2.2" + + def configure(self, node): + node.validate_keys(["aliases", "url", "project", "ref"]) + self.set_supported_aliases(node.get_str_list("aliases")) + + self.url = node.get_str("url") + self.project = node.get_str("project") + self.ref = node.get_str("ref", "master") + + def translate_url(self, alias, alias_url, source_url, extra_data): + project_id = quote(self.project.format(alias=alias), safe="") + filename = quote(source_url, safe="") + + translated_url = join( + self.url, + "api/v4/projects", + project_id, + "repository/files", + filename, + f"raw?ref={self.ref}&lfs=true", + ) + + if extra_data is not None: + extra_data["http-auth"] = "bearer" + + return translated_url + + +def setup(): + return GitlabLFSMirror