diff --git a/doc/Makefile b/doc/Makefile index eae1340..f19537c 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -72,12 +72,15 @@ clean: templates-clean templates: mkdir -p source/elements mkdir -p source/sources + mkdir -p source/sourcemirrors $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,elements) $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sources) + $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sourcemirrors) templates-clean: rm -rf source/elements rm -rf source/sources + rm -rf source/sourcemirrors # Targets which generate docs with sphinx build # @@ -88,7 +91,8 @@ html devhelp: templates $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \ $(wildcard source/*.rst) \ $(wildcard source/elements/*.rst) \ - $(wildcard source/sources/*.rst) + $(wildcard source/sources/*.rst) \ + $(wildcard source/sourcemirrors/*.rst) @echo @echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@" diff --git a/doc/source/index.rst b/doc/source/index.rst index 7aec4ea..88c5bc5 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -42,3 +42,10 @@ To these plugins in your project, follow the sources/git sources/patch sources/pip + +.. toctree:: + :maxdepth: 1 + :caption: Source Mirror Plugins + + sourcemirrors/simple_mirror + sourcemirrors/gitlab_lfs_mirror diff --git a/project.conf b/project.conf index a685d47..bad78fe 100644 --- a/project.conf +++ b/project.conf @@ -39,3 +39,9 @@ plugins: - git - patch - pip + +- origin: local + path: src/buildstream_plugins/sourcemirrors + source-mirrors: + - gitlab_lfs_mirror + - simple_mirror diff --git a/setup.py b/setup.py index b5dad20..5417244 100755 --- a/setup.py +++ b/setup.py @@ -92,6 +92,10 @@ "pip = buildstream_plugins.sources.pip", "zip = buildstream_plugins.sources.zip", ], + "buildstream.plugins.sourcemirrors": [ + "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror", + "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror", + ], }, extras_require={ "cargo": ['tomli; python_version < "3.11"'], diff --git a/src/buildstream_plugins/__init__.py b/src/buildstream_plugins/__init__.py index cfdef39..e7191e5 100644 --- a/src/buildstream_plugins/__init__.py +++ b/src/buildstream_plugins/__init__.py @@ -15,4 +15,4 @@ # # Remember to adjust this version number before tagging releases # -__version__ = "2.3.0" +__version__ = "2.4.0" diff --git a/src/buildstream_plugins/elements/make.py b/src/buildstream_plugins/elements/make.py index c9d6375..7e64b3c 100644 --- a/src/buildstream_plugins/elements/make.py +++ b/src/buildstream_plugins/elements/make.py @@ -26,11 +26,6 @@ `_ implementation for using GNU make based build. -.. note:: - - The ``make`` element is available since `format version 9 - `_ - Here is the default configuration for the ``make`` element in full: .. literalinclude:: ../../../src/buildstream_plugins/elements/make.yaml diff --git a/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py new file mode 100644 index 0000000..c258cd8 --- /dev/null +++ b/src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py @@ -0,0 +1,89 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab +========================================================================== + +.. note:: + + The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0* + +If you store files in a git-lfs repository on gitlab.com, you can access +them using a URL like + +https://gitlab.com/path/to/repo/-/raw/master/path/to/file + +which you can then use as a mirror for buildstream. However this only works for +public repositories. For internal (on self-hosted GitLab instances) and private +repositories, the above doesn't work since buildstream cannot authenticate with +the GitLab web UI. + +This plugin solves this by going through the GitLab REST API. You need an +access token that access the Repository Files API (i.e. with read_api or +read_repository scope). As of this writing, the GitLab CI/CD job token doesn't +allow access to this API endpoint, so you need a dedicated access token. + +**Usage:** + +.. code:: yaml + + - name: my-mirror + kind: gitlab_lfs_mirror + config: + url: https://gitlab.example.com/ + project: mirrors/{alias} + ref: main # optional, defaults to master + aliases: + - my-alias + - another-alias +""" + +from posixpath import join +from urllib.parse import quote + +from buildstream import SourceMirror + + +class GitlabLFSMirror(SourceMirror): + BST_MIN_VERSION = "2.2" + + def configure(self, node): + node.validate_keys(["aliases", "url", "project", "ref"]) + self.set_supported_aliases(node.get_str_list("aliases")) + + self.url = node.get_str("url") + self.project = node.get_str("project") + self.ref = node.get_str("ref", "master") + + def translate_url(self, alias, alias_url, source_url, extra_data): + project_id = quote(self.project.format(alias=alias), safe="") + filename = quote(source_url, safe="") + + translated_url = join( + self.url, + "api/v4/projects", + project_id, + "repository/files", + filename, + f"raw?ref={self.ref}&lfs=true", + ) + + if extra_data is not None: + extra_data["http-auth"] = "bearer" + + return translated_url + + +def setup(): + return GitlabLFSMirror diff --git a/src/buildstream_plugins/sourcemirrors/simple_mirror.py b/src/buildstream_plugins/sourcemirrors/simple_mirror.py new file mode 100644 index 0000000..031e632 --- /dev/null +++ b/src/buildstream_plugins/sourcemirrors/simple_mirror.py @@ -0,0 +1,59 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +simple_mirror - plugin for simplifying mirror definitions +========================================================= + +.. note:: + + The ``simple_mirror`` plugin is available *Since 2.4.0* + +**Usage:** + +.. code:: yaml + + - name: my-mirror + kind: simple_mirror + config: + url: https://example.com/mirrors/{alias}/ + aliases: + - my-alias + - another-alias + +This plugin simplifies defining mirrors for projects where the mirrors follow +a predictable URL format that only varies with the alias name. +""" + +from posixpath import join +from buildstream import SourceMirror + + +class SimpleMirror(SourceMirror): + BST_MIN_VERSION = "2.2" + + def configure(self, node): + node.validate_keys(["url", "aliases"]) + self.set_supported_aliases(node.get_str_list("aliases")) + + self.url = node.get_str("url") + + def translate_url(self, alias, alias_url, source_url, extra_data): + base_url = self.url.format(alias=alias) + translated_url = join(base_url, source_url) + + return translated_url + + +def setup(): + return SimpleMirror