Adding mirror plugins

doc/Makefile

@@ -72,12 +72,15 @@ clean: templates-clean
 templates:
 	mkdir -p source/elements
 	mkdir -p source/sources
+	mkdir -p source/sourcemirrors
 	$(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,elements)
 	$(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sources)
+	$(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream_plugins,sourcemirrors)
 
 templates-clean:
 	rm -rf source/elements
 	rm -rf source/sources
+	rm -rf source/sourcemirrors
 
 # Targets which generate docs with sphinx build
 #
@@ -88,7 +91,8 @@ html devhelp: templates
 	    $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
 	    $(wildcard source/*.rst) \
 	    $(wildcard source/elements/*.rst) \
-	    $(wildcard source/sources/*.rst)
+	    $(wildcard source/sources/*.rst) \
+	    $(wildcard source/sourcemirrors/*.rst)
 	@echo
 	@echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@"
 

doc/source/index.rst

@@ -42,3 +42,10 @@ To these plugins in your project, follow the
    sources/git
    sources/patch
    sources/pip
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Source Mirror Plugins
+
+   sourcemirrors/simple_mirror
+   sourcemirrors/gitlab_lfs_mirror

project.conf

@@ -39,3 +39,9 @@ plugins:
   - git
   - patch
   - pip
+
+- origin: local
+  path: src/buildstream_plugins/sourcemirrors
+  source-mirrors:
+  - gitlab_lfs_mirror
+  - simple_mirror

setup.py

@@ -92,6 +92,10 @@
             "pip = buildstream_plugins.sources.pip",
             "zip = buildstream_plugins.sources.zip",
         ],
+        "buildstream.plugins.sourcemirrors": [
+            "gitlab_lfs_mirror = buildstream_plugins.sourcemirrors.gitlab_lfs_mirror",
+            "simple_mirror = buildstream_plugins.sourcemirrors.simple_mirror",
+        ],
     },
     extras_require={
         "cargo": ['tomli; python_version < "3.11"'],

src/buildstream_plugins/__init__.py

@@ -15,4 +15,4 @@
 #
 # Remember to adjust this version number before tagging releases
 #
-__version__ = "2.3.0"
+__version__ = "2.4.0"

src/buildstream_plugins/elements/make.py

@@ -26,11 +26,6 @@
 <https://docs.buildstream.build/master/buildstream.scriptelement.html#module-buildstream.scriptelement>`_
 implementation for using GNU make based build.
 
-.. note::
-
-   The ``make`` element is available since `format version 9
-   <https://docs.buildstream.build/master/format_project.html#project-format-version>`_
-
 Here is the default configuration for the ``make`` element in full:
 
   .. literalinclude:: ../../../src/buildstream_plugins/elements/make.yaml

src/buildstream_plugins/sourcemirrors/gitlab_lfs_mirror.py

@@ -0,0 +1,89 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+gitlab_lfs_mirror - plugin for accessing files stored in git-lfs on GitLab
+==========================================================================
+
+.. note::
+
+   The ``gitlab_lfs_mirror`` plugin is available *Since 2.4.0*
+
+If you store files in a git-lfs repository on gitlab.com, you can access
+them using a URL like
+
+https://gitlab.com/path/to/repo/-/raw/master/path/to/file
+
+which you can then use as a mirror for buildstream. However this only works for
+public repositories. For internal (on self-hosted GitLab instances) and private
+repositories, the above doesn't work since buildstream cannot authenticate with
+the GitLab web UI.
+
+This plugin solves this by going through the GitLab REST API. You need an
+access token that access the Repository Files API (i.e. with read_api or
+read_repository scope). As of this writing, the GitLab CI/CD job token doesn't
+allow access to this API endpoint, so you need a dedicated access token.
+
+**Usage:**
+
+.. code:: yaml
+
+   - name: my-mirror
+     kind: gitlab_lfs_mirror
+     config:
+       url: https://gitlab.example.com/
+       project: mirrors/{alias}
+       ref: main # optional, defaults to master
+       aliases:
+       - my-alias
+       - another-alias
+"""
+
+from posixpath import join
+from urllib.parse import quote
+
+from buildstream import SourceMirror
+
+
+class GitlabLFSMirror(SourceMirror):
+    BST_MIN_VERSION = "2.2"
+
+    def configure(self, node):
+        node.validate_keys(["aliases", "url", "project", "ref"])
+        self.set_supported_aliases(node.get_str_list("aliases"))
+
+        self.url = node.get_str("url")
+        self.project = node.get_str("project")
+        self.ref = node.get_str("ref", "master")
+
+    def translate_url(self, alias, alias_url, source_url, extra_data):
+        project_id = quote(self.project.format(alias=alias), safe="")
+        filename = quote(source_url, safe="")
+
+        translated_url = join(
+            self.url,
+            "api/v4/projects",
+            project_id,
+            "repository/files",
+            filename,
+            f"raw?ref={self.ref}&lfs=true",
+        )
+
+        if extra_data is not None:
+            extra_data["http-auth"] = "bearer"
+
+        return translated_url
+
+
+def setup():
+    return GitlabLFSMirror

src/buildstream_plugins/sourcemirrors/simple_mirror.py

@@ -0,0 +1,59 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+simple_mirror - plugin for simplifying mirror definitions
+=========================================================
+
+.. note::
+
+   The ``simple_mirror`` plugin is available *Since 2.4.0*
+
+**Usage:**
+
+.. code:: yaml
+
+   - name: my-mirror
+     kind: simple_mirror
+     config:
+       url: https://example.com/mirrors/{alias}/
+       aliases:
+       - my-alias
+       - another-alias
+
+This plugin simplifies defining mirrors for projects where the mirrors follow
+a predictable URL format that only varies with the alias name.
+"""
+
+from posixpath import join
+from buildstream import SourceMirror
+
+
+class SimpleMirror(SourceMirror):
+    BST_MIN_VERSION = "2.2"
+
+    def configure(self, node):
+        node.validate_keys(["url", "aliases"])
+        self.set_supported_aliases(node.get_str_list("aliases"))
+
+        self.url = node.get_str("url")
+
+    def translate_url(self, alias, alias_url, source_url, extra_data):
+        base_url = self.url.format(alias=alias)
+        translated_url = join(base_url, source_url)
+
+        return translated_url
+
+
+def setup():
+    return SimpleMirror