diff --git a/doc/source/hacking/writing_documentation.rst b/doc/source/hacking/writing_documentation.rst index f744c6311..c0f22885a 100644 --- a/doc/source/hacking/writing_documentation.rst +++ b/doc/source/hacking/writing_documentation.rst @@ -112,9 +112,11 @@ into the ``setup.py``, as such, whenever the frontend command line interface changes, the static man pages should be regenerated and committed with that. -To do this, run the following from the the toplevel directory of BuildStream:: +To do this, make sure you know the version you are generating the docs +for, which should usually be the *next* ``major.minor`` version of BuildStream, +and then run the following from the the toplevel directory of BuildStream:: - tox -e man + tox -e man -- And commit the result, ensuring that you have added anything in the ``man/`` subdirectory, which will be automatically included diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1 index 89cf7f2a1..34b22c991 100644 --- a/man/bst-artifact-checkout.1 +++ b/man/bst-artifact-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT CHECKOUT" "1" "2024-08-08" "" "bst artifact checkout Manual" +.TH "BST ARTIFACT CHECKOUT" "1" "2025-04-24" "2.5" "bst artifact checkout Manual" .SH NAME bst\-artifact\-checkout \- Checkout contents of an artifact .SH SYNOPSIS @@ -7,9 +7,9 @@ bst\-artifact\-checkout \- Checkout contents of an artifact .SH DESCRIPTION Checkout contents of an artifact .PP - When this command is executed from a workspace directory, the default - is to checkout the artifact of the workspace element. - +When this command is executed from a workspace directory, the default +is to checkout the artifact of the workspace element. +.PP .SH OPTIONS .TP \fB\-f,\fP \-\-force diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1 index 733d9bdc8..ea650ccb0 100644 --- a/man/bst-artifact-delete.1 +++ b/man/bst-artifact-delete.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT DELETE" "1" "2024-08-08" "" "bst artifact delete Manual" +.TH "BST ARTIFACT DELETE" "1" "2025-04-24" "2.5" "bst artifact delete Manual" .SH NAME bst\-artifact\-delete \- Remove artifacts from the local cache .SH SYNOPSIS diff --git a/man/bst-artifact-list-contents.1 b/man/bst-artifact-list-contents.1 index 25156fa4a..86ea9a2c4 100644 --- a/man/bst-artifact-list-contents.1 +++ b/man/bst-artifact-list-contents.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LIST-CONTENTS" "1" "2024-08-08" "" "bst artifact list-contents Manual" +.TH "BST ARTIFACT LIST-CONTENTS" "1" "2025-04-24" "2.5" "bst artifact list-contents Manual" .SH NAME bst\-artifact\-list-contents \- List the contents of an artifact .SH SYNOPSIS @@ -7,9 +7,9 @@ bst\-artifact\-list-contents \- List the contents of an artifact .SH DESCRIPTION List the contents of an artifact. .PP - Note that 'artifacts' can be element names, which must end in '.bst', - or artifact references, which must be in the format `//`. - +Note that 'artifacts' can be element names, which must end in '.bst', +or artifact references, which must be in the format `//`. +.PP .SH OPTIONS .TP \fB\-l,\fP \-\-long diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1 index b31f9e29f..abd09ebbc 100644 --- a/man/bst-artifact-log.1 +++ b/man/bst-artifact-log.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LOG" "1" "2024-08-08" "" "bst artifact log Manual" +.TH "BST ARTIFACT LOG" "1" "2025-04-24" "2.5" "bst artifact log Manual" .SH NAME bst\-artifact\-log \- Show logs of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1 index 2ebe98daa..239a3b3f2 100644 --- a/man/bst-artifact-pull.1 +++ b/man/bst-artifact-pull.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PULL" "1" "2024-08-08" "" "bst artifact pull Manual" +.TH "BST ARTIFACT PULL" "1" "2025-04-24" "2.5" "bst artifact pull Manual" .SH NAME bst\-artifact\-pull \- Pull a built artifact .SH SYNOPSIS @@ -7,25 +7,25 @@ bst\-artifact\-pull \- Pull a built artifact .SH DESCRIPTION Pull a built artifact from the configured remote artifact cache. .PP - Specifying no elements will result in pulling the default targets - of the project. If no default targets are configured, all project - elements will be pulled. +Specifying no elements will result in pulling the default targets +of the project. If no default targets are configured, all project +elements will be pulled. .PP - When this command is executed from a workspace directory, the default - is to pull the workspace element. +When this command is executed from a workspace directory, the default +is to pull the workspace element. .PP - By default the artifact will be pulled one of the configured caches - if possible, following the usual priority order. If the `--artifact-remote` - flag is given, only the specified cache will be queried. +By default the artifact will be pulled one of the configured caches +if possible, following the usual priority order. If the `--artifact-remote` +flag is given, only the specified cache will be queried. .PP - Specify `--deps` to control which artifacts to pull: +Specify `--deps` to control which artifacts to pull: +.PP + + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies .PP -  - none: No dependencies, just the element itself - run: Runtime dependencies, including the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies - .SH OPTIONS .TP \fB\-d,\fP \-\-deps [build|none|run|all] diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1 index 3ed364675..334b95ab7 100644 --- a/man/bst-artifact-push.1 +++ b/man/bst-artifact-push.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PUSH" "1" "2024-08-08" "" "bst artifact push Manual" +.TH "BST ARTIFACT PUSH" "1" "2025-04-24" "2.5" "bst artifact push Manual" .SH NAME bst\-artifact\-push \- Push a built artifact .SH SYNOPSIS @@ -7,25 +7,25 @@ bst\-artifact\-push \- Push a built artifact .SH DESCRIPTION Push built artifacts to a remote artifact cache, possibly pulling them first. .PP - Specifying no elements will result in pushing the default targets - of the project. If no default targets are configured, all project - elements will be pushed. +Specifying no elements will result in pushing the default targets +of the project. If no default targets are configured, all project +elements will be pushed. .PP - When this command is executed from a workspace directory, the default - is to push the workspace element. +When this command is executed from a workspace directory, the default +is to push the workspace element. .PP - If bst has been configured to include build trees on artifact pulls, - an attempt will be made to pull any required build trees to avoid the - skipping of partial artifacts being pushed. +If bst has been configured to include build trees on artifact pulls, +an attempt will be made to pull any required build trees to avoid the +skipping of partial artifacts being pushed. .PP - Specify `--deps` to control which artifacts to push: +Specify `--deps` to control which artifacts to push: +.PP + + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies .PP -  - none: No dependencies, just the element itself - run: Runtime dependencies, including the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies - .SH OPTIONS .TP \fB\-d,\fP \-\-deps [build|none|run|all] diff --git a/man/bst-artifact-show.1 b/man/bst-artifact-show.1 index 0c69bea29..7d1fa4223 100644 --- a/man/bst-artifact-show.1 +++ b/man/bst-artifact-show.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT SHOW" "1" "2024-08-08" "" "bst artifact show Manual" +.TH "BST ARTIFACT SHOW" "1" "2025-04-24" "2.5" "bst artifact show Manual" .SH NAME bst\-artifact\-show \- Show the cached state of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact.1 b/man/bst-artifact.1 index 83f5919bf..9086a0313 100644 --- a/man/bst-artifact.1 +++ b/man/bst-artifact.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT" "1" "2024-08-08" "" "bst artifact Manual" +.TH "BST ARTIFACT" "1" "2025-04-24" "2.5" "bst artifact Manual" .SH NAME bst\-artifact \- Manipulate cached artifacts. .SH SYNOPSIS @@ -7,34 +7,34 @@ bst\-artifact \- Manipulate cached artifacts. .SH DESCRIPTION Manipulate cached artifacts .PP - Some subcommands take artifact references as arguments. Artifacts - can be specified in two ways: +Some subcommands take artifact references as arguments. Artifacts +can be specified in two ways: .PP -  - - artifact refs: triples of the form // - - element names + +- artifact refs: triples of the form // +- element names .PP - When elements are given, the artifact is looked up by observing the element - and it's current cache key. +When elements are given, the artifact is looked up by observing the element +and it's current cache key. .PP - The commands also support shell-style wildcard expansion: `?` matches a - single character, `*` matches zero or more characters but does not match the `/` - path separator, and `**` matches zero or more characters including `/` path separators. +The commands also support shell-style wildcard expansion: `?` matches a +single character, `*` matches zero or more characters but does not match the `/` +path separator, and `**` matches zero or more characters including `/` path separators. .PP - If the wildcard expression ends with `.bst`, then it will be used to search - element names found in the project, otherwise, it will be used to search artifacts - that are present in the local artifact cache. +If the wildcard expression ends with `.bst`, then it will be used to search +element names found in the project, otherwise, it will be used to search artifacts +that are present in the local artifact cache. .PP - Some example arguments are: +Some example arguments are: +.PP + +- `myproject/hello/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` +- `myproject/he*/827637*` +- `core/*.bst` (all elements in the core directory) +- `**.bst` (all elements) +- `myproject/**` (all artifacts from myproject) +- `myproject/myelement/*` (all cached artifacts for a specific element) .PP -  - - `myproject/hello/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` - - `myproject/he*/827637*` - - `core/*.bst` (all elements in the core directory) - - `**.bst` (all elements) - - `myproject/**` (all artifacts from myproject) - - `myproject/myelement/*` (all cached artifacts for a specific element) - .SH COMMANDS .PP \fBshow\fP diff --git a/man/bst-build.1 b/man/bst-build.1 index e5de53cb1..86c826596 100644 --- a/man/bst-build.1 +++ b/man/bst-build.1 @@ -1,4 +1,4 @@ -.TH "BST BUILD" "1" "2024-08-08" "" "bst build Manual" +.TH "BST BUILD" "1" "2025-04-24" "2.5" "bst build Manual" .SH NAME bst\-build \- Build elements in a pipeline .SH SYNOPSIS @@ -7,23 +7,23 @@ bst\-build \- Build elements in a pipeline .SH DESCRIPTION Build elements in a pipeline .PP - Specifying no elements will result in building the default targets - of the project. If no default targets are configured, all project - elements will be built. +Specifying no elements will result in building the default targets +of the project. If no default targets are configured, all project +elements will be built. .PP - When this command is executed from a workspace directory, the default - is to build the workspace element. +When this command is executed from a workspace directory, the default +is to build the workspace element. .PP - Specify `--deps` to control which dependencies must be built: +Specify `--deps` to control which dependencies must be built: .PP -  - none: No dependencies, just the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies + + none: No dependencies, just the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies +.PP +Dependencies that are consequently required to build the requested +elements will be built on demand. .PP - Dependencies that are consequently required to build the requested - elements will be built on demand. - .SH OPTIONS .TP \fB\-d,\fP \-\-deps [none|build|all] diff --git a/man/bst-help.1 b/man/bst-help.1 index 7e4ef5af8..e475127ce 100644 --- a/man/bst-help.1 +++ b/man/bst-help.1 @@ -1,4 +1,4 @@ -.TH "BST HELP" "1" "2024-08-08" "" "bst help Manual" +.TH "BST HELP" "1" "2025-04-24" "2.5" "bst help Manual" .SH NAME bst\-help \- Print usage information .SH SYNOPSIS diff --git a/man/bst-init.1 b/man/bst-init.1 index 600bacb86..80d2f3251 100644 --- a/man/bst-init.1 +++ b/man/bst-init.1 @@ -1,4 +1,4 @@ -.TH "BST INIT" "1" "2024-08-08" "" "bst init Manual" +.TH "BST INIT" "1" "2025-04-24" "2.5" "bst init Manual" .SH NAME bst\-init \- Initialize a new BuildStream project .SH SYNOPSIS @@ -7,19 +7,19 @@ bst\-init \- Initialize a new BuildStream project .SH DESCRIPTION Initialize a new BuildStream project .PP - Creates a new BuildStream project.conf in the project - directory. +Creates a new BuildStream project.conf in the project +directory. +.PP +Unless `--project-name` is specified, this will be an +interactive session. .PP - Unless `--project-name` is specified, this will be an - interactive session. - .SH OPTIONS .TP \fB\-\-project\-name\fP TEXT The project name to use .TP \fB\-\-min\-version\fP TEXT -The required format version [default: 2.3] +The required format version [default: 2.4] .TP \fB\-\-element\-path\fP PATH The subdirectory to store elements in [default: elements] diff --git a/man/bst-shell.1 b/man/bst-shell.1 index 387f5f128..50bfa25d9 100644 --- a/man/bst-shell.1 +++ b/man/bst-shell.1 @@ -1,4 +1,4 @@ -.TH "BST SHELL" "1" "2024-08-08" "" "bst shell Manual" +.TH "BST SHELL" "1" "2025-04-24" "2.5" "bst shell Manual" .SH NAME bst\-shell \- Shell into an element's sandbox environment .SH SYNOPSIS @@ -7,25 +7,25 @@ bst\-shell \- Shell into an element's sandbox environment .SH DESCRIPTION Run a command in the target element's sandbox environment .PP - When this command is executed from a workspace directory, the default - is to shell into the workspace element. +When this command is executed from a workspace directory, the default +is to shell into the workspace element. .PP - This will stage a temporary sysroot for running the target - element, assuming it has already been built and all required - artifacts are in the local cache. +This will stage a temporary sysroot for running the target +element, assuming it has already been built and all required +artifacts are in the local cache. .PP - Use '--' to separate a command from the options to bst, - otherwise bst may respond to them instead. e.g. +Use '--' to separate a command from the options to bst, +otherwise bst may respond to them instead. e.g. .PP -  - bst shell example.bst -- df -h + + bst shell example.bst -- df -h .PP - Use the --build option to create a temporary sysroot for - building the element instead. +Use the --build option to create a temporary sysroot for +building the element instead. +.PP +If no COMMAND is specified, the default is to attempt +to run an interactive shell. .PP - If no COMMAND is specified, the default is to attempt - to run an interactive shell. - .SH OPTIONS .TP \fB\-b,\fP \-\-build diff --git a/man/bst-show.1 b/man/bst-show.1 index b71281156..1524d18d1 100644 --- a/man/bst-show.1 +++ b/man/bst-show.1 @@ -1,4 +1,4 @@ -.TH "BST SHOW" "1" "2024-08-08" "" "bst show Manual" +.TH "BST SHOW" "1" "2025-04-24" "2.5" "bst show Manual" .SH NAME bst\-show \- Show elements in the pipeline .SH SYNOPSIS @@ -7,59 +7,60 @@ bst\-show \- Show elements in the pipeline .SH DESCRIPTION Show elements in the pipeline .PP - Specifying no elements will result in showing the default targets - of the project. If no default targets are configured, all project - elements will be shown. +Specifying no elements will result in showing the default targets +of the project. If no default targets are configured, all project +elements will be shown. .PP - When this command is executed from a workspace directory, the default - is to show the workspace element. +When this command is executed from a workspace directory, the default +is to show the workspace element. .PP - By default this will show all of the dependencies of the - specified target element. +By default this will show all of the dependencies of the +specified target element. .PP - Specify ``--deps`` to control which elements to show: +Specify ``--deps`` to control which elements to show: .PP -  - none: No dependencies, just the element itself - run: Runtime dependencies, including the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies + + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies .PP - **FORMAT** +**FORMAT** .PP - The ``--format`` option controls what should be printed for each element, - the following symbols can be used in the format string: +The ``--format`` option controls what should be printed for each element, +the following symbols can be used in the format string: .PP -  - %{name} The element name - %{description} The element description, on a single line (Since: 2.3) - %{key} The abbreviated cache key (if all sources are consistent) - %{full-key} The full cache key (if all sources are consistent) - %{state} cached, buildable, waiting, inconsistent or junction - %{config} The element configuration - %{vars} Variable configuration - %{env} Environment settings - %{public} Public domain data - %{workspaced} If the element is workspaced - %{workspace-dirs} A list of workspace directories - %{deps} A list of all dependencies - %{build-deps} A list of build dependencies - %{runtime-deps} A list of runtime dependencies + + %{name} The element name + %{description} The element description, on a single line (Since: 2.3) + %{key} The abbreviated cache key (if all sources are consistent) + %{full-key} The full cache key (if all sources are consistent) + %{state} cached, buildable, waiting, inconsistent or junction + %{config} The element configuration + %{vars} Variable configuration + %{env} Environment settings + %{public} Public domain data + %{workspaced} If the element is workspaced + %{workspace-dirs} A list of workspace directories + %{deps} A list of all dependencies + %{build-deps} A list of build dependencies + %{runtime-deps} A list of runtime dependencies + %{source-info} Source provenance information .PP - The value of the %{symbol} without the leading '%' character is understood - as a pythonic formatting string, so python formatting features apply, - example: +The value of the %{symbol} without the leading '%' character is understood +as a pythonic formatting string, so python formatting features apply, +example: .PP -  - bst show target.bst --format \ - 'Name: %{name: ^20} Key: %{key: ^8} State: %{state}' + + bst show target.bst --format \ + 'Name: %{name: ^20} Key: %{key: ^8} State: %{state}' .PP - If you want to use a newline in a format string in bash, use the '$' modifier: +If you want to use a newline in a format string in bash, use the '$' modifier: +.PP + + bst show target.bst --format \ + $'---------- %{name} ----------\n%{vars}' .PP -  - bst show target.bst --format \ - $'---------- %{name} ----------\n%{vars}' - .SH OPTIONS .TP \fB\-\-except\fP PATH diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1 index 9fc31afc8..190b54b78 100644 --- a/man/bst-source-checkout.1 +++ b/man/bst-source-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE CHECKOUT" "1" "2024-08-08" "" "bst source checkout Manual" +.TH "BST SOURCE CHECKOUT" "1" "2025-04-24" "2.5" "bst source checkout Manual" .SH NAME bst\-source\-checkout \- Checkout sources of an element .SH SYNOPSIS @@ -7,9 +7,9 @@ bst\-source\-checkout \- Checkout sources of an element .SH DESCRIPTION Checkout sources of an element to the specified location .PP - When this command is executed from a workspace directory, the default - is to checkout the sources of the workspace element. - +When this command is executed from a workspace directory, the default +is to checkout the sources of the workspace element. +.PP .SH OPTIONS .TP \fB\-f,\fP \-\-force diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1 index 58d735886..258dab668 100644 --- a/man/bst-source-fetch.1 +++ b/man/bst-source-fetch.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE FETCH" "1" "2024-08-08" "" "bst source fetch Manual" +.TH "BST SOURCE FETCH" "1" "2025-04-24" "2.5" "bst source fetch Manual" .SH NAME bst\-source\-fetch \- Fetch sources in a pipeline .SH SYNOPSIS @@ -7,24 +7,24 @@ bst\-source\-fetch \- Fetch sources in a pipeline .SH DESCRIPTION Fetch sources required to build the pipeline .PP - Specifying no elements will result in fetching the default targets - of the project. If no default targets are configured, all project - elements will be fetched. +Specifying no elements will result in fetching the default targets +of the project. If no default targets are configured, all project +elements will be fetched. .PP - When this command is executed from a workspace directory, the default - is to fetch the workspace element. +When this command is executed from a workspace directory, the default +is to fetch the workspace element. .PP - By default this will only try to fetch sources for the specified - elements. +By default this will only try to fetch sources for the specified +elements. .PP - Specify `--deps` to control which sources to fetch: +Specify `--deps` to control which sources to fetch: +.PP + + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies .PP -  - none: No dependencies, just the element itself - run: Runtime dependencies, including the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies - .SH OPTIONS .TP \fB\-\-except\fP PATH diff --git a/man/bst-source-push.1 b/man/bst-source-push.1 index 560281606..6a71818e7 100644 --- a/man/bst-source-push.1 +++ b/man/bst-source-push.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE PUSH" "1" "2024-08-08" "" "bst source push Manual" +.TH "BST SOURCE PUSH" "1" "2025-04-24" "2.5" "bst source push Manual" .SH NAME bst\-source\-push \- Push sources in a pipeline .SH SYNOPSIS @@ -7,24 +7,24 @@ bst\-source\-push \- Push sources in a pipeline .SH DESCRIPTION Push sources required to build the pipeline .PP - Specifying no elements will result in pushing the sources of the default - targets of the project. If no default targets are configured, sources of - all project elements will be pushed. +Specifying no elements will result in pushing the sources of the default +targets of the project. If no default targets are configured, sources of +all project elements will be pushed. .PP - When this command is executed from a workspace directory, the default - is to push the sources of the workspace element. +When this command is executed from a workspace directory, the default +is to push the sources of the workspace element. .PP - By default this will only try to push sources for the specified - elements. +By default this will only try to push sources for the specified +elements. .PP - Specify `--deps` to control which sources to fetch: +Specify `--deps` to control which sources to fetch: +.PP + + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies .PP -  - none: No dependencies, just the element itself - run: Runtime dependencies, including the element itself - build: Build time dependencies, excluding the element itself - all: All dependencies - .SH OPTIONS .TP \fB\-\-except\fP PATH diff --git a/man/bst-source-track.1 b/man/bst-source-track.1 index a6207c7b7..fdcd2ad5e 100644 --- a/man/bst-source-track.1 +++ b/man/bst-source-track.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE TRACK" "1" "2024-08-08" "" "bst source track Manual" +.TH "BST SOURCE TRACK" "1" "2025-04-24" "2.5" "bst source track Manual" .SH NAME bst\-source\-track \- Track new source references .SH SYNOPSIS @@ -6,26 +6,26 @@ bst\-source\-track \- Track new source references [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Consults the specified tracking branches for new versions available - to build and updates the project with any newly available references. +to build and updates the project with any newly available references. .PP - Specifying no elements will result in tracking the default targets - of the project. If no default targets are configured, all project - elements will be tracked. +Specifying no elements will result in tracking the default targets +of the project. If no default targets are configured, all project +elements will be tracked. .PP - When this command is executed from a workspace directory, the default - is to track the workspace element. +When this command is executed from a workspace directory, the default +is to track the workspace element. .PP - If no default is declared, all elements in the project will be tracked +If no default is declared, all elements in the project will be tracked .PP - By default this will track just the specified element, but you can also - update a whole tree of dependencies in one go. +By default this will track just the specified element, but you can also +update a whole tree of dependencies in one go. .PP - Specify `--deps` to control which sources to track: +Specify `--deps` to control which sources to track: +.PP + + none: No dependencies, just the specified elements + all: All dependencies of all specified elements .PP -  - none: No dependencies, just the specified elements - all: All dependencies of all specified elements - .SH OPTIONS .TP \fB\-\-except\fP PATH diff --git a/man/bst-source.1 b/man/bst-source.1 index a041bba94..6730bd665 100644 --- a/man/bst-source.1 +++ b/man/bst-source.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE" "1" "2024-08-08" "" "bst source Manual" +.TH "BST SOURCE" "1" "2025-04-24" "2.5" "bst source Manual" .SH NAME bst\-source \- Manipulate sources for an element .SH SYNOPSIS diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1 index d73a587ed..b6f223a1e 100644 --- a/man/bst-workspace-close.1 +++ b/man/bst-workspace-close.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE CLOSE" "1" "2024-08-08" "" "bst workspace close Manual" +.TH "BST WORKSPACE CLOSE" "1" "2025-04-24" "2.5" "bst workspace close Manual" .SH NAME bst\-workspace\-close \- Close workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1 index 9ecd01734..113c047d3 100644 --- a/man/bst-workspace-list.1 +++ b/man/bst-workspace-list.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE LIST" "1" "2024-08-08" "" "bst workspace list Manual" +.TH "BST WORKSPACE LIST" "1" "2025-04-24" "2.5" "bst workspace list Manual" .SH NAME bst\-workspace\-list \- List open workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1 index b607a6563..e894a6a5e 100644 --- a/man/bst-workspace-open.1 +++ b/man/bst-workspace-open.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE OPEN" "1" "2024-08-08" "" "bst workspace open Manual" +.TH "BST WORKSPACE OPEN" "1" "2025-04-24" "2.5" "bst workspace open Manual" .SH NAME bst\-workspace\-open \- Open a new workspace .SH SYNOPSIS diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1 index 07b008213..8e7fc9987 100644 --- a/man/bst-workspace-reset.1 +++ b/man/bst-workspace-reset.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE RESET" "1" "2024-08-08" "" "bst workspace reset Manual" +.TH "BST WORKSPACE RESET" "1" "2025-04-24" "2.5" "bst workspace reset Manual" .SH NAME bst\-workspace\-reset \- Reset a workspace to its original state .SH SYNOPSIS diff --git a/man/bst-workspace.1 b/man/bst-workspace.1 index fe0eccbd5..15f9cd570 100644 --- a/man/bst-workspace.1 +++ b/man/bst-workspace.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE" "1" "2024-08-08" "" "bst workspace Manual" +.TH "BST WORKSPACE" "1" "2025-04-24" "2.5" "bst workspace Manual" .SH NAME bst\-workspace \- Manipulate developer workspaces .SH SYNOPSIS diff --git a/man/bst.1 b/man/bst.1 index f05acc54d..0a113b6d3 100644 --- a/man/bst.1 +++ b/man/bst.1 @@ -1,4 +1,4 @@ -.TH "BST" "1" "2024-08-08" "" "bst Manual" +.TH "BST" "1" "2025-04-24" "2.5" "bst Manual" .SH NAME bst \- Build and manipulate BuildStream projects .SH SYNOPSIS @@ -7,9 +7,9 @@ bst \- Build and manipulate BuildStream projects .SH DESCRIPTION Build and manipulate BuildStream projects .PP - Most of the main options override options in the - user preferences configuration file. - +Most of the main options override options in the +user preferences configuration file. +.PP .SH OPTIONS .TP \fB\-\-version\fP diff --git a/src/buildstream/__init__.py b/src/buildstream/__init__.py index b96a2b405..01d1d6535 100644 --- a/src/buildstream/__init__.py +++ b/src/buildstream/__init__.py @@ -31,7 +31,15 @@ from .types import CoreWarnings, OverlapAction, FastEnum, SourceRef from .node import MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode from .plugin import Plugin - from .source import Source, SourceError, SourceFetcher + from .source import ( + Source, + SourceError, + SourceImplError, + SourceFetcher, + SourceInfo, + SourceInfoMedium, + SourceVersionType, + ) from .sourcemirror import SourceMirror, SourceMirrorError from .downloadablefilesource import DownloadableFileSource from .element import Element, ElementError, DependencyConfiguration diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index a79efcdb3..0b162fd18 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -613,6 +613,7 @@ def show(app, elements, deps, except_, order, format_): %{deps} A list of all dependencies %{build-deps} A list of build dependencies %{runtime-deps} A list of runtime dependencies + %{source-info} Source provenance information The value of the %{symbol} without the leading '%' character is understood as a pythonic formatting string, so python formatting features apply, diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 28174032b..8303f0148 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -23,6 +23,7 @@ from .profile import Profile from ..types import _Scope +from ..source import SourceImplError from .. import _yaml from .. import __version__ as bst_version from .. import FileType @@ -437,6 +438,29 @@ def show_pipeline(self, dependencies, format_): runtime_deps = [e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False)] line = p.fmt_subst(line, "runtime-deps", _yaml.roundtrip_dump_string(runtime_deps).rstrip("\n")) + # Source Information + if "%{source-info" in format_: + + # Get all the SourceInfo objects + # + all_source_infos = [] + for source in element.sources(): + try: + source_infos = source.collect_source_info() + except SourceImplError as e: + source.warn(str(e)) + continue + + serialized_sources = [] + for s in source_infos: + serialized = s._serialize() + serialized_sources.append(serialized) + + all_source_infos += serialized_sources + + # Dump the SourceInfo provenance objects in yaml format + line = p.fmt_subst(line, "source-info", _yaml.roundtrip_dump_string(all_source_infos)) + report += line + "\n" return report.rstrip("\n") diff --git a/src/buildstream/downloadablefilesource.py b/src/buildstream/downloadablefilesource.py index 48c164670..b382214d7 100644 --- a/src/buildstream/downloadablefilesource.py +++ b/src/buildstream/downloadablefilesource.py @@ -25,6 +25,48 @@ implementation. +.. _core_downloadable_source_builtins: + +Built-in functionality +---------------------- +The DownloadableFileSource class provides built in keys which can be set when +intantiating any Source which derives from DownloadableFileSource + +* Guess pattern + + The ``version-guess-pattern`` sets the regular expression which will be used to attempt + to guess the version of a source when parsing the source's URI. + + The DownloadableFileSource provides a default implementation of + :func:`Source.collect_source_info() `, + which will use the ``version-guess-pattern`` to attempt to extract a human readable + version string from the specified URI, in order to fill out the reported + :attr:`~buildstream.source.SourceInfo.version_guess`. + + The URI will be *searched* using this regular expression, and is allowed to + yield a number of *groups*. For example the value ``(\\d+)_(\\d+)_(\\d+)`` would + report 3 *groups* if 3 numerical values separated by underscores were found in + the URI. + + The default value for ``version-guess-pattern`` is ``\\d+\\.\\d+(?:\\.\\d+)?``. + + .. note: + + The version guessing mechanism will not be observed if ``version`` is specified. + + **Since: 2.5**. + +* Version + + The ``version`` explicitly sets the :attr:`~buildstream.source.SourceInfo.version_guess` + attribute of the :class:`SourceInfo ` reported for this + source, overriding any guessing. + + This is useful for remote files which do not express their version in their filenames. + + **Since: 2.5**. + + SourceMirror extra data "http-auth" -------------------------------------------- The DownloadableFileSource, and consequently any :class:`Source ` @@ -81,17 +123,38 @@ def translate_url( Authorization: Bearer 1234 + +.. _core_downloadable_source_info: + +Default reporting of :class:`.SourceInfo` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Source plugins which derive from the DownloadableFileSource, unless overridden +and specified in the documentation of the specific source plugin, will behave +as described here. + +The DownloadableFileSource reports the URL of the remote file as the *url*. + +Further, the DownloadableFileSourcebzr source reports the +:attr:`SourceInfoMedium.REMOTE_FILE ` *medium* and the +:attr:`SourceVersionType.SHA256 ` *version_type*, +for which it reports the sha256 checksum of the remote file content as the *version*. + +An attempt to guess the version based on the remote filename will be made +for the reporting of the *guess_version*. Control over how the guess is made +or overridden is explained above in the +:ref:`built-in functionality documentation `. """ import os +import re import urllib.request import urllib.error import contextlib import shutil import netrc -from .source import Source, SourceError +from .source import Source, SourceError, SourceInfoMedium, SourceVersionType from . import utils @@ -202,9 +265,10 @@ def _download_file(opener_creator, url, etag, directory, bearer_auth): class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init - COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref"] + COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref", "version-guess-pattern", "version"] __default_mirror_file = None + __default_guess_pattern = re.compile(r"\d+\.\d+(?:\.\d+)?") def configure(self, node): self.original_url = node.get_str("url") @@ -216,11 +280,28 @@ def configure(self, node): self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) + self._guess_pattern_string = node.get_str("version-guess-pattern", None) + if self._guess_pattern_string is None: + self._guess_pattern = self.__default_guess_pattern + else: + self._guess_pattern = re.compile(self._guess_pattern_string) + + self._version = node.get_str("version", None) + def preflight(self): return def get_unique_key(self): - return [self.original_url, self.ref] + unique_key = [self.original_url, self.ref] + + # Backwards compatible method of supporting configuration + # attributes which affect SourceInfo generation. + if self._version is not None: + unique_key.append(self._version) + elif self._guess_pattern is not self.__default_guess_pattern: + unique_key.append(self._guess_pattern_string) + + return unique_key def is_cached(self) -> bool: return os.path.isfile(self._get_mirror_file()) @@ -270,6 +351,24 @@ def fetch(self): # pylint: disable=arguments-differ "File downloaded from {} has sha256sum '{}', not '{}'!".format(self.url, sha256, self.ref) ) + def collect_source_info(self): + if self._version is None: + version_match = self._guess_pattern.search(self.original_url) + if not version_match: + version_guess = None + elif self._guess_pattern.groups == 0: + version_guess = version_match.group(0) + else: + version_guess = ".".join(version_match.groups()) + else: + version_guess = self._version + + return [ + self.create_source_info( + self.url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.ref, version_guess=version_guess + ) + ] + def _get_etag(self, ref): etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) if os.path.exists(etagfilename): diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py index 2cb7bdc49..207210883 100644 --- a/src/buildstream/plugins/sources/local.py +++ b/src/buildstream/plugins/sources/local.py @@ -31,10 +31,23 @@ See :ref:`built-in functionality doumentation ` for details on common configuration options for sources. + + +Reporting :class:`.SourceInfo` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The local source reports the project relative path of the file or directory as the *url*. + +Further, the local source reports the +:attr:`SourceInfoMedium.LOCAL ` *medium* and the +:attr:`SourceVersionType.CAS_DIGEST ` *version_type*, +for which it reports the CAS digest of the local source as the *version*. + +The *guess_version* of a local source is meaningless, as it is tied instead to +the BuildStream project in which it is contained. """ import os -from buildstream import Source, SourceError, Directory +from buildstream import Source, SourceError, SourceInfoMedium, SourceVersionType, Directory class LocalSource(Source): @@ -67,11 +80,7 @@ def get_unique_key(self): # * Do the regular staging activity into the Directory # * Use the hash of the cached digest as the unique key # - if not self.__digest: - with self._cache_directory() as directory: - self.__do_stage(directory) - self.__digest = directory._get_digest() - + self.__ensure_digest() return self.__digest.hash # We dont have a ref, we're a local file... @@ -110,12 +119,25 @@ def init_workspace_directory(self, directory): # self.__do_stage(directory) + def collect_source_info(self): + self.__ensure_digest() + version = "{}/{}".format(self.__digest.hash, self.__digest.size_bytes) + return [self.create_source_info(self.path, SourceInfoMedium.LOCAL, SourceVersionType.CAS_DIGEST, version)] + # As a core element, we speed up some scenarios when this is used for # a junction, by providing the local path to this content directly. # def _get_local_path(self): return self.fullpath + # Ensure that the digest is resolved + # + def __ensure_digest(self): + if not self.__digest: + with self._cache_directory() as directory: + self.__do_stage(directory) + self.__digest = directory._get_digest() + # Staging is implemented internally, we preemptively put it in the CAS # as a side effect of resolving the cache key, at stage time we just # do an internal CAS stage. diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py index e2bb0e574..ab97c73c7 100644 --- a/src/buildstream/plugins/sources/remote.py +++ b/src/buildstream/plugins/sources/remote.py @@ -41,8 +41,16 @@ # Specify the ref. It's a sha256sum of the file you download. ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b -See :ref:`built-in functionality doumentation ` for -details on common configuration options for sources. +See :ref:`built-in base class functionality doumentation ` +and :ref:`built-in downloadable file source functionality doumentation ` +for details on common configuration options applicable to this source. + + +Reporting :class:`.SourceInfo` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The remote source does not override any of the DownloadableFileSource reporting functionality and +as such, behaves as described in the :ref:`default reporting of SourceInfo ` +documentation. """ import os from buildstream import DownloadableFileSource, SourceError, utils diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index f346a9f9e..f1d939119 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -48,8 +48,16 @@ # to an empty string. base-dir: '*' -See :ref:`built-in functionality doumentation ` for -details on common configuration options for sources. +See :ref:`built-in base class functionality doumentation ` +and :ref:`built-in downloadable file source functionality doumentation ` +for details on common configuration options applicable to this source. + + +Reporting :class:`.SourceInfo` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The tar source does not override any of the DownloadableFileSource reporting functionality and +as such, behaves as described in the :ref:`default reporting of SourceInfo ` +documentation. """ import functools diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py index 237c02026..5c16ad47e 100644 --- a/src/buildstream/plugins/sources/workspace.py +++ b/src/buildstream/plugins/sources/workspace.py @@ -21,20 +21,32 @@ The workspace plugin must not be directly used. This plugin is used as the kind for a synthetic node representing the sources of an element with an open -workspace. The node constructed would be specified as follows: +workspace. -.. code:: yaml - # Specify the workspace source kind - kind: workspace +Reporting :class:`.SourceInfo` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The workspace source reports the project relative or absolute path to the open +workspace as the *url*. - # Specify the absolute path to the directory - path: /path/to/workspace +Further, the workspace source reports the +:attr:`SourceInfoMedium.WORKSPACE ` *medium* and the +:attr:`SourceVersionType.CAS_DIGEST ` *version_type*, +for which it reports the CAS digest of the workspace source as the *version*. + +The *guess_version* of a workspace source is meaningless and omitted. + +.. attention:: + + Observing a SourceInfo with the ``SourceInfoMedium.WORKSPACE`` in the output of + :ref:`bst show --format %{source-info} ` is most likely undesirable, given + that you are likely interested in observing the source provenance information of the + project in a clean state rather than in a state with open workspaces. """ import os -from buildstream import Source, SourceError, Directory, MappingNode +from buildstream import Source, SourceError, SourceInfoMedium, SourceVersionType, Directory, MappingNode from buildstream.types import SourceRef @@ -71,11 +83,7 @@ def get_unique_key(self): # * Do the regular staging activity into the Directory # * Use the hash of the cached digest as the unique key # - if not self.__digest: - with self._cache_directory() as directory: - self.__do_stage(directory) - self.__digest = directory._get_digest() - + self.__ensure_digest() return self.__digest.hash def get_ref(self) -> None: @@ -108,12 +116,25 @@ def stage_directory(self, directory): with self._cache_directory(digest=self.__digest) as cached_directory: directory._import_files_internal(cached_directory, collect_result=False) + def collect_source_info(self): + self.__ensure_digest() + version = "{}/{}".format(self.__digest.hash, self.__digest.size_bytes) + return [self.create_source_info(self.path, SourceInfoMedium.WORKSPACE, SourceVersionType.CAS_DIGEST, version)] + # As a core element, we speed up some scenarios when this is used for # a junction, by providing the local path to this content directly. # def _get_local_path(self) -> str: return self.path + # Ensure that the digest is resolved + # + def __ensure_digest(self): + if not self.__digest: + with self._cache_directory() as directory: + self.__do_stage(directory) + self.__digest = directory._get_digest() + # Staging is implemented internally, we preemptively put it in the CAS # as a side effect of resolving the cache key, at stage time we just # do an internal CAS stage. diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 81104db28..594e1e8cf 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -21,9 +21,8 @@ Built-in functionality ---------------------- - -The Source base class provides built in functionality that may be overridden -by individual plugins. +The Source base class provides built in keys which can be set when instantiating +any Source. * Directory @@ -101,6 +100,15 @@ **Optional**: This is completely optional and will do nothing if left unimplemented. +* :func:`Source.collect_source_info() ` + + Collect SourceInfo objects to describe the provenance of sources. + + **Optional**: BuildStream will function correctly if this is unimplemented, but the + ability to generate SBoMs will be impaired, it is highly recommented to implement this. + + See: :ref:`documentation on generating SourceInfo `. + .. _core_source_ref: @@ -163,6 +171,10 @@ as well as any configuration which uniquely identifies the source, which of course includes the :attr:`~buildstream.types.SourceRef`. +When plugins :ref:`generate SourceInfo `, it is also +important that any configuration attributes which contribute to the +generation of SourceInfo also be included in the unique key. + Accessing previous sources -------------------------- @@ -195,6 +207,122 @@ * Implementations must not introduce host contamination. +.. _core_source_info: + +Generating SourceInfo for provenance information +------------------------------------------------ +Source plugins should implement either of the +:func:`Source.collect_source_info() ` or +:func:`SourceFetcher.get_source_info() ` +methods in order to properly report provenance information and contribute to reports +such as SBoMs. + +To implement these methods, you must use +:func:`Source.create_source_info() ` to +instantiate the :class:`.SourceInfo` object to return from these methods. + +.. attention:: + + It is **not** recommented to consider the parameters used for implementing + tracking with :func:`Source.track() `. + + Instead, any versioning information reported should be congruent with the URL + and the *current* :ref:`source reference `. + + Furthermore, if any of the configuration attributes implemented by the plugin + contribute to the generation of the SourceInfo objects, these configuration + values must be considered in the plugin's + :func:`Plugin.get_unique_key() ` + implementation. + +What follows here, are some guidelines and conventions for doing this properly. + + +The URL +~~~~~~~ +The URL argument represents the location from which the source is obtained, and +should normally be the translated URL, as returned by +:func:`Source.translate_url() `. + +In the case of ``SourceInfoMedium.LOCAL``, the URL can instead be a project +relative path to the local data. + + +The medium and version_type arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +These refer to the medium by which the source data was obtained, and the +meaning/type of the following "version" argument, respectively. + +When possible, you should use the :class:`.SourceInfoMedium` and +:class:`.SourceVersionType` values which correspond to the the medium +and version type which your Source plugin is using. + +In cases where there is not a suitable value available for your plugin, +you can alternatively provide a freeform string which provides these. + + +Documentation +''''''''''''' +Your plugin's module level docstring which is used for documenting your +plugin, should have a section describing the meaning of these values. + +This is especially useful to promote interoperability with other tooling, +which might want to perform some automations based on the :class:`.SourceInfo` object(s) +which your plugin reports. + + +Version +~~~~~~~ +This is a string which uniquely identifies the version of the source, and its meaning +is described by the "version_type" you specified. + + +Version guess +~~~~~~~~~~~~~ +This is a human readable simplified version, more suitable for a cursory reading +of a report like an SBoM. + +Since it is, in most cases not possible to accurately automate the version string +intended by upstream maintainers based on the knowledge you have, we refer to this +as a *guessed version*. For example, just because you have a tarball named ``pony-1.2.3.tgz`` +somewhere, does not guarantee that this is really version ``1.2.3`` of the "pony" project. + + +Configurability +''''''''''''''' +When implementing a technique for guessing the version based on the information +you have at hand, it is recommended to provide some flexability to users of your +plugin, who may have better knowledge about the conventions used by the upstream +project and how they choose to express their versioning information. + +An example of this is the ``version-guess-pattern`` configuration made available +in the :ref:`DownloadableFileSource built-in functionality `. + + +Explicit versioning +''''''''''''''''''' +In some use cases, it is impossible to derive a guessed version from the information +available to the plugin. + +For instance, consider an upstream which indexes their releases on a web page and +then hosts their releases without namespacing their release archives. In such +a case you might have a URL that looks something like: +``https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz`` + +For this reason, the implementing plugin should provide a way for users to manually +annotate the source version. + +An example of this is the ``version`` configuration made available in the +:ref:`DownloadableFileSource built-in functionality `. + + +Extra data +~~~~~~~~~~ +In the case that the existing fields are insufficient to accurately describe the +provenance of this source, extra key/values can be specified when calling +:func:`Source.create_source_info() `. + + .. _core_source_fetcher: SourceFetcher - Object for fetching individual URLs @@ -211,6 +339,13 @@ Fetches the URL associated with this SourceFetcher, optionally taking an alias override. +* :func:`SourceFetcher.get_source_info() ` + + Get a SourceInfo object to describe the provenance of this source. + + **Optional**: BuildStream will function correctly if this is unimplemented, but the + ability to generate SBoMs will be impaired, it is highly recommented to implement this. + Class Reference --------------- """ @@ -224,7 +359,7 @@ from .node import MappingNode from .plugin import Plugin from .sourcemirror import SourceMirror -from .types import SourceRef, CoreWarnings +from .types import SourceRef, CoreWarnings, FastEnum from ._exceptions import BstError, ImplError, PluginError from .exceptions import ErrorDomain from ._loader.metasource import MetaSource @@ -260,12 +395,243 @@ def __init__( super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary) +class SourceImplError(BstError): + """This exception is expected to be raised from some unimplemented abstract methods. + + There is no need to raise this exception, however some public abstract methods which + are intended to be called by plugins may advertize the raising of this exception + in the case of a source plugin which does not implement the said method, in which case + it must be handled by the calling plugin. + """ + + def __init__(self, message, reason=None): + super().__init__(message, domain=ErrorDomain.IMPL, reason=reason) + + @dataclass class AliasSubstitution: + """AliasSubstitution() + An opaque data structure which may be passed through + :func:`SourceFetcher.fetch() ` and in such cases + must be provided to :func:`Source.translate_url() `. + """ + _effective_alias: str _mirror: Union[SourceMirror, str] +class SourceInfoMedium(FastEnum): + """ + Indicates the medium in which the source is obtained + + *Since: 2.5* + """ + + WORKSPACE = "workspace" + """ + Files in an open workspace + """ + + LOCAL = "local" + """ + Files stored locally in the project + """ + + REMOTE_FILE = "remote-file" + """ + A remote file + """ + + GIT = "git" + """ + A git repository + """ + + BAZAAR = "bzr" + """ + The Bazaar revision control system + """ + + OCI_IMAGE = "oci-image" + """ + An OCI image, such as docker or podman images. + """ + + PYTHON_PACKAGE_INDEX = "pypi" + """ + A python package obtained from a python package index like https://pypi.org + """ + + +class SourceVersionType(FastEnum): + """ + Indicates the type of the version string + + *Since: 2.5* + """ + + COMMIT = "commit" + """ + A commit string which accurately represents a version in a source + code repository or VCS + """ + + SHA256 = "sha256" + """ + An sha256 checksum of the content of a file + """ + + CAS_DIGEST = "cas-digest" + """ + A CAS digest expressed as ``{hash}/{size}``. + + The ``hash`` and ``size`` components represent the members of a ``Digest`` message as + defined in the `remote execution protocol + `_ + """ + + OCI_DIGEST = "oci-digest" + """ + An OCI image digest, as can be used to address images in a docker registry. + """ + + INDEXED_VERSION = "indexed-version" + """ + This type of version is used in cases where we have repositories which + have an interface to index content by version, and that no additional validation + is performed to insure the uniqueness of the downloaded content (not recommended). + + In the case of plugins which use this version type, it is probable that + ``SourceInfo.version_guess == SourceInfo.version``. + """ + + +class SourceInfo: + """SourceInfo() + + An object representing the provenance of input reported by + :func:`Source.collect_source_info() ` + and/or :func:`SourceFetcher.get_source_info() ` + + See: :ref:`documentation on generating SourceInfo `. + + .. attention:: + + A given SourceInfo for a given element is **not** guaranteed to be unique for + a given :ref:`cache key `. + + While it is true that plugins which :ref:`generate SourceInfo ` + must consider any configuration attributes in their cache keys, so as to produce + differing cache keys when source provenance information can be reported differently, + this does not account for the special nature of *urls*. + + When considering the *urls* reported in SourceInfo, the urls are only guaranteed to be the + primary urls as defined by the project's :ref:`source aliases `, + and arbitrary :ref:`mirror urls ` will not be reported here. + + Since these aliases are intentionally allowed to change without affecting cache + keys, or can be :ref:`redirected with junctions `, + it possible to have a *differing* set of SourceInfo objects reported for a project which + reports identical *cache keys*, in cases where primary alias mappings are changed. + + *Since: 2.5* + """ + + def __init__( + self, + kind: str, + url: str, + medium: Union[SourceInfoMedium, str], + version_type: Union[SourceVersionType, str], + version: str, + *, + version_guess: Optional[str] = None, + extra_data: Optional[Dict[str, str]] = None, + ): + self.kind: str = kind + """ + The Source plugin kind which reported this SourceInfo + """ + + self.url: str = url + """ + The url of the source input + """ + + self.medium: Union[SourceInfoMedium, str] = medium + """ + The :class:`.SourceInfoMedium` of the source input, or in the case + that an appropriate medium is not defined, a freeform string of the plugin's + choice describing the medium. + """ + + self.version_type: Union[SourceVersionType, str] = version_type + """ + The :class:`.SourceVersionType` of the source input version, or in the case + that an appropriate version type is not defined, a freeform string of the plugin's + choice depicting the type of version. + """ + + self.version: str = version + """ + A string which represents a unique version of this source input + """ + + self.version_guess: Optional[str] = version_guess + """ + A string representing the guessed human readable version of this source input + """ + + self.extra_data: Optional[Dict[str, str]] = extra_data + """ + Additional plugin defined key/values + """ + + # _serialize() + # + # Produce a dictionary object suitable to be dumped in YAML format + # in the `bst show` command line interface. + # + # Returns: A dictionary used to dump this out on the CLI with _yaml.roundtrip_dump_string() + # + def _serialize(self) -> Dict[str, Any]: + # + # WARNING: This return value produces output for an API stable interface. + # + # Dictionary member names cannot be removed, and the meaning of + # their values cannot be changed. + # + version_info: Dict[str, Union[str, Dict[str, str]]] + medium_str: str + version_type_str: str + + if isinstance(self.medium, SourceInfoMedium): + medium_str = str(self.medium.value) + else: + medium_str = self.medium + + if isinstance(self.version_type, SourceVersionType): + version_type_str = str(self.version_type.value) + else: + version_type_str = self.version_type + + version_info = { + "kind": self.kind, + "url": self.url, + "medium": medium_str, + "version-type": version_type_str, + "version": self.version, + } + + if self.version_guess is not None: + version_info["version-guess"] = self.version_guess + + if self.extra_data: + version_info["extra-data"] = self.extra_data + + return version_info + + class SourceFetcher: """SourceFetcher() @@ -293,8 +659,9 @@ def fetch(self, alias_override: Optional[AliasSubstitution] = None, **kwargs) -> Args: alias_override: The alias to use instead of the default one - defined by the :ref:`aliases ` field - in the project's config. + defined by the :ref:`aliases ` field in the + project's config. If provided, it must be used when calling + :func:`Source.translate_url() `. Raises: :class:`.SourceError` @@ -304,6 +671,25 @@ def fetch(self, alias_override: Optional[AliasSubstitution] = None, **kwargs) -> """ raise ImplError("SourceFetcher '{}' does not implement fetch()".format(type(self))) + def get_source_info(self) -> SourceInfo: + """Get the :class:`.SourceInfo` object describing this source + + This method should only be called whenever + :func:`Source.is_resolved() ` + returns ``True``. + + SourceInfo objects created by implementors should be created with + :func:`Source.create_source_info() `. + + Returns: the :class:`.SourceInfo` objects describing this source + + Raises: + :class:`.SourceImplError`: if this method is unimplemented + + *Since: 2.5* + """ + raise SourceImplError("SourceFetcher '{}' does not implement get_source_info()".format(type(self))) + ############################################################# # Public Methods # ############################################################# @@ -684,6 +1070,43 @@ def is_cached(self) -> bool: """ raise ImplError("Source plugin '{}' does not implement is_cached()".format(self.get_kind())) + def collect_source_info(self) -> Iterable[SourceInfo]: + """Get the :class:`.SourceInfo` objects describing this source + + This method should only be called whenever + :func:`Source.is_resolved() ` + returns ``True``. + + SourceInfo objects created by implementors should be created with + :func:`Source.create_source_info() `. + + Returns: the :class:`.SourceInfo` objects describing this source + + Raises: + :class:`.SourceImplError`: if the source class does not implement this method and does not implement + :func:`SourceFether.get_source_info() ` + + .. note:: + + If your plugin uses :class:`.SourceFetcher` objects, you can implement + :func:`Source.collect_source_info() ` instead. + + *Since: 2.5* + """ + source_info = [] + for fetcher in self.get_source_fetchers(): + source_info.append(fetcher.get_source_info()) + + # If there are source fetchers, they can either have returned + # SourceInfo objects, OR they may have raised SourceImplError, we need + # to raise ImplError here in the case there were no source fetchers. + if not source_info: + raise SourceImplError( + "Source plugin '{}' does not implement collect_source_info()".format(self.get_kind()) + ) + + return source_info + ############################################################# # Public Methods # ############################################################# @@ -903,6 +1326,41 @@ def is_resolved(self) -> bool: """ return self.get_ref() is not None + def create_source_info( + self, + url: str, + medium: Union[SourceInfoMedium, str], + version_type: Union[SourceVersionType, str], + version: str, + *, + version_guess: Optional[str] = None, + extra_data: Optional[Dict[str, str]] = None, + ) -> SourceInfo: + """Create a :class:`.SourceInfo` object + + This function should be used to generate SourceInfo objects in + :func:`Source.is_resolved() ` + and :func:`Source.is_resolved() ` + implementations. + + Args: + url: The translated URL + medium: The :class:`.SourceInfoMedium` of the source input, or in the case + that an appropriate medium is not defined, a freeform string of the plugin's + choice describing the medium. + version_type: The :class:`.SourceVersionType` of the source input version, or in the case + that an appropriate version type is not defined, a freeform string of the plugin's + choice depicting the type of version. + version: A string which represents a unique version of this source input + version_guess: An optional string representing the guessed human readable version + extra_data: Additional plugin defined key/values + + *Since: 2.5* + """ + return SourceInfo( + self.get_kind(), url, medium, version_type, version, version_guess=version_guess, extra_data=extra_data + ) + ############################################################# # Private Abstract Methods used in BuildStream # ############################################################# diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 9b486f62d..3dfc8f55c 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -20,6 +20,7 @@ import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import +from buildstream._testing import create_repo from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream.types import CoreWarnings @@ -575,3 +576,194 @@ def test_invalid_alias(cli, tmpdir, datafiles): configure_project(project, {"aliases": {"pony": "https://pony.org/tarballs", "horsy": "http://horsy.tv/shows"}}) result = cli.run(project=project, silent=True, args=["show", "invalid-alias.bst"]) result.assert_main_error(ErrorDomain.SOURCE, "invalid-source-alias") + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +@pytest.mark.parametrize( + "target, expected_kind, expected_url, expected_medium, expected_version_type, expected_version, expected_guess_version", + [ + ( + "local.bst", + "local", + "files/testfile", + "local", + "cas-digest", + "9391a5943daf287b46520c4289d41cab5f6b33e643f7661bcf620de7f02c1c9b/82", + None, + ), + ( + "tar.bst", + "tar", + "https://flying-ponies.com/releases/pony-flight-1.2.3.tgz", + "remote-file", + "sha256", + "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", + "1.2.3", + ), + ( + "tar-no-micro.bst", + "tar", + "https://flying-ponies.com/releases/pony-flight-1.2.tgz", + "remote-file", + "sha256", + "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", + "1.2", + ), + ( + "tar-custom-version.bst", + "tar", + "https://flying-ponies.com/releases/pony_v2_4_93.tgz", + "remote-file", + "sha256", + "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", + "2.4.93", + ), + ( + "tar-explicit.bst", + "tar", + "https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz", + "remote-file", + "sha256", + "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", + "3.2.1", + ), + ( + "testsource.bst", + "testsource", + "http://ponyfarm.com/ponies", + "pony-ride", + "pony-age", + "1234567", + "12", + ), + ], + ids=["local", "tar-full-version", "tar-no-micro", "tar-custom-version", "tar-explicit", "testsource"], +) +def test_source_info( + cli, + datafiles, + target, + expected_url, + expected_kind, + expected_medium, + expected_version_type, + expected_version, + expected_guess_version, +): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", target]) + result.assert_success() + + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence(target) + source_info = sources.mapping_at(0) + + assert source_info.get_str("kind") == expected_kind + assert source_info.get_str("url") == expected_url + assert source_info.get_str("medium") == expected_medium + assert source_info.get_str("version-type") == expected_version_type + assert source_info.get_str("version") == expected_version + + guess_version = source_info.get_str("version-guess", None) + if guess_version or expected_guess_version: + assert guess_version == expected_guess_version + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +def test_source_info_extra_data(cli, datafiles): + project = str(datafiles) + result = cli.run( + project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "extradata.bst"] + ) + result.assert_success() + + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("extradata.bst") + source_info = sources.mapping_at(0) + + assert source_info.get_str("kind") == "extradata" + assert source_info.get_str("url") == "http://ponyfarm.com/ponies" + assert source_info.get_str("medium") == "pony-ride" + assert source_info.get_str("version-type") == "pony-age" + assert source_info.get_str("version") == "1234567" + assert source_info.get_str("version-guess", None) == "12" + + extra_data = source_info.get_mapping("extra-data", None) + assert extra_data is not None + assert extra_data.get_str("pony", "green") + + +# Test what happens when encountering a source that doesn't implement collect_source_info() +# +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +def test_source_info_unimplemented(cli, datafiles): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["show", "--format", "%{source-info}", "unimplemented.bst"]) + result.assert_success() + + # Assert empty list but no errors for a source not implementing collect_source_info() + # + # Note that buildstream internal _yaml doesn't support loading a list as a toplevel element + # in the stream, so we just assert the string instead. + assert result.output == "[]\n\n" + + +# This checks how Source.collect_source_info() works on a workspace, +# in order to do this we use a workspace *on a tarball* which we generate, +# so that we avoid the weird case of opening a workspace on a local source. +# +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +def test_source_info_workspace(cli, datafiles, tmpdir): + project = str(datafiles) + element_dir = os.path.join(project, "elements") + workspace_path = os.path.join(project, "workspace") + + # Create a tar repo + repo = create_repo("tar", str(tmpdir)) + repo.create(os.path.join(str(datafiles), "files")) + + # Generate an import element for this tarball + input_config = { + "kind": "import", + "sources": [repo.source_config()], + } + input_file = os.path.join(element_dir, "test-workspace.bst") + _yaml.roundtrip_dump(input_config, input_file) + + # Track the element so that there is a ref (required for workspace checkout) + result = cli.run( + project=project, + silent=True, + args=["source", "track", "test-workspace.bst"], + ) + result.assert_success() + + # Open a workspace on the tarball + result = cli.run( + project=project, + silent=True, + args=["workspace", "open", "--directory", workspace_path, "test-workspace.bst"], + ) + result.assert_success() + + # Lets see what happens + result = cli.run( + project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "test-workspace.bst"] + ) + result.assert_success() + + # Lets check the results + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("test-workspace.bst") + source_info = sources.mapping_at(0) + + # The URL is a local path for a workspace, so we just check it exists + assert source_info.get_str("url") is not None + assert source_info.get_str("medium") == "workspace" + assert source_info.get_str("version-type") == "cas-digest" + + # Tarball repo generation is not deterministic, so we just assert that there is a digest + assert source_info.get_str("version", None) is not None + + # There is no version guessing for a workspace + assert source_info.get_str("version-guess", None) is None diff --git a/tests/frontend/source-info/elements/extradata.bst b/tests/frontend/source-info/elements/extradata.bst new file mode 100644 index 000000000..101fe2642 --- /dev/null +++ b/tests/frontend/source-info/elements/extradata.bst @@ -0,0 +1,4 @@ +kind: import + +sources: +- kind: extradata diff --git a/tests/frontend/source-info/elements/local.bst b/tests/frontend/source-info/elements/local.bst new file mode 100644 index 000000000..3a1353b9f --- /dev/null +++ b/tests/frontend/source-info/elements/local.bst @@ -0,0 +1,5 @@ +kind: import + +sources: +- kind: local + path: files/testfile diff --git a/tests/frontend/source-info/elements/tar-custom-version.bst b/tests/frontend/source-info/elements/tar-custom-version.bst new file mode 100644 index 000000000..8aebe0b95 --- /dev/null +++ b/tests/frontend/source-info/elements/tar-custom-version.bst @@ -0,0 +1,8 @@ +kind: import + +sources: +- kind: tar + version-guess-pattern: >- + (\d+)_(\d+)_(\d+) + url: https://flying-ponies.com/releases/pony_v2_4_93.tgz + ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 diff --git a/tests/frontend/source-info/elements/tar-explicit.bst b/tests/frontend/source-info/elements/tar-explicit.bst new file mode 100644 index 000000000..40f6e0940 --- /dev/null +++ b/tests/frontend/source-info/elements/tar-explicit.bst @@ -0,0 +1,7 @@ +kind: import + +sources: +- kind: tar + url: https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz + ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 + version: 3.2.1 diff --git a/tests/frontend/source-info/elements/tar-no-micro.bst b/tests/frontend/source-info/elements/tar-no-micro.bst new file mode 100644 index 000000000..40eaace68 --- /dev/null +++ b/tests/frontend/source-info/elements/tar-no-micro.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: tar + url: https://flying-ponies.com/releases/pony-flight-1.2.tgz + ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 diff --git a/tests/frontend/source-info/elements/tar.bst b/tests/frontend/source-info/elements/tar.bst new file mode 100644 index 000000000..0dd9bc537 --- /dev/null +++ b/tests/frontend/source-info/elements/tar.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: tar + url: https://flying-ponies.com/releases/pony-flight-1.2.3.tgz + ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 diff --git a/tests/frontend/source-info/elements/testsource.bst b/tests/frontend/source-info/elements/testsource.bst new file mode 100644 index 000000000..a3d55297b --- /dev/null +++ b/tests/frontend/source-info/elements/testsource.bst @@ -0,0 +1,4 @@ +kind: import + +sources: +- kind: testsource diff --git a/tests/frontend/source-info/elements/unimplemented.bst b/tests/frontend/source-info/elements/unimplemented.bst new file mode 100644 index 000000000..d8527e9f5 --- /dev/null +++ b/tests/frontend/source-info/elements/unimplemented.bst @@ -0,0 +1,4 @@ +kind: import + +sources: +- kind: unimplemented diff --git a/tests/frontend/source-info/files/testfile b/tests/frontend/source-info/files/testfile new file mode 100644 index 000000000..c3740277e --- /dev/null +++ b/tests/frontend/source-info/files/testfile @@ -0,0 +1 @@ +The perverted pink pony mounts the unwilling hippopotamus \ No newline at end of file diff --git a/tests/frontend/source-info/plugins/extradata.py b/tests/frontend/source-info/plugins/extradata.py new file mode 100644 index 000000000..90abe3450 --- /dev/null +++ b/tests/frontend/source-info/plugins/extradata.py @@ -0,0 +1,43 @@ +from buildstream import Source + + +class ExtraData(Source): + BST_MIN_VERSION = "2.0" + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + return {} + + def load_ref(self, node): + pass + + def get_ref(self): + return {} + + def set_ref(self, ref, node): + pass + + def is_cached(self): + return False + + def collect_source_info(self): + return [ + self.create_source_info( + "http://ponyfarm.com/ponies", + "pony-ride", + "pony-age", + "1234567", + version_guess="12", + extra_data={"pony": "green"}, + ) + ] + + +# Plugin entry point +def setup(): + return ExtraData diff --git a/tests/frontend/source-info/plugins/testsource.py b/tests/frontend/source-info/plugins/testsource.py new file mode 100644 index 000000000..0763285f5 --- /dev/null +++ b/tests/frontend/source-info/plugins/testsource.py @@ -0,0 +1,38 @@ +from buildstream import Source + + +class Sample(Source): + BST_MIN_VERSION = "2.0" + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + return {} + + def load_ref(self, node): + pass + + def get_ref(self): + return {} + + def set_ref(self, ref, node): + pass + + def is_cached(self): + return False + + def collect_source_info(self): + return [ + self.create_source_info( + "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12" + ) + ] + + +# Plugin entry point +def setup(): + return Sample diff --git a/tests/frontend/source-info/plugins/unimplemented.py b/tests/frontend/source-info/plugins/unimplemented.py new file mode 100644 index 000000000..828d7889b --- /dev/null +++ b/tests/frontend/source-info/plugins/unimplemented.py @@ -0,0 +1,31 @@ +from buildstream import Source + + +class Unimplemented(Source): + BST_MIN_VERSION = "2.0" + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + return {} + + def load_ref(self, node): + pass + + def get_ref(self): + return {} + + def set_ref(self, ref, node): + pass + + def is_cached(self): + return False + + +# Plugin entry point +def setup(): + return Unimplemented diff --git a/tests/frontend/source-info/project.conf b/tests/frontend/source-info/project.conf new file mode 100644 index 000000000..86cfd087b --- /dev/null +++ b/tests/frontend/source-info/project.conf @@ -0,0 +1,16 @@ +# Project config for bst show source-info test +name: test +min-version: 2.0 +element-path: elements + +plugins: +- origin: pip + package-name: sample-plugins + sources: + - git +- origin: local + path: plugins + sources: + - extradata + - testsource + - unimplemented diff --git a/tox.ini b/tox.ini index 380b4408b..2967e9bc8 100644 --- a/tox.ini +++ b/tox.ini @@ -208,16 +208,21 @@ passenv = allowlist_externals = make + # # (re-)Generating man pages # [testenv:man] commands = - python3 setup.py --command-packages=click_man.commands man_pages + # We set the version to 0, because click-man would now + # otherwise want to pull a hardcoded version from pyproject.toml + # that somehow gets magically transported to the EntryPoint.version + click-man --man-version {posargs} --target man bst deps = - click-man >= 0.3.0 - Cython + click-man >= 0.5.0 -rrequirements/requirements.txt + -rrequirements/dev-requirements.txt + Cython #