From 92f8139e2fef99d0e4a13e87c4557563febd1527 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:19:27 +0200 Subject: [PATCH 1/9] WPB-17086: testing the issue for overriding the versions.json when building latest and releasing a version in dev env --- .github/workflows/release.yaml | 17 ++++++++++++++++- .github/workflows/test.yaml | 16 ++++++++++++++-- build/build_versions.sh | 3 +-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4c2a63c..d0ec2e4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -39,7 +39,22 @@ jobs: run: | mkdir -p tmp_extracted tar -xzf ./wire-docs-${{ github.ref_name }}.tar.gz -C tmp_extracted - aws s3 sync tmp_extracted/ s3://origin-docs.wire.com/ + + # fetching upstream versions to retain information on already existing versions + aws s3 cp s3://${{ secrets.DEV_BUCKET }}/versions.json tmp_extracted/upstream_versions.json + + # merge the both versions.json files and keep the unique entries while giving priority to current versions.json + jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/upstream_versions.json tmp_extracted/versions.json > tmp_extracted/all_versions.json + + # removing old objects from the bucket for the ref_name to ensure, we don't keep objects at old path + aws s3 rm s3://${{ secrets.DEV_BUCKET }}/${{ github.ref_name }} --recursive + + # pushing all the ref_name build documents to S3 + aws s3 sync tmp_extracted/${{ github.ref_name }} s3://${{ secrets.DEV_BUCKET }}/ + + # syncying the all_versions.json with versions.json + aws s3 cp tmp_extracted/all_versions.json s3://${{ secrets.DEV_BUCKET }}/versions.json + rm -rf tmp_extracted - name: Upload Release Asset diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7c429af..62aa28a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Test Env Latest Build and Deploy to S3 on: - pull_request: + push: branches: - test-dev paths-ignore: @@ -41,9 +41,21 @@ jobs: mkdir -p tmp_extracted tar -xzf ./wire-docs.tar.gz -C tmp_extracted + # fetching upstream versions to retain information on already existing versions + aws s3 cp s3://${{ secrets.DEV_BUCKET }}/versions.json tmp_extracted/upstream_versions.json + + # merge the both versions.json files and keep the unique entries while giving priority to current versions.json + jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/upstream_versions.json tmp_extracted/versions.json > tmp_extracted/all_versions.json + # removing old objects from the bucket to ensure, we don't keep objects at old path aws s3 rm s3://${{ secrets.DEV_BUCKET }}/latest --recursive + + # pushing all the latest build documents aws s3 sync tmp_extracted/latest s3://${{ secrets.DEV_BUCKET }}/latest + + # syncing the index.html file + aws s3 sync tmp_extracted/index.html s3://${{ secrets.DEV_BUCKET }}/index.html - # aws s3 cp tmp_extracted/versions.json s3://${{ secrets.DEV_BUCKET }}/versions.json + # syncying the all_versions.json with versions.json + aws s3 cp tmp_extracted/all_versions.json s3://${{ secrets.DEV_BUCKET }}/versions.json rm -rf tmp_extracted diff --git a/build/build_versions.sh b/build/build_versions.sh index 521c3dc..eeecb65 100644 --- a/build/build_versions.sh +++ b/build/build_versions.sh @@ -1,8 +1,7 @@ #!/bin/bash -set -ex +set -e mike="pipenv run mike" -printenv CURRENT=$(git branch --show-current) # using dummy values for user.name and user.email as they are not required for git operations but a requirement for mike to have gh-pages branch From 4e16f4b898e0532da57247a80c345b5b80c862e7 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Tue, 3 Jun 2025 21:33:07 +0200 Subject: [PATCH 2/9] WPB-17086: add mermaid diagrams to release and readme.md and fixed the documentation --- .github/workflows/build.yaml | 8 +- .github/workflows/release.yaml | 2 +- README.md | 253 ++++++++++++++++++++++++--------- Release.md | 33 +++-- 4 files changed, 218 insertions(+), 78 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 54d2b05..685edd2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,4 +1,4 @@ -name: Prod Env Latest Build +name: Build Test on: pull_request: @@ -21,10 +21,10 @@ jobs: - name: Setup Nix uses: cachix/install-nix-action@v16 - - name: Build Archive - id: build_archive + - name: Build Docs + id: build_docs run: | - make archive + make build if [ ! -f "wire-docs.tar.gz" ]; then echo "Artifact not found!" exit 1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d0ec2e4..9fee841 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -name: Build and Release Artifact +name: Build, Deploy and Release Artifact on: release: diff --git a/README.md b/README.md index e59a20a..6282079 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,215 @@ # Wire-documents-structure -Wire documentation is hosted on . This project is made using Mkdocs. +Wire documentation is hosted on . This project uses MkDocs as its documentation generator. ## Structure of the repository -- src - - It contains the files and directories for actual source of the documentation. The `src` directory has been processed based on [docs](https://github.com/wireapp/wire-server/tree/develop/docs). The earlier version was based on Sphinx, so it was converted to markdown and then ported for Mkdocs. Find the process for doing in `build/old-docs.md`. +### src Directory +Contains the source files and directories for the documentation. The `src` directory has been migrated from the [wire-server docs](https://github.com/wireapp/wire-server/tree/develop/docs). The previous version used Sphinx, so we converted it to Markdown and adapted it for MkDocs. See `build/old-docs.md` for the migration process. -- Pages from external repositories (via Git submodules): +### External Pages (Git Submodules) +We import documentation pages from the backend repository [wire-server](https://github.com/wireapp/wire-server/tree/develop). Many files currently reference wire-server, which you can identify using: - - We intend to import some documentation pages from the backend repository [wire-server](https://github.com/wireapp/wire-server/tree/develop) into our project. Currently, many files reference wire-server; you can identify these files using the following command - ``` - find . -type l -printf '%p -> %l\n' | grep './src' - ``` - - To fetch the latest updates from wire-server, run the following command. Note: Execute this command only when the remote repository has been updated with new changes. +```bash +find . -type l -printf '%p -> %l\n' | grep './src' +``` - ``` - git submodule update --remote --checkout --depth 1 -- wire-server - ``` +#### Submodule Management - - To checkout to a specific commit: - ``` - cd wire-server && git fetch --depth=1 origin && git checkout - ``` - - - To verify the submodule status: - ``` - git submodule status - ``` +**Fetch latest updates from wire-server:** +```bash +git submodule update --remote --checkout --depth 1 -- wire-server +``` +*Note: Only run this command when the remote repository has new changes.* - - To optionally update the src/changelog/README.md based on the new changelog.md, run the following command: - ``` - rm src/changelog/README.md && \ - grep '^# ' src/changelog/changelog.md | \ - sed 's/^# //' | while IFS= read -r heading; \ - do anchor=$(echo "$heading" | sed -E 's/^\[?([0-9-]+)\]? *(\(([^)]+)\)|# *([0-9]+))$/\1-\3\4/' | tr '[:upper:]' '[:lower:]' | sed 's/ /-/g;s/\.//g'); echo "* [$heading](changelog.md#$anchor)"; done > src/changelog/README.md - ``` +**Checkout a specific commit:** +```bash +cd wire-server && git fetch --depth=1 origin && git checkout +``` -- build - - It contains scripts used by Makefile to support different usecases for builds. It is designed to run one build process at a time. All the local targets use a temporary directory stored in `.tmpdir` file for building/serving the changes. This is due to git-operations by tools like `mike` for mkdocs. - ### Prerequisites - - - make - - nix-shell - - git - - rsync - - Docker (optional) - if building a docker image, useful when testing without installing nix-shell +**Verify submodule status:** +```bash +git submodule status +``` - ### Makefile Targets +**Update changelog (optional):** +To optionally update the src/changelog/README.md based on the new changelog.md, run the following command: - - `make current` - - This target runs the documentation site locally using Mike (Mkdocs) as we do in `make run` but only for the `current` branch. The current changes will be visible under branch name as version name. It will be hosted on `0.0.0.0:8000`. It can also show other versions if they have been pre-built on your local host. +``` bash +rm src/changelog/README.md && \ +grep '^# ' src/changelog/changelog.md | \ +sed 's/^# //' | while IFS= read -r heading; \ +do anchor=$(echo "$heading" | sed -E 's/^\[?([0-9-]+)\]? *(\(([^)]+)\)|# *([0-9]+))$/\1-\3\4/' | tr '[:upper:]' '[:lower:]' | sed 's/ /-/g;s/\.//g'); echo "* [$heading](changelog.md#$anchor)"; done > src/changelog/README.md +``` - - `make run` - - This target serves the documentation site locally by first building all the existing tags and building current branch. It later hosts the webserver using python http module. It allows you to preview the documentation as it will appear when hosted `for all the tags and current branch`. It will be hosted on `0.0.0.0:8000`. Use it only if you want to build all existing tags by yourself. +### build Directory +Contains scripts used by the Makefile to support different build use cases. The system is designed to run one build process at a time. All local targets use a temporary directory (stored in `.tmpdir` file) for building and serving changes due to git operations required by tools like `mike` for MkDocs. - - `make archive` - - This target archive the processed web pages for documentation for all the tags and current branch from the github branch gh-pages. The output is generated in the main directory as `wire-docs.tar.gz`. +## Prerequisites - - `make build` - - This target is being used by run and archive targets for building all the tags and current branch. It serves the layer to economise the re-building each time for archive and run targets. Note: if there are uncomitted changes in the `current` branchm then it is going to re-build the current branch everytime. +- make +- nix-shell +- git +- rsync +- Docker (optional) - for building Docker images when testing without nix-shell - - `main docker` - - This target builds a Docker image for the documentation. It uses the Dockerfile present in the repository to create an image. It will be using `mike` module from python to host the documents, this is experimental to use `mike` for the container and not python `http` module. It processes the tags present in the `online` [repo](https://github.com/wireapp/wire-docs.git). To test current changes, rely on `make run` or `make current`. To run and test the docker image locally, we recommend the following command: - ```bash - docker run -d -p 8000:8000 --restart=always --health-cmd="curl --fail http://localhost:8000 || exit 1" --health-interval=30s --health-retries=3 --health-timeout=5s wire-docs - ``` - - `make clean` - - This target cleans up the generated tar files and tmp directories from the build process. +## Makefile Targets - - `make SHELL="/bin/bash -x" target` - - To increase verbosity for make commands. +### `make current` +Runs the documentation site locally using Mike (MkDocs plugin) for the `current` branch only. Current changes appear under the branch name as the version name. Hosted at `0.0.0.0:8000`. -## Versioning criteria: - - The version number will follow a three-part format: **A.B.C** +### `make run` +Serves the documentation site locally by building the current branch as static web pages, then hosts a web server using Python's HTTP module. This allows you to preview how the documentation will appear when hosted for the current branch. Hosted at `0.0.0.0:8000`. - - **A (Major Version)**: This indicates the minimum API contract version supported by the release. When this number increases, it signals the deprecation of an older API contract version. For example, V5.0.0 means the release supports API V5 and newer. Major versions can include breaking changes, such as database migrations or changes to the backend service structure. +### `make archive` +Archives the processed web pages for the current branch from the GitHub `gh-pages` branch (created by `make build`). Output is generated in the main directory as `wire-docs.tar.gz`. - - **B (Feature Iteration)**: This number reflects feature updates that don’t affect API contract compatibility or introduce breaking changes. For example, V5.1.0 means the release supports API V5 and newer, and includes new features or improvements without altering the compatibility matrix. +### `make build` +Used by `run` and `archive` targets to build the current branch. Attempts to identify the version name for the environment. In GitHub environments, it finds an appropriate name for the version to be built instead of using a branch name. - - **C (Fix Iteration)**: This is incremented for bug fixes or security updates that don’t introduce new features or modify compatibility. There are no breaking changes in a fix iteration. +### `make docker` *(Experimental)* +Builds a Docker image for the documentation using the repository's Dockerfile. Uses Python's `mike` module to host documents (experimental - not the standard Python HTTP module). For testing current changes, use `make run` or `make current` instead. - - The *first release*, `v0.0.0`, indicates that the documentation supports all API versions. +To test the Docker image locally: +```bash +docker run -d -p 8000:8000 --restart=always --health-cmd="curl --fail http://localhost:8000 || exit 1" --health-interval=30s --health-retries=3 --health-timeout=5s wire-docs +``` + +### `make clean` +Cleans up generated tar files and temporary directories from the build process. + +### Verbose Mode +For increased verbosity in make commands: +```bash +make SHELL="/bin/bash -x" target +``` + +## GitHub Actions for Build and Deployment + +```mermaid + graph TD + + %% Triggers + PR[Pull Request to main] --> |triggers| BuildWorkflow + PushMain[Push to main Branch] --> |triggers| DeployWorkflow + CreateRelease[Create Release] --> |triggers| ReleaseWorkflow + TestEnvironment[Push to test-dev Branch] --> |triggers| TestWorkflow + + %% Styling + classDef workflow fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + classDef action fill:#e8f5e8,stroke:#1b5e20,stroke-width:2px + + class BuildWorkflow,DeployWorkflow,ReleaseWorkflow,TestWorkflow workflow + class PR,PushMain,CreateRelease,TestEnvironment action + +``` + +The repository includes four automated builds: + +### 1. [Build Test](.github/workflows/build.yaml) +Runs `make build` to verify MkDocs build structure for each pull request. + +```mermaid + graph TD + %% Build Workflow (PR) + BuildWorkflow[Build Workflow
build.yaml] --> |runs| BuildMake[make build] + BuildMake --> |verifies| DocStructure[Documentation Structure] + DocStructure --> |identifies| NavIssues[Navigation/Markdown Issues] + + %% Styling + classDef output fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef workflow fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + + class BuildWorkflow workflow + class DocStructure,NavIssues output + +``` + + +### 2. [Prod Env Latest Build and Deploy to S3](.github/workflows/deploy.yaml) +Runs `make archive` on every push to the main branch as the `latest` version. Cleans old `latest` documents before uploading to ensure the S3 bucket matches the main branch state. Maintains `versions.json` (tracking all document versions) without overwriting, keeping it synchronized with the latest version with commit as an alias. + +```mermaid + graph TD + %% Deploy Workflow (Push to Main) + DeployWorkflow[Deploy Workflow
deploy.yaml] --> |runs| ArchiveMake[make archive] + ArchiveMake --> |creates| LatestVersion[latest Version] + LatestVersion --> |cleans old| OldDocs[Old latest Documents] + LatestVersion --> |pushes to| S3Latest[S3 Bucket - Latest] + LatestVersion --> |maintains| VersionsJSON[versions.json] + + %% Styling + classDef output fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef workflow fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + + class DeployWorkflow workflow + class LatestVersion,S3Latest,VersionsJSON,OldDocs output + +``` + +### 3. [Build, Deploy and Release Artifact](.github/workflows/release.yaml) +Runs `make archive` when creating new releases, using the git tag (ref_name) as the version. Follows the same cleaning and S3 management process as the production deployment. Creates a release asset equivalent to the `make archive` output for that git tag. + +```mermaid + graph TD + %% Release Workflow (Create Release) + ReleaseWorkflow[Release Workflow
release.yaml] --> |runs| ArchiveMakeRelease[make archive] + ArchiveMakeRelease --> |creates| TaggedVersion[Tagged Version] + TaggedVersion --> |pushes to| S3Tag[S3 Bucket - ref_name] + TaggedVersion --> |maintains| VersionsJSON[versions.json] + TaggedVersion --> |cleans old| OldDocs[Old ref_name Documents] + TaggedVersion --> |creates| ReleaseAsset[Release Asset
wire-docs-ref_name.tar.gz] + + + %% Styling + classDef output fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef workflow fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + + class ReleaseWorkflow workflow + class TaggedVersion,ReleaseAsset,S3Tag,VersionsJSON,OldDocs output + +``` + +### 4. [Test Environment Build and S3 Deploy](.github/workflows/test.yaml) +Tests the above operations in a staging bucket environment. + +```mermaid + graph TD + %% Test Workflow + TestWorkflow[Test Workflow
test.yaml] + TestWorkflow --> |runs| TestArchive[make archive] + TestArchive --> |creates| LatestVersion[latest Version] + LatestVersion --> |cleans old| OldDocs[Old latest Documents Staging] + LatestVersion --> |pushes to| S3Staging[S3 Staging Bucket - Latest] + LatestVersion --> |maintains| VersionsJSON[Staging versions.json] + + %% Styling + classDef output fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef workflow fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + + class TestWorkflow workflow + class LatestVersion,S3Staging,VersionsJSON,OldDocs output +``` + +## Versioning System + +### Version Format +Version numbers follow a three-part format: **A.B.C** + +- **A (Major Version)**: Indicates the minimum API contract version supported by this release. When this increases, it signals deprecation of older API contract versions. For example, V5.0.0 supports API V5 and newer. Major versions may include breaking changes such as database migrations or backend service structure changes. + +- **B (Feature Iteration)**: Reflects feature updates that don't affect API contract compatibility or introduce breaking changes. For example, V5.1.0 supports API V5 and newer while adding new features or improvements without altering compatibility. + +- **C (Fix Iteration)**: Incremented for bug fixes or security updates that don’t introduce new features or modify compatibility. Fix iterations contain no breaking changes. + +**Note**: The first release, `v0.0.0`, indicates that the documentation supports all API versions. + +## Version and Tag Relationship ## Relationship Between Versions and Tags: - - We plan to offer multiple versions of our documentation, with each version corresponding to a specific release in the wire-docs repository. +- We provide multiple documentation versions, with each version corresponding to a specific release in the wire-docs repository. + +- Releases are based on Git tags, which mark a specific point in your repository's history. - - Each version release includes all built documents from previous releases. Releases are based on Git tags, which mark a specific point in your repository's history. +- Once a version or tag is released, it becomes immutable. Future updates require creating new versions—no further edits are permitted on existing tags. - - Once a new version or tag is released, it becomes immutable; future updates will require creating new versions, and no further edits will be permitted on that tag. +-The latest version is regularly updated based on commits to the main branch. diff --git a/Release.md b/Release.md index 2737c13..8d5ca35 100644 --- a/Release.md +++ b/Release.md @@ -1,21 +1,36 @@ ## Generating a Release and Artifact: +### Release Creation Process - We follow GitHub's standard process for creating a `release`, as described in [GitHub's documentation](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release). We use the UI to create a release with the `Create new tag` option, which synchronizes the release creation with tag creation. **NOTE:** Each release should initially be marked as `Set as a pre-release` to allow for verification of the release notes and artifacts. If the release workflow fails, you can fix the issues and create a new release before publishing the final release. - - In the release notes, you can either specify the changes based on commits or use the `Generate release notes` feature to automatically compile a history of commits on the main branch. +### Release Notes +For release notes, you can either: +- Manually specify changes based on commits +- Use the `Generate release notes` feature to automatically compile commit history from the main branch - - All releases should be based on the *main* branch. For each release, changes to the main branch are aggregated through *pull requests*. +### Branch and Pull Request Workflow +- All releases should be based on the *main* branch. For each release, changes to the main branch are aggregated through *pull requests*. - - Each pull request triggers a [build workflow](https://github.com/wireapp/wire-docs/blob/main/.github/workflows/build.yaml) to verify the structure of the documentation. This build process runs for all existing tags to ensure that building the current release does not break the processing of previous versions. It also builds the current branch to identify any navigation or markdown issues. +- Each pull request triggers a [build workflow](.github/workflows/build.yaml) that: + - Verifies documentation structure + - Builds the current branch (as `latest`) to identify navigation or Markdown issues - - As some of the pages are being fetched from [wire-server](https://github.com/wireapp/wire-server/tree/develop) repo via a submodule. We expect to have a PR for updating the submodule pointer. In future, we will automate this process of updating the submodule to latest commits in wire-docs repo. Note: for each commit in wire-server repo for /docs, we have a github workflow (build) to verfiy the structure of documents. +### Deployment Process +Each push to the main branch triggers a [deploy workflow](.github/workflows/deploy.yaml) that: +- Builds the `latest` documentation +- Pushes the `latest` build to the S3 bucket for live serving - - Some pages are fetched from the [wire-server repository](https://github.com/wireapp/wire-server/tree/develop) via a submodule. We expect to open a pull request to update the submodule pointer when necessary. In the future, we plan to automate the process of updating the submodule to the latest commits from the wire-server repository. - - **Note:** For each commit in the wire-server repository that affects the /docs directory, a GitHub workflow (build) is triggered to verify the documentation structure. +### Submodule Management +Some pages are fetched from the [wire-server](https://github.com/wireapp/wire-server/tree/develop) repository via submodules. We expect to submit pull requests for updating submodule pointers. In the future, we plan to automate this process to keep the wire-docs repository synchronized with the latest commits. - - Creating a release triggers a [release](https://github.com/wireapp/wire-docs/blob/main/.github/workflows/release.yaml) which builds all the previous tags and builts the document from current tag, at the end it creates tar file with all the documentation webpages. This tar can be used to serve the documentation via S3 web hosting or normal web server like apache, ngnix or python http.server. +**Note:** For each commit to `/docs` in the wire-server repository, a [build workflow](https://github.com/wireapp/wire-server/blob/develop/.github/workflows/build.yaml) verifies the document structure. + +### Release Workflow +Creating a release triggers a [release workflow](https://github.com/wireapp/wire-docs/blob/main/.github/workflows/release.yaml) that: +1. Builds documentation for the current tag +2. Creates a tar file containing all documentation web pages +3. Pushes the tag (`ref_name`) to the S3 bucket for live serving - - Creating a release triggers a [release workflow](https://github.com/wireapp/wire-docs/blob/main/.github/workflows/release.yaml) that builds all previous tags and the documentation from the current tag. At the end of this process, a tar file containing all the documentation webpages is created. This tar file can be used to serve the documentation via S3 web hosting or with a standard web server such as Apache, Nginx, or Python's http.server. +The tar file created during the release can be used to serve documentation via standard web servers such as Apache, Nginx, or Python's `http.server`. From 8934d00d74f65cbb18d52164b7d8e79cf6e90f6c Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Tue, 3 Jun 2025 21:36:36 +0200 Subject: [PATCH 3/9] WPB-17086: commnting out paths to ignore to test the test build --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 62aa28a..348d7d3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,7 +7,7 @@ on: paths-ignore: - 'README.md' - 'Release.md' - - '.github/**' +## - '.github/**' jobs: test: From 711dce6137e96658cb9f9fac08003e02f78cdd44 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:05:57 +0200 Subject: [PATCH 4/9] WPB-17086: enable debugging for bash commands --- build/build_versions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/build_versions.sh b/build/build_versions.sh index eeecb65..f114f00 100644 --- a/build/build_versions.sh +++ b/build/build_versions.sh @@ -1,6 +1,7 @@ #!/bin/bash -set -e +set -ex +printenv mike="pipenv run mike" CURRENT=$(git branch --show-current) From 60e8f157574c06ac260ae30b81f7d3375b6a625e Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 14:43:56 +0200 Subject: [PATCH 5/9] WPB-17086: fix build_versions.sh to always build latest in case of push to a branch --- README.md | 6 +++--- build/build_versions.sh | 18 +++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6282079..97b208b 100644 --- a/README.md +++ b/README.md @@ -55,16 +55,16 @@ Contains scripts used by the Makefile to support different build use cases. The ## Makefile Targets ### `make current` -Runs the documentation site locally using Mike (MkDocs plugin) for the `current` branch only. Current changes appear under the branch name as the version name. Hosted at `0.0.0.0:8000`. +Runs the documentation site locally using Mike (MkDocs plugin) for the `current` branch only. Current changes appear under the `latest` as the version name. Hosted at `0.0.0.0:8000`. ### `make run` -Serves the documentation site locally by building the current branch as static web pages, then hosts a web server using Python's HTTP module. This allows you to preview how the documentation will appear when hosted for the current branch. Hosted at `0.0.0.0:8000`. +Serves the documentation site locally by building the current branch as static web pages, then hosts a web server using Python's HTTP module. This allows you to preview how the documentation will appear when hosted for the current branch under `latest` version. Hosted at `0.0.0.0:8000`. ### `make archive` Archives the processed web pages for the current branch from the GitHub `gh-pages` branch (created by `make build`). Output is generated in the main directory as `wire-docs.tar.gz`. ### `make build` -Used by `run` and `archive` targets to build the current branch. Attempts to identify the version name for the environment. In GitHub environments, it finds an appropriate name for the version to be built instead of using a branch name. +Used by `run` and `archive` targets to build the current branch. Attempts to identify the version name for the environment. In GitHub environments, it identifies a suitable name for the version being built, based on the type of trigger event, such as a Pull Request, a Push, or the creation of a Tag. ### `make docker` *(Experimental)* Builds a Docker image for the documentation using the repository's Dockerfile. Uses Python's `mike` module to host documents (experimental - not the standard Python HTTP module). For testing current changes, use `make run` or `make current` instead. diff --git a/build/build_versions.sh b/build/build_versions.sh index f114f00..2523cee 100644 --- a/build/build_versions.sh +++ b/build/build_versions.sh @@ -1,7 +1,6 @@ #!/bin/bash -set -ex -printenv +set -e mike="pipenv run mike" CURRENT=$(git branch --show-current) @@ -29,8 +28,9 @@ validate_output() { # checking if it is building from a branch if [ -n "$CURRENT" ]; then - # if building locally, the local tag would be CURRENT_TAG - CURRENT_TAG="$CURRENT" + # Rule: if building locally or building on a push to main/test-dev + # the tag will be latest, the branch name won't matter + CURRENT_TAG="latest" # useful for local users to see their diffs with each mike deploy git --no-pager diff @@ -47,20 +47,16 @@ if [ -n "$CURRENT" ]; then else if [ -n "${GITHUB_REF}" ]; then if [[ "${GITHUB_REF}" == refs/tags/* ]]; then - # For a tag, strip the "refs/tags/" prefix. + # Rule: For a tag, strip the "refs/tags/" prefix. + # when built from a tag and from github env, we build a TAG CURRENT_TAG="${GITHUB_REF#refs/tags/}" elif [[ "${GITHUB_REF}" == refs/pull/* ]]; then - # we build latest everytime there is PR + # Rule: we build latest everytime there is PR CURRENT_TAG="latest" fi fi fi -# Rule: Build latest when push is successful to main branch -if [[ "${CURRENT_TAG}" == "main" ]]; then - CURRENT_TAG="latest" -fi - git tag -f $CURRENT_TAG || true # Fetch the existing tags and their commits from mike From 4b9de1c97b0b819905eaccc3ce31806c5a8d20f4 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:06:50 +0200 Subject: [PATCH 6/9] WPB-17086: fix s3 copy logic for index.html --- .github/workflows/test.yaml | 2 +- build/prepare.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 348d7d3..0caed1e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -54,7 +54,7 @@ jobs: aws s3 sync tmp_extracted/latest s3://${{ secrets.DEV_BUCKET }}/latest # syncing the index.html file - aws s3 sync tmp_extracted/index.html s3://${{ secrets.DEV_BUCKET }}/index.html + aws s3 cp tmp_extracted/index.html s3://${{ secrets.DEV_BUCKET }}/index.html # syncying the all_versions.json with versions.json aws s3 cp tmp_extracted/all_versions.json s3://${{ secrets.DEV_BUCKET }}/versions.json diff --git a/build/prepare.sh b/build/prepare.sh index 2744291..70dbf2f 100644 --- a/build/prepare.sh +++ b/build/prepare.sh @@ -50,5 +50,5 @@ else git branch --show-current > "${TEMP_DIR}/.current_branch" fi -echo "Syncing all the other files changes from ${ORIGINAL_DIR}/ to ${TEMP_DIR}, to have uncommited changes, if any" +echo "Syncing all the other files changes from ${ORIGINAL_DIR}/ to ${TEMP_DIR}, to work on uncommited changes, if any" rsync -a --exclude='/.git' --exclude="/wire-server" --exclude='wire-docs*.tar.gz' --exclude=".tmpdir" "${ORIGINAL_DIR}/" "$TEMP_DIR/" From 2ba3da4694e24a833dd75bc185c8a4e371664e2b Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 18:56:33 +0200 Subject: [PATCH 7/9] WPB-17086: enable debugging for bash commands --- build/build_versions.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build/build_versions.sh b/build/build_versions.sh index 2523cee..49a9a77 100644 --- a/build/build_versions.sh +++ b/build/build_versions.sh @@ -1,9 +1,9 @@ #!/bin/bash -set -e +set -ex mike="pipenv run mike" CURRENT=$(git branch --show-current) - +printenv # using dummy values for user.name and user.email as they are not required for git operations but a requirement for mike to have gh-pages branch git config --local user.name "Wire Docs" git config --local user.email "wire-docs-author@wire.com" @@ -16,7 +16,7 @@ validate_output() { local output="$1" # to get building logs echo "$output" - # ignoring changelog.md as it is talking about past changes + # Rule: ignoring changelog.md as it is talking about past changes remaining_warnings=$(echo "$output" | grep -i "WARNING" | grep -v "Doc file 'changelog/changelog.md' contains a link" | awk '{$1=$1};1') if [ -n "$remaining_warnings" ]; then @@ -65,9 +65,10 @@ while read -r tag commit; do existing_tags[$tag]=$commit done < <($mike list | awk -F '[][]' '{print $1, $2}') +git show-ref + commit=$(git show-ref "refs/tags/${CURRENT_TAG}" | awk '{print $1}') git checkout ${CURRENT_TAG} - # pull the submodule git submodule update --init --depth 1 wire-server From 280b24b21f7862952f3a4362b0922407f548b6e6 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 19:20:59 +0200 Subject: [PATCH 8/9] WPB-17086: fix the logic for merging versions.json --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yaml | 2 +- build/build_versions.sh | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9fee841..247770d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -44,7 +44,7 @@ jobs: aws s3 cp s3://${{ secrets.DEV_BUCKET }}/versions.json tmp_extracted/upstream_versions.json # merge the both versions.json files and keep the unique entries while giving priority to current versions.json - jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/upstream_versions.json tmp_extracted/versions.json > tmp_extracted/all_versions.json + jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/versions.json tmp_extracted/upstream_versions.json > tmp_extracted/all_versions.json # removing old objects from the bucket for the ref_name to ensure, we don't keep objects at old path aws s3 rm s3://${{ secrets.DEV_BUCKET }}/${{ github.ref_name }} --recursive diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0caed1e..7a84dbe 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,7 @@ jobs: aws s3 cp s3://${{ secrets.DEV_BUCKET }}/versions.json tmp_extracted/upstream_versions.json # merge the both versions.json files and keep the unique entries while giving priority to current versions.json - jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/upstream_versions.json tmp_extracted/versions.json > tmp_extracted/all_versions.json + jq -s '.[0] + .[1] | unique_by(.version + .title)' tmp_extracted/versions.json tmp_extracted/upstream_versions.json > tmp_extracted/all_versions.json # removing old objects from the bucket to ensure, we don't keep objects at old path aws s3 rm s3://${{ secrets.DEV_BUCKET }}/latest --recursive diff --git a/build/build_versions.sh b/build/build_versions.sh index 49a9a77..47b7dc8 100644 --- a/build/build_versions.sh +++ b/build/build_versions.sh @@ -1,9 +1,9 @@ #!/bin/bash -set -ex +set -e mike="pipenv run mike" CURRENT=$(git branch --show-current) -printenv + # using dummy values for user.name and user.email as they are not required for git operations but a requirement for mike to have gh-pages branch git config --local user.name "Wire Docs" git config --local user.email "wire-docs-author@wire.com" @@ -65,10 +65,10 @@ while read -r tag commit; do existing_tags[$tag]=$commit done < <($mike list | awk -F '[][]' '{print $1, $2}') -git show-ref - commit=$(git show-ref "refs/tags/${CURRENT_TAG}" | awk '{print $1}') + git checkout ${CURRENT_TAG} + # pull the submodule git submodule update --init --depth 1 wire-server From 1904c006831a9abfb126ab42f0f29448459f9286 Mon Sep 17 00:00:00 2001 From: Mohit Rajain <20745774+mohitrajain@users.noreply.github.com> Date: Wed, 4 Jun 2025 19:55:31 +0200 Subject: [PATCH 9/9] WPB-17086: remove artifact check condition in build.yaml as for a PR artifact building isn't required --- .github/workflows/build.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 685edd2..ec3d8e8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,7 +25,3 @@ jobs: id: build_docs run: | make build - if [ ! -f "wire-docs.tar.gz" ]; then - echo "Artifact not found!" - exit 1 - fi