From 8bb3cf4c5015d286b26e244ff5228688784f7afa Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Mon, 28 Jul 2025 16:10:23 +1200 Subject: [PATCH 01/29] Bumped versions of base image dependencies (#365) --- images/awx-ee/execution-environment.yml | 12 ++++++------ images/ci-builder/Dockerfile | 4 ++-- images/mailpit/Dockerfile | 2 +- images/node/Dockerfile | 2 +- images/php/Dockerfile.cli | 4 ++-- images/php/Dockerfile.fpm | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/images/awx-ee/execution-environment.yml b/images/awx-ee/execution-environment.yml index 58f0c8cc..b704bb66 100644 --- a/images/awx-ee/execution-environment.yml +++ b/images/awx-ee/execution-environment.yml @@ -23,17 +23,17 @@ additional_build_steps: - LABEL org.opencontainers.image.title="SDP AWX Execution Environment image." - LABEL org.opencontainers.image.description="Provides an AWX execution environment image optimised for use with SDP. Built with ansible-builder." - LABEL org.opencontainers.image.source="https://github.com/dpc-sdp/bay/blob/6.x/images/awx-ee/" - - ARG LAGOON_CLI_VERSION=v0.31.2 - - ARG NVM_INSTALL_VERSION=v0.39.7 - - ARG NODE_VERSION=v20.9.0 + - ARG LAGOON_CLI_VERSION=v0.32.0 + - ARG NVM_INSTALL_VERSION=v0.40.3 + - ARG NODE_VERSION=v20.19.3 - ARG NVM_DIR="/runner/.nvm" - ARG PHP_VERSION="8.3" - ARG COMPOSER_VERSION="2.7.7" - ARG HUB_VERSION="2.14.2" - ARG GOJQ_VERSION="0.12.17" - - ARG HELM_VERSION="3.17.0" - - ARG YAMLFMT_VERSION="0.15.0" - - ARG KUBECTL_VERSION="1.32.0" + - ARG HELM_VERSION="3.18.3" + - ARG YAMLFMT_VERSION="0.17.2" + - ARG KUBECTL_VERSION="1.33.2" append_final: - | # Required dependencies. diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index e2bcda5f..39aba327 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -1,10 +1,10 @@ FROM hashicorp/terraform:latest AS terraform FROM ghcr.io/dpc-sdp/sumocli:v0.11.1 AS sumocli FROM php:8.3-cli-alpine -ARG AHOY_VERSION=2.2.0 +ARG AHOY_VERSION=2.4.0 ARG GOJQ_VERSION=0.12.17 ARG HUB_VERSION=2.14.2 -ARG LAGOON_CLI_VERSION=0.31.2 +ARG LAGOON_CLI_VERSION=0.32.0 ARG SHIPSHAPE_VERSION=1.0.0-alpha.1.5.1 # Ensure temp files dont end up in image. diff --git a/images/mailpit/Dockerfile b/images/mailpit/Dockerfile index 6feaf6c8..85da1b96 100644 --- a/images/mailpit/Dockerfile +++ b/images/mailpit/Dockerfile @@ -3,7 +3,7 @@ # FROM alpine:latest -ARG MAILPIT_VERSION=1.21.8 +ARG MAILPIT_VERSION=1.26.2 # Install ca-certificates, required for the "release message" feature: RUN apk --no-cache add \ diff --git a/images/node/Dockerfile b/images/node/Dockerfile index 5a4d1613..c0c8c870 100644 --- a/images/node/Dockerfile +++ b/images/node/Dockerfile @@ -1,5 +1,5 @@ FROM uselagoon/node-20:latest -ARG BAY_CLI_VERSION=v1.3.2 +ARG BAY_CLI_VERSION=v1.4.1 RUN apk --update add curl git findutils openssh-client && \ diff --git a/images/php/Dockerfile.cli b/images/php/Dockerfile.cli index e2f65615..f96d9ca3 100644 --- a/images/php/Dockerfile.cli +++ b/images/php/Dockerfile.cli @@ -1,6 +1,6 @@ ARG PHP_VERSION=8.3 FROM php:${PHP_VERSION}-cli-alpine AS php-cli -FROM ghcr.io/skpr/mtk:v2.1.0 AS mtk +FROM ghcr.io/skpr/mtk:v2.1.1 AS mtk FROM uselagoon/php-${PHP_VERSION}-cli-drupal:latest # Remove unnecessary packages that increase our attack surface area. @@ -8,7 +8,7 @@ RUN apk del postgresql-client ARG GOJQ_VERSION=0.12.17 ARG DOCKERIZE_VERSION=v0.9.2 -ARG BAY_CLI_VERSION=v1.3.2 +ARG BAY_CLI_VERSION=v1.4.1 ARG SHIPSHAPE_VERSION=1.0.0-alpha.1.5.1 COPY --from=php-cli /usr/local/bin/phpdbg /usr/local/bin/ diff --git a/images/php/Dockerfile.fpm b/images/php/Dockerfile.fpm index ba07ff15..845f29a8 100644 --- a/images/php/Dockerfile.fpm +++ b/images/php/Dockerfile.fpm @@ -2,7 +2,7 @@ ARG PHP_VERSION=8.3 FROM ghcr.io/dpc-sdp/bay/php-fpm-exporter:6.x AS php-fpm-exporter FROM uselagoon/php-${PHP_VERSION}-fpm:latest -ARG BAY_CLI_VERSION=v1.3.2 +ARG BAY_CLI_VERSION=v1.4.1 RUN mkdir /bay COPY 01-bay.ini /usr/local/etc/php/conf.d/ From e97cc1dd0620281d9eef431ed1d4dcaa381ab49e Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Mon, 28 Jul 2025 17:23:08 +1200 Subject: [PATCH 02/29] Adds workflow file to sync 6.x to 7.x (#368) --- .github/workflows/branch-sync.yml | 68 +++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/branch-sync.yml diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml new file mode 100644 index 00000000..5386428e --- /dev/null +++ b/.github/workflows/branch-sync.yml @@ -0,0 +1,68 @@ +name: Merge 6.x into 7.x + +on: + workflow_dispatch: {} + schedule: + - cron: '0 23 * * 0' # Run every Sunday at 11 PM UTC (Monday 9 AM AEST) + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + create-pull-request: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Prepare sync branch + run: | + # Fetch all branches + git fetch origin + + # Create a new sync branch from 7.x (target) + git checkout -b temp-sync-branch origin/7.x + + # Find all 6.x commits not on 7.x and cherry-pick them + COMMITS=$(git log --reverse --pretty=format:"%H" origin/7.x..origin/6.x) + if [ -z "$COMMITS" ]; then + echo "NO_CHANGES=true" >> "${GITHUB_ENV}" + exit 0 + fi + + for COMMIT in $COMMITS; do + git log -1 --format=%s "${COMMIT}" + git cherry-pick "${COMMIT}" || { echo "Cherry-pick failed"; exit 1; } + done + + - name: Create a pull request from sync branch to 7.x + if: env.NO_CHANGES != 'true' + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: 7.x + branch: temp-sync-branch + title: "Sync changes from 6.x" + body: | + This pull request syncs relevant changes from the `6.x` branch into the `7.x` branch. + + This PR was automatically generated by the CI workflow. + labels: sync, automated + draft: false + sign-commits: true + + - name: Enable auto-merge + if: env.NO_CHANGES != 'true' + run: | + gh pr merge temp-sync-branch --auto --merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dc77275287356f658dd335f3d09fc289ce8e160a Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 1 Aug 2025 13:56:24 +1200 Subject: [PATCH 03/29] Added aws-es-proxy base image (#370) --- .github/workflows/vulnerability-scan.yml | 1 + gh-actions-bake.hcl | 9 ++++ images/aws-es-proxy/Dockerfile | 24 +++++++++ images/aws-es-proxy/README.md | 62 ++++++++++++++++++++++++ images/aws-es-proxy/entrypoint.sh | 44 +++++++++++++++++ 5 files changed, 140 insertions(+) create mode 100644 images/aws-es-proxy/Dockerfile create mode 100644 images/aws-es-proxy/README.md create mode 100755 images/aws-es-proxy/entrypoint.sh diff --git a/.github/workflows/vulnerability-scan.yml b/.github/workflows/vulnerability-scan.yml index 863b08b5..bd6eb6c1 100644 --- a/.github/workflows/vulnerability-scan.yml +++ b/.github/workflows/vulnerability-scan.yml @@ -39,6 +39,7 @@ jobs: exclude: - images: mailpit - images: php-fpm-exporter + - images: aws-es-proxy - if: inputs.tag != '5.x' uses: druzsan/setup-matrix@v2 with: diff --git a/gh-actions-bake.hcl b/gh-actions-bake.hcl index 291bd0cd..ed954397 100755 --- a/gh-actions-bake.hcl +++ b/gh-actions-bake.hcl @@ -93,6 +93,15 @@ target "php-fpm-exporter" { "org.opencontainers.image.source" = "https://github.com/dpc-sdp/bay/blob/6.x/images/bay-php-exporter/Dockerfile" } } + +target "aws-es-proxy" { + inherits = ["docker-metadata-action"] + context = "${CONTEXT}/aws-es-proxy" + dockerfile = "Dockerfile" + + platforms = ["linux/amd64", "linux/arm64"] + +} target "ripple-static" { inherits = ["docker-metadata-action"] context = "${CONTEXT}/ripple-static" diff --git a/images/aws-es-proxy/Dockerfile b/images/aws-es-proxy/Dockerfile new file mode 100644 index 00000000..32876112 --- /dev/null +++ b/images/aws-es-proxy/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:alpine AS build + +RUN apk add --no-cache git +RUN git clone https://github.com/abutaha/aws-es-proxy.git /go/src/github.com/abutaha/aws-es-proxy +WORKDIR /go/src/github.com/abutaha/aws-es-proxy + +RUN CGO_ENABLED=0 GOOS=linux go build -o aws-es-proxy + +FROM alpine:latest + +RUN apk --no-cache add ca-certificates +WORKDIR /home/ +COPY --from=build /go/src/github.com/abutaha/aws-es-proxy/aws-es-proxy /usr/local/bin/ +COPY entrypoint.sh /entrypoint.sh +RUN apk add --no-cache bash aws-cli + +ENV BAY_OPENSEARCH_ENDPOINT= +ENV BAY_OPENSEARCH_ROLE= +ENV BAY_OPENSEARCH_PROXY_TIMEOUT=60 +ENV BAY_OPENSEARCH_PROXY_FLAGS= +ENV BAY_OPENSEARCH_PROXY_PORT=3000 +EXPOSE 3000 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/aws-es-proxy/README.md b/images/aws-es-proxy/README.md new file mode 100644 index 00000000..c397db05 --- /dev/null +++ b/images/aws-es-proxy/README.md @@ -0,0 +1,62 @@ +# Container Image - aws-es-proxy + +This container provides a secure proxy for requests to an AWS OpenSearch endpoint using the [aws-es-proxy](https://github.com/abutaha/aws-es-proxy) +tool. It is designed for seamless integration and automated IAM authentication, with robust defaults and runtime +configuration via environment variables. + +## Features + +- Secure proxying to AWS OpenSearch services. +- Automatic validation of critical environment variables and AWS credentials. +- Configurable timeouts, port, and proxy flags. +- Flexible runtime configuration for debugging and verbosity. + +## Usage + +This image is typically intended for use as a proxy in your infrastructure. +You can use it in your Docker Compose stack with the following snippet: + +```yaml +services: + aws-es-proxy: + image: ghcr.io/dpc-sdp/bay/aws-es-proxy:6.x + environment: + BAY_OPENSEARCH_ENDPOINT=https://your-opensearch-endpoint.amazonaws.com + BAY_OPENSEARCH_ROLE=arn:aws:iam::123456789012:role/your-role + ports: + - "3000:3000" +``` + + +## Environment Variables + +| Name | Default Value | Description | +|---------------------------|--------------|-----------------------------------------------------------------------------------------------| +| `BAY_OPENSEARCH_ENDPOINT` | _(required)_ | The AWS OpenSearch domain endpoint to proxy requests to. | +| `BAY_OPENSEARCH_ROLE` | _(required)_ | The AWS IAM role to assume for accessing the OpenSearch domain. | +| `BAY_OPENSEARCH_PROXY_PORT` | `3000` | The port that the proxy listens on, inside the container. | +| `BAY_OPENSEARCH_PROXY_TIMEOUT` | `60` | Timeout (in seconds) for incoming connections. | +| `BAY_OPENSEARCH_PROXY_FLAGS` | (empty) | Extra flags passed to aws-es-proxy (e.g., `-debug -verbose`). See [aws-es-proxy docs](https://github.com/abutaha/aws-es-proxy?tab=readme-ov-file#usage-example) for options. | + +### Example: Enabling Debug and Verbose Logging + +``` +BAY_OPENSEARCH_PROXY_FLAGS=-debug -verbose +``` + +## Ports + +- **3000** (default, can be customized with `BAY_OPENSEARCH_PROXY_PORT`) – Proxy HTTP port + +## Entrypoint + +The container runs an entrypoint script that: + +1. Verifies mandatory environment variables and AWS credentials. +2. Launches `aws-es-proxy` with your configuration. + +## AWS Credentials + +The container expects valid AWS credentials to be supplied via standard mechanisms (environment variables, mounted credentials files, or IAM roles if running in AWS ECS/EKS environments). + +For more advanced configuration, refer to the [aws-es-proxy documentation](https://github.com/abutaha/aws-es-proxy). \ No newline at end of file diff --git a/images/aws-es-proxy/entrypoint.sh b/images/aws-es-proxy/entrypoint.sh new file mode 100755 index 00000000..62165e87 --- /dev/null +++ b/images/aws-es-proxy/entrypoint.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# This script acts as a wrapper to securely proxy requests to an OpenSearch endpoint +# using the aws-es-proxy tool. It performs the following steps: +# 1. Validates that required environment variables (BAY_OPENSEARCH_ENDPOINT and BAY_OPENSEARCH_ROLE) +# are set and not empty. +# 2. Verifies that valid AWS credentials are present. If credentials are invalid or missing, the +# script exits with an error. +# 3. Starts the aws-es-proxy service. +# +# The following environment variables can be used to configure the behavior of this script: +# BAY_OPENSEARCH_ENDPOINT: The AWS opensearch domain endpoint. +# BAY_OPENSEARCH_ROLE: The AWS IAM role that should be assumed to access the opensearch +# domain. +# BAY_OPENSEARCH_PROXY_PORT: Port that the aws-es-proxy should bind to. +# BAY_OPENSEARCH_PROXY_TIMEOUT: Timeout for incoming connections. +# BAY_OPENSEARCH_PROXY_FLAGS: See [aws-es-proxy docs](https://github.com/abutaha/aws-es-proxy?tab=readme-ov-file#usage-example) +# for supported options here. +# +# If verbose or debug logs are required, set env var BAY_OPENSEARCH_PROXY_FLAGS="-debug -verbose" + +set -euo pipefail + +# Check if BAY_OPENSEARCH_ENDPOINT is unset or empty +if [ -z "${BAY_OPENSEARCH_ENDPOINT:-}" ]; then + echo "Error: BAY_OPENSEARCH_ENDPOINT is not set or is empty" >&2 + exit 1 +fi + +# Check if BAY_OPENSEARCH_ROLE is unset or empty +if [ -z "${BAY_OPENSEARCH_ROLE:-}" ]; then + echo "Error: BAY_OPENSEARCH_ROLE= is not set or is empty" >&2 + exit 1 +fi + +# Ensure AWS credentials exist and are valid +AWS_PAGER="" aws sts get-caller-identity || (echo "Error: AWS credentials invalid" && exit 1) + +# Rest of your script here +aws-es-proxy "${BAY_OPENSEARCH_PROXY_FLAGS:-}" \ + -listen "0.0.0.0:${BAY_OPENSEARCH_PROXY_PORT:-3000}" \ + -timeout "${BAY_OPENSEARCH_PROXY_TIMEOUT:-60}" \ + -assume "${BAY_OPENSEARCH_ROLE}" \ + -endpoint "${BAY_OPENSEARCH_ENDPOINT}" From d2dc51aefcf3acd8c8f748ac955d5e675e54b09c Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 1 Aug 2025 14:10:13 +1200 Subject: [PATCH 04/29] Added required label to aws-es-proxy image --- gh-actions-bake.hcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gh-actions-bake.hcl b/gh-actions-bake.hcl index ed954397..2133fa73 100755 --- a/gh-actions-bake.hcl +++ b/gh-actions-bake.hcl @@ -100,7 +100,9 @@ target "aws-es-proxy" { dockerfile = "Dockerfile" platforms = ["linux/amd64", "linux/arm64"] - + labels = { + "org.opencontainers.image.source" = "https://github.com/dpc-sdp/bay/blob/6.x/images/aws-es-proxy/Dockerfile" + } } target "ripple-static" { inherits = ["docker-metadata-action"] From cbd91bb18085bea38fdd6cd1e68eb253609c0bba Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 1 Aug 2025 16:31:18 +1200 Subject: [PATCH 05/29] Fixed issue with aws-es-proxy with various proxy flags envvar combinations (#371) --- images/aws-es-proxy/Dockerfile | 3 ++- images/aws-es-proxy/README.md | 15 ++++++++------- images/aws-es-proxy/entrypoint.sh | 23 ++++++++++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/images/aws-es-proxy/Dockerfile b/images/aws-es-proxy/Dockerfile index 32876112..3374fbff 100644 --- a/images/aws-es-proxy/Dockerfile +++ b/images/aws-es-proxy/Dockerfile @@ -17,7 +17,8 @@ RUN apk add --no-cache bash aws-cli ENV BAY_OPENSEARCH_ENDPOINT= ENV BAY_OPENSEARCH_ROLE= ENV BAY_OPENSEARCH_PROXY_TIMEOUT=60 -ENV BAY_OPENSEARCH_PROXY_FLAGS= +ENV BAY_OPENSEARCH_PROXY_VERBOSE= +ENV BAY_OPENSEARCH_PROXY_DEBUG= ENV BAY_OPENSEARCH_PROXY_PORT=3000 EXPOSE 3000 diff --git a/images/aws-es-proxy/README.md b/images/aws-es-proxy/README.md index c397db05..58456ef6 100644 --- a/images/aws-es-proxy/README.md +++ b/images/aws-es-proxy/README.md @@ -30,13 +30,14 @@ services: ## Environment Variables -| Name | Default Value | Description | -|---------------------------|--------------|-----------------------------------------------------------------------------------------------| -| `BAY_OPENSEARCH_ENDPOINT` | _(required)_ | The AWS OpenSearch domain endpoint to proxy requests to. | -| `BAY_OPENSEARCH_ROLE` | _(required)_ | The AWS IAM role to assume for accessing the OpenSearch domain. | -| `BAY_OPENSEARCH_PROXY_PORT` | `3000` | The port that the proxy listens on, inside the container. | -| `BAY_OPENSEARCH_PROXY_TIMEOUT` | `60` | Timeout (in seconds) for incoming connections. | -| `BAY_OPENSEARCH_PROXY_FLAGS` | (empty) | Extra flags passed to aws-es-proxy (e.g., `-debug -verbose`). See [aws-es-proxy docs](https://github.com/abutaha/aws-es-proxy?tab=readme-ov-file#usage-example) for options. | +| Name | Default Value | Description | +|---------------------------|---------------|-----------------------------------------------------------------| +| `BAY_OPENSEARCH_ENDPOINT` | _(required)_ | The AWS OpenSearch domain endpoint to proxy requests to. | +| `BAY_OPENSEARCH_ROLE` | _(required)_ | The AWS IAM role to assume for accessing the OpenSearch domain. | +| `BAY_OPENSEARCH_PROXY_PORT` | `3000` | The port that the proxy listens on, inside the container. | +| `BAY_OPENSEARCH_PROXY_TIMEOUT` | `60` | Timeout (in seconds) for incoming connections. | +| `BAY_OPENSEARCH_PROXY_VERBOSE` | `false` | enables verbose logging | +| `BAY_OPENSEARCH_PROXY_DEBUG` | `false` | enables debug logging | ### Example: Enabling Debug and Verbose Logging diff --git a/images/aws-es-proxy/entrypoint.sh b/images/aws-es-proxy/entrypoint.sh index 62165e87..3bd1620c 100755 --- a/images/aws-es-proxy/entrypoint.sh +++ b/images/aws-es-proxy/entrypoint.sh @@ -14,10 +14,8 @@ # domain. # BAY_OPENSEARCH_PROXY_PORT: Port that the aws-es-proxy should bind to. # BAY_OPENSEARCH_PROXY_TIMEOUT: Timeout for incoming connections. -# BAY_OPENSEARCH_PROXY_FLAGS: See [aws-es-proxy docs](https://github.com/abutaha/aws-es-proxy?tab=readme-ov-file#usage-example) -# for supported options here. -# -# If verbose or debug logs are required, set env var BAY_OPENSEARCH_PROXY_FLAGS="-debug -verbose" +# BAY_OPENSEARCH_PROXY_VERBOSE: "true" to enable proxy verbose logs +# BAY_OPENSEARCH_PROXY_DEBUG: "true" to enable proxy debug logs set -euo pipefail @@ -33,11 +31,26 @@ if [ -z "${BAY_OPENSEARCH_ROLE:-}" ]; then exit 1 fi +# Configure debug and verbose flags +AWS_ES_PROXY_DEBUG_FLAG="" +AWS_ES_PROXY_VERBOSE_FLAG="" + +if [ "${BAY_OPENSEARCH_PROXY_DEBUG:-false}" = "true" ]; then + AWS_ES_PROXY_DEBUG_FLAG="-debug" +fi + +if [ "${BAY_OPENSEARCH_PROXY_VERBOSE:-false}" = "true" ]; then + AWS_ES_PROXY_VERBOSE_FLAG="-verbose" +fi + + # Ensure AWS credentials exist and are valid AWS_PAGER="" aws sts get-caller-identity || (echo "Error: AWS credentials invalid" && exit 1) # Rest of your script here -aws-es-proxy "${BAY_OPENSEARCH_PROXY_FLAGS:-}" \ +aws-es-proxy \ + ${AWS_ES_PROXY_DEBUG_FLAG} \ + ${AWS_ES_PROXY_VERBOSE_FLAG} \ -listen "0.0.0.0:${BAY_OPENSEARCH_PROXY_PORT:-3000}" \ -timeout "${BAY_OPENSEARCH_PROXY_TIMEOUT:-60}" \ -assume "${BAY_OPENSEARCH_ROLE}" \ From f6fe31923a9eafa95e63d79310edab28b80319a1 Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Mon, 4 Aug 2025 11:48:55 +1200 Subject: [PATCH 06/29] Updated aws-es-proxy port default from 3000 to 9200 (#372) --- images/aws-es-proxy/Dockerfile | 4 ++-- images/aws-es-proxy/README.md | 10 +++++----- images/aws-es-proxy/entrypoint.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/images/aws-es-proxy/Dockerfile b/images/aws-es-proxy/Dockerfile index 3374fbff..a2ff799d 100644 --- a/images/aws-es-proxy/Dockerfile +++ b/images/aws-es-proxy/Dockerfile @@ -19,7 +19,7 @@ ENV BAY_OPENSEARCH_ROLE= ENV BAY_OPENSEARCH_PROXY_TIMEOUT=60 ENV BAY_OPENSEARCH_PROXY_VERBOSE= ENV BAY_OPENSEARCH_PROXY_DEBUG= -ENV BAY_OPENSEARCH_PROXY_PORT=3000 -EXPOSE 3000 +ENV BAY_OPENSEARCH_PROXY_PORT=9200 +EXPOSE 9200 ENTRYPOINT ["/entrypoint.sh"] diff --git a/images/aws-es-proxy/README.md b/images/aws-es-proxy/README.md index 58456ef6..f25f0b2e 100644 --- a/images/aws-es-proxy/README.md +++ b/images/aws-es-proxy/README.md @@ -24,7 +24,7 @@ services: BAY_OPENSEARCH_ENDPOINT=https://your-opensearch-endpoint.amazonaws.com BAY_OPENSEARCH_ROLE=arn:aws:iam::123456789012:role/your-role ports: - - "3000:3000" + - "9200:9200" ``` @@ -34,10 +34,10 @@ services: |---------------------------|---------------|-----------------------------------------------------------------| | `BAY_OPENSEARCH_ENDPOINT` | _(required)_ | The AWS OpenSearch domain endpoint to proxy requests to. | | `BAY_OPENSEARCH_ROLE` | _(required)_ | The AWS IAM role to assume for accessing the OpenSearch domain. | -| `BAY_OPENSEARCH_PROXY_PORT` | `3000` | The port that the proxy listens on, inside the container. | +| `BAY_OPENSEARCH_PROXY_PORT` | `9200` | The port that the proxy listens on, inside the container. | | `BAY_OPENSEARCH_PROXY_TIMEOUT` | `60` | Timeout (in seconds) for incoming connections. | -| `BAY_OPENSEARCH_PROXY_VERBOSE` | `false` | enables verbose logging | -| `BAY_OPENSEARCH_PROXY_DEBUG` | `false` | enables debug logging | +| `BAY_OPENSEARCH_PROXY_VERBOSE` | `false` | enables verbose logging | +| `BAY_OPENSEARCH_PROXY_DEBUG` | `false` | enables debug logging | ### Example: Enabling Debug and Verbose Logging @@ -47,7 +47,7 @@ BAY_OPENSEARCH_PROXY_FLAGS=-debug -verbose ## Ports -- **3000** (default, can be customized with `BAY_OPENSEARCH_PROXY_PORT`) – Proxy HTTP port +- **9200** (default, can be customized with `BAY_OPENSEARCH_PROXY_PORT`) – Proxy HTTP port ## Entrypoint diff --git a/images/aws-es-proxy/entrypoint.sh b/images/aws-es-proxy/entrypoint.sh index 3bd1620c..c17d73a7 100755 --- a/images/aws-es-proxy/entrypoint.sh +++ b/images/aws-es-proxy/entrypoint.sh @@ -51,7 +51,7 @@ AWS_PAGER="" aws sts get-caller-identity || (echo "Error: AWS credentials invali aws-es-proxy \ ${AWS_ES_PROXY_DEBUG_FLAG} \ ${AWS_ES_PROXY_VERBOSE_FLAG} \ - -listen "0.0.0.0:${BAY_OPENSEARCH_PROXY_PORT:-3000}" \ + -listen "0.0.0.0:${BAY_OPENSEARCH_PROXY_PORT:-9200}" \ -timeout "${BAY_OPENSEARCH_PROXY_TIMEOUT:-60}" \ -assume "${BAY_OPENSEARCH_ROLE}" \ -endpoint "${BAY_OPENSEARCH_ENDPOINT}" From 672dccec0976f592257dd5c841eddc0bf316b04e Mon Sep 17 00:00:00 2001 From: ross-b Date: Wed, 6 Aug 2025 09:39:46 +1000 Subject: [PATCH 07/29] SDPPE-42: removed BAY_INGRESS_* environment variables and related logic. (#374) --- README.md | 16 ----------- images/nginx/Dockerfile | 10 +------ images/nginx/README.md | 7 +---- .../nginx/prepend/100-ingress-protection.conf | 27 ------------------- 4 files changed, 2 insertions(+), 58 deletions(-) delete mode 100644 images/nginx/prepend/100-ingress-protection.conf diff --git a/README.md b/README.md index a370eb2e..5e15826e 100644 --- a/README.md +++ b/README.md @@ -26,22 +26,6 @@ Learn more from https://docs.lagoon.sh/ ## Bay Features -### Lock-down Ingress with Pre-Shared Key - -Using the nginx image, you can lock down access to your application with using a pre-shared key added at your CDN. - -Set these environment variables in your nginx deployment: - -- `BAY_INGRESS_HEADER` defines the header which has the pre-shared key. -- `BAY_INGRESS_PSK` is the token / PSK value. -- `BAY_INGRESS_ENABLED` is a toggle for this feature, must be set to `"true"`. - -In your CDN configuration, set the header defined in `BAY_INGRESS_HEADER` with the token defined in `BAY_INGRESS_PSK`. - -- [Cloudfront](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/add-origin-custom-headers.html) - -Once deployed, if the header is missing in the request nginx will return a `405 Not Allowed` HTTP response. - ### Multiple architecture support Bay images are available in both amd64 and arm64 architectures. diff --git a/images/nginx/Dockerfile b/images/nginx/Dockerfile index bd52888e..eb457a73 100644 --- a/images/nginx/Dockerfile +++ b/images/nginx/Dockerfile @@ -2,19 +2,11 @@ FROM uselagoon/nginx-drupal:latest ENV WEBROOT=docroot ENV TZ=Australia/Melbourne -ENV BAY_INGRESS_ENABLED=false -ENV BAY_INGRESS_HEADER="" -ENV BAY_INGRESS_PSK="" + EXPOSE 8080 EXPOSE 50000 -# Add ingress protection environment variable supprot to nginx. -RUN sed -i '/env\ LAGOON_ENVIRONMENT_TYPE\;/a env BAY_INGRESS_ENABLED\;' /etc/nginx/nginx.conf \ - && sed -i '/env\ LAGOON_ENVIRONMENT_TYPE\;/a env BAY_INGRESS_HEADER\;' /etc/nginx/nginx.conf \ - && sed -i '/env\ LAGOON_ENVIRONMENT_TYPE\;/a env BAY_INGRESS_PSK\;' /etc/nginx/nginx.conf - COPY helpers/ /etc/nginx/helpers/ -COPY prepend/ /etc/nginx/conf.d/drupal/ COPY content /etc/nginx/conf.d/drupal/content # Add server append directives. diff --git a/images/nginx/README.md b/images/nginx/README.md index f4c79188..8fecbc15 100644 --- a/images/nginx/README.md +++ b/images/nginx/README.md @@ -3,7 +3,6 @@ Provides a nginx image optimised for the Bay container platform with the following features - Drupal compatible server block -- Ingress protection with pre-shared keys - Optimised health checks for section.io ## Usage @@ -27,11 +26,7 @@ services: ## Environment Variables -| Name | Default Value | Description | -|------|---------------|-------------| -| `BAY_INGRESS_ENABLED` | `false` | Global toggle for ingress protection. Set to "true" to enable. | -| `BAY_INGRESS_HEADER` | `` | Name of header with PSK. | -| `BAY_INGRESS_PSK` | `` | Pre-shared key value | +None. ## Ports diff --git a/images/nginx/prepend/100-ingress-protection.conf b/images/nginx/prepend/100-ingress-protection.conf deleted file mode 100644 index 1c89a1d4..00000000 --- a/images/nginx/prepend/100-ingress-protection.conf +++ /dev/null @@ -1,27 +0,0 @@ -# Bay Ingress Protection -# -# Allows nginx to reject requests if the corresponding PSK is not given by -# the requesting client. - -access_by_lua_block { - local ingress_protection_enabled = os.getenv('BAY_INGRESS_ENABLED') - local ingress_protection_psk = os.getenv('BAY_INGRESS_PSK') - local ingress_protection_key = os.getenv('BAY_INGRESS_HEADER') - - if (ingress_protection_enabled == nil) or (ingress_protection_psk == nil) or (ingress_protection_key == nil) then - return - end - - local sent_psk = ngx.req.get_headers()[ingress_protection_key] - - if (ingress_protection_enabled == "true") and (ingress_protection_psk ~= sent_psk) then - local path = "/etc/nginx/conf.d/drupal/content/404.html" - local file = io.open(path, 'r') - local content = file:read "*all" - file:close() - ngx.header['Content-Type'] = 'text/html' - ngx.status = ngx.HTTP_NOT_FOUND - ngx.say(content) - ngx.exit(ngx.OK) - end -} From 44e598559872be14c72ab4a5d32ef3e61c4997a9 Mon Sep 17 00:00:00 2001 From: Md Nadim Hossain Date: Tue, 12 Aug 2025 12:11:17 +1000 Subject: [PATCH 08/29] [SD-1147] Removed the colour indicator from bay. --- images/php/settings.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/php/settings.php b/images/php/settings.php index a05344db..785dc882 100755 --- a/images/php/settings.php +++ b/images/php/settings.php @@ -411,7 +411,5 @@ $label = ($tag !== 'No tag found') ? "$msg - $authorName ($short_sha)" : $short_sha; $config['environment_indicator.indicator']['name'] = "Deployed: $label"; - $config['environment_indicator.indicator']['bg_color'] = '#fff176'; - $config['environment_indicator.indicator']['fg_color'] = '#000000'; } } From 87d4c033a9e68c556236c7ae49ab4ac8d84e78e0 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Wed, 13 Aug 2025 09:19:17 +1000 Subject: [PATCH 09/29] [SDPPE-124] Fix 'aws es proxy not found' (#377) --- images/awx-ee/execution-environment.yml | 2 +- images/ci-builder/Dockerfile | 2 +- images/php/Dockerfile.cli | 2 +- images/php/Dockerfile.fpm | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/awx-ee/execution-environment.yml b/images/awx-ee/execution-environment.yml index b704bb66..ffa98b08 100644 --- a/images/awx-ee/execution-environment.yml +++ b/images/awx-ee/execution-environment.yml @@ -27,7 +27,7 @@ additional_build_steps: - ARG NVM_INSTALL_VERSION=v0.40.3 - ARG NODE_VERSION=v20.19.3 - ARG NVM_DIR="/runner/.nvm" - - ARG PHP_VERSION="8.3" + - ARG PHP_VERSION="8.4" - ARG COMPOSER_VERSION="2.7.7" - ARG HUB_VERSION="2.14.2" - ARG GOJQ_VERSION="0.12.17" diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index 39aba327..0bd9c4d7 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -1,6 +1,6 @@ FROM hashicorp/terraform:latest AS terraform FROM ghcr.io/dpc-sdp/sumocli:v0.11.1 AS sumocli -FROM php:8.3-cli-alpine +FROM php:8.4-cli-alpine ARG AHOY_VERSION=2.4.0 ARG GOJQ_VERSION=0.12.17 ARG HUB_VERSION=2.14.2 diff --git a/images/php/Dockerfile.cli b/images/php/Dockerfile.cli index f96d9ca3..e9865359 100644 --- a/images/php/Dockerfile.cli +++ b/images/php/Dockerfile.cli @@ -1,4 +1,4 @@ -ARG PHP_VERSION=8.3 +ARG PHP_VERSION=8.4 FROM php:${PHP_VERSION}-cli-alpine AS php-cli FROM ghcr.io/skpr/mtk:v2.1.1 AS mtk FROM uselagoon/php-${PHP_VERSION}-cli-drupal:latest diff --git a/images/php/Dockerfile.fpm b/images/php/Dockerfile.fpm index 845f29a8..cd73a03f 100644 --- a/images/php/Dockerfile.fpm +++ b/images/php/Dockerfile.fpm @@ -1,4 +1,4 @@ -ARG PHP_VERSION=8.3 +ARG PHP_VERSION=8.4 FROM ghcr.io/dpc-sdp/bay/php-fpm-exporter:6.x AS php-fpm-exporter FROM uselagoon/php-${PHP_VERSION}-fpm:latest From 4b83b6c0cc37781acec794e4a774418c371ea739 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Wed, 13 Aug 2025 14:31:33 +1000 Subject: [PATCH 10/29] Revert "[SDPPE-124] Fix 'aws es proxy not found' (#377)" This reverts commit 87d4c033a9e68c556236c7ae49ab4ac8d84e78e0. --- images/awx-ee/execution-environment.yml | 2 +- images/ci-builder/Dockerfile | 2 +- images/php/Dockerfile.cli | 2 +- images/php/Dockerfile.fpm | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/awx-ee/execution-environment.yml b/images/awx-ee/execution-environment.yml index ffa98b08..b704bb66 100644 --- a/images/awx-ee/execution-environment.yml +++ b/images/awx-ee/execution-environment.yml @@ -27,7 +27,7 @@ additional_build_steps: - ARG NVM_INSTALL_VERSION=v0.40.3 - ARG NODE_VERSION=v20.19.3 - ARG NVM_DIR="/runner/.nvm" - - ARG PHP_VERSION="8.4" + - ARG PHP_VERSION="8.3" - ARG COMPOSER_VERSION="2.7.7" - ARG HUB_VERSION="2.14.2" - ARG GOJQ_VERSION="0.12.17" diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index 0bd9c4d7..39aba327 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -1,6 +1,6 @@ FROM hashicorp/terraform:latest AS terraform FROM ghcr.io/dpc-sdp/sumocli:v0.11.1 AS sumocli -FROM php:8.4-cli-alpine +FROM php:8.3-cli-alpine ARG AHOY_VERSION=2.4.0 ARG GOJQ_VERSION=0.12.17 ARG HUB_VERSION=2.14.2 diff --git a/images/php/Dockerfile.cli b/images/php/Dockerfile.cli index e9865359..f96d9ca3 100644 --- a/images/php/Dockerfile.cli +++ b/images/php/Dockerfile.cli @@ -1,4 +1,4 @@ -ARG PHP_VERSION=8.4 +ARG PHP_VERSION=8.3 FROM php:${PHP_VERSION}-cli-alpine AS php-cli FROM ghcr.io/skpr/mtk:v2.1.1 AS mtk FROM uselagoon/php-${PHP_VERSION}-cli-drupal:latest diff --git a/images/php/Dockerfile.fpm b/images/php/Dockerfile.fpm index cd73a03f..845f29a8 100644 --- a/images/php/Dockerfile.fpm +++ b/images/php/Dockerfile.fpm @@ -1,4 +1,4 @@ -ARG PHP_VERSION=8.4 +ARG PHP_VERSION=8.3 FROM ghcr.io/dpc-sdp/bay/php-fpm-exporter:6.x AS php-fpm-exporter FROM uselagoon/php-${PHP_VERSION}-fpm:latest From 507cabfe16139a396f866a8edb934efa9b7adca0 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Thu, 14 Aug 2025 08:30:41 +1000 Subject: [PATCH 11/29] [SDPPE-127] Pinned version of EE community.general collection (#380) --- images/awx-ee/requirements.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/awx-ee/requirements.yml b/images/awx-ee/requirements.yml index 63aa9f31..f66a4eb7 100644 --- a/images/awx-ee/requirements.yml +++ b/images/awx-ee/requirements.yml @@ -3,7 +3,8 @@ collections: - ansible.posix - ansible.utils - awx.awx - - community.general + - name: community.general + version: "11.1.2" - kubernetes.core - name: lagoon.api source: https://github.com/salsadigitalauorg/lagoon_ansible_collection.git From 185e0006dec2a2a8c64ba5edf70126e3d36d6c3a Mon Sep 17 00:00:00 2001 From: rcb Date: Thu, 14 Aug 2025 17:34:08 +1000 Subject: [PATCH 12/29] SDPPE-124: Update branch-sync workflow resilient to cherry-pick conflicts --- .github/workflows/branch-sync.yml | 85 ++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 8 deletions(-) diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml index 5386428e..662147ef 100644 --- a/.github/workflows/branch-sync.yml +++ b/.github/workflows/branch-sync.yml @@ -25,25 +25,90 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" - name: Prepare sync branch + shell: bash run: | + set -euo pipefail + # Fetch all branches git fetch origin # Create a new sync branch from 7.x (target) git checkout -b temp-sync-branch origin/7.x - # Find all 6.x commits not on 7.x and cherry-pick them - COMMITS=$(git log --reverse --pretty=format:"%H" origin/7.x..origin/6.x) - if [ -z "$COMMITS" ]; then + # Collect 6.x commits not on 7.x (oldest first), excluding "sync: skip" + mapfile -t COMMITS < <(git log --reverse --pretty=format:'%H %s' origin/7.x..origin/6.x | grep -v -E 'sync:\s*skip' || true) + + if [ ${#COMMITS[@]} -eq 0 ]; then echo "NO_CHANGES=true" >> "${GITHUB_ENV}" exit 0 fi - for COMMIT in $COMMITS; do - git log -1 --format=%s "${COMMIT}" - git cherry-pick "${COMMIT}" || { echo "Cherry-pick failed"; exit 1; } + # Paths that 7.x "owns" and should NOT be overridden by 6.x syncs. + # Start conservative; expand as you learn which ones regularly conflict. + OWNED_PATHS_REGEX='^(images/ci-builder/Dockerfile|images/php/Dockerfile\.fpm|images/php/Dockerfile\.cli|images/awx-ee/execution-environment\.yml)$' + + HAD_SKIPS=false + APPLIED_ANY=false + + for entry in "${COMMITS[@]}"; do + sha="${entry%% *}" + subj="${entry#* }" + + echo "==> Cherry-picking ${sha}: ${subj}" + + # Try to cherry-pick without committing so we can fix paths before finalizing + if ! git cherry-pick -n "${sha}"; then + echo "Conflict detected; attempting auto-resolution for owned paths..." + + # List conflicted files + mapfile -t CONFLICTS < <(git diff --name-only --diff-filter=U || true) + + if [ ${#CONFLICTS[@]} -gt 0 ]; then + # For owned/conflict-prone files, keep 7.x (current branch) version + for f in "${CONFLICTS[@]}"; do + if [[ "$f" =~ $OWNED_PATHS_REGEX ]]; then + echo "Keeping 7.x version for: $f" + git restore --source=HEAD -- "$f" || true + fi + done + git add -A || true + fi + + # Try to continue after auto-fix. If it still fails, skip this commit. + if ! git -c core.editor=true cherry-pick --continue; then + echo "Unable to auto-resolve ${sha}; skipping." + HAD_SKIPS=true + git cherry-pick --abort + continue + fi + fi + + # Even when no conflict occurred, ensure owned files stay as 7.x + TO_RESTORE="$(git diff --name-only --cached | grep -E "$OWNED_PATHS_REGEX" || true)" + if [ -n "$TO_RESTORE" ]; then + git restore --source=HEAD -- $TO_RESTORE || true + git add -A || true + fi + + # Finalize the cherry-pick commit (re-use picked message) + if git commit --no-edit; then + APPLIED_ANY=true + else + # If nothing to commit (e.g., all changes reverted), clean up and move on + git reset --hard + fi done + # If nothing actually changed vs origin/7.x, don't open a PR + if ! $APPLIED_ANY || git diff --quiet origin/7.x..HEAD; then + echo "NO_CHANGES=true" >> "${GITHUB_ENV}" + fi + + # Surface whether we had skips (optional – for PR labelling/message) + if $HAD_SKIPS; then + echo "HAD_SKIPS=true" >> "${GITHUB_ENV}" + fi + - name: Create a pull request from sync branch to 7.x if: env.NO_CHANGES != 'true' uses: peter-evans/create-pull-request@v7 @@ -54,9 +119,13 @@ jobs: title: "Sync changes from 6.x" body: | This pull request syncs relevant changes from the `6.x` branch into the `7.x` branch. - + + - Commits with `sync: skip` in their message were excluded automatically. + - If a commit conflicted only in known 7.x-owned files (Dockerfiles or AWX EE defs), those files were kept from `7.x`. + - Commits that still could not be auto-resolved were skipped (if any, the label below will be present). + This PR was automatically generated by the CI workflow. - labels: sync, automated + labels: ${{ env.HAD_SKIPS == 'true' && 'sync, automated, sync-skipped-commits' || 'sync, automated' }} draft: false sign-commits: true From a9f8cc9db3e5f387c94e7a209eff921330b76034 Mon Sep 17 00:00:00 2001 From: rcb Date: Thu, 14 Aug 2025 18:09:00 +1000 Subject: [PATCH 13/29] SDPPE-124: Update branch-sync workflow resilient to cherry-pick conflicts --- .github/workflows/branch-sync.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml index 662147ef..40a3e4db 100644 --- a/.github/workflows/branch-sync.yml +++ b/.github/workflows/branch-sync.yml @@ -44,7 +44,6 @@ jobs: fi # Paths that 7.x "owns" and should NOT be overridden by 6.x syncs. - # Start conservative; expand as you learn which ones regularly conflict. OWNED_PATHS_REGEX='^(images/ci-builder/Dockerfile|images/php/Dockerfile\.fpm|images/php/Dockerfile\.cli|images/awx-ee/execution-environment\.yml)$' HAD_SKIPS=false From 2619b662fc52870f4401252da4c801026bcc5e93 Mon Sep 17 00:00:00 2001 From: rcb Date: Fri, 15 Aug 2025 11:10:51 +1000 Subject: [PATCH 14/29] SDPPE-124: use merge strategy to prefer 7.x on conflicts --- .github/workflows/branch-sync.yml | 87 ++++++------------------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml index 40a3e4db..3e57ee4c 100644 --- a/.github/workflows/branch-sync.yml +++ b/.github/workflows/branch-sync.yml @@ -29,83 +29,30 @@ jobs: run: | set -euo pipefail - # Fetch all branches git fetch origin # Create a new sync branch from 7.x (target) git checkout -b temp-sync-branch origin/7.x - # Collect 6.x commits not on 7.x (oldest first), excluding "sync: skip" - mapfile -t COMMITS < <(git log --reverse --pretty=format:'%H %s' origin/7.x..origin/6.x | grep -v -E 'sync:\s*skip' || true) + HAD_CONFLICTS=false - if [ ${#COMMITS[@]} -eq 0 ]; then - echo "NO_CHANGES=true" >> "${GITHUB_ENV}" - exit 0 + # Merge 6.x into temp branch, preferring 7.x on conflicts + if ! git merge --no-ff -X ours origin/6.x -m "Sync 6.x into 7.x (prefer 7.x on conflicts)"; then + echo "Merge reported conflicts; force-resolving all to 7.x (ours)..." + HAD_CONFLICTS=true + # Accept 'ours' version for all remaining conflicts + git checkout --ours . + git add -A + git commit -m "Sync 6.x into 7.x (auto-resolved conflicts preferring 7.x)" fi - # Paths that 7.x "owns" and should NOT be overridden by 6.x syncs. - OWNED_PATHS_REGEX='^(images/ci-builder/Dockerfile|images/php/Dockerfile\.fpm|images/php/Dockerfile\.cli|images/awx-ee/execution-environment\.yml)$' - - HAD_SKIPS=false - APPLIED_ANY=false - - for entry in "${COMMITS[@]}"; do - sha="${entry%% *}" - subj="${entry#* }" - - echo "==> Cherry-picking ${sha}: ${subj}" - - # Try to cherry-pick without committing so we can fix paths before finalizing - if ! git cherry-pick -n "${sha}"; then - echo "Conflict detected; attempting auto-resolution for owned paths..." - - # List conflicted files - mapfile -t CONFLICTS < <(git diff --name-only --diff-filter=U || true) - - if [ ${#CONFLICTS[@]} -gt 0 ]; then - # For owned/conflict-prone files, keep 7.x (current branch) version - for f in "${CONFLICTS[@]}"; do - if [[ "$f" =~ $OWNED_PATHS_REGEX ]]; then - echo "Keeping 7.x version for: $f" - git restore --source=HEAD -- "$f" || true - fi - done - git add -A || true - fi - - # Try to continue after auto-fix. If it still fails, skip this commit. - if ! git -c core.editor=true cherry-pick --continue; then - echo "Unable to auto-resolve ${sha}; skipping." - HAD_SKIPS=true - git cherry-pick --abort - continue - fi - fi - - # Even when no conflict occurred, ensure owned files stay as 7.x - TO_RESTORE="$(git diff --name-only --cached | grep -E "$OWNED_PATHS_REGEX" || true)" - if [ -n "$TO_RESTORE" ]; then - git restore --source=HEAD -- $TO_RESTORE || true - git add -A || true - fi - - # Finalize the cherry-pick commit (re-use picked message) - if git commit --no-edit; then - APPLIED_ANY=true - else - # If nothing to commit (e.g., all changes reverted), clean up and move on - git reset --hard - fi - done - - # If nothing actually changed vs origin/7.x, don't open a PR - if ! $APPLIED_ANY || git diff --quiet origin/7.x..HEAD; then + # If the merge results in no net change, don't open a PR + if git diff --quiet origin/7.x..HEAD; then echo "NO_CHANGES=true" >> "${GITHUB_ENV}" fi - # Surface whether we had skips (optional – for PR labelling/message) - if $HAD_SKIPS; then - echo "HAD_SKIPS=true" >> "${GITHUB_ENV}" + if $HAD_CONFLICTS; then + echo "HAD_CONFLICTS=true" >> "${GITHUB_ENV}" fi - name: Create a pull request from sync branch to 7.x @@ -119,12 +66,12 @@ jobs: body: | This pull request syncs relevant changes from the `6.x` branch into the `7.x` branch. - - Commits with `sync: skip` in their message were excluded automatically. - - If a commit conflicted only in known 7.x-owned files (Dockerfiles or AWX EE defs), those files were kept from `7.x`. - - Commits that still could not be auto-resolved were skipped (if any, the label below will be present). + - Merge performed with `-X ours` to prefer `7.x` on conflicts. + - Any remaining conflicts were auto-resolved to `ours` to keep CI green; reviewers can adjust in this PR if needed. + - Non-conflicting changes from `6.x` are preserved. This PR was automatically generated by the CI workflow. - labels: ${{ env.HAD_SKIPS == 'true' && 'sync, automated, sync-skipped-commits' || 'sync, automated' }} + labels: ${{ env.HAD_CONFLICTS == 'true' && 'sync, automated, merge-conflicts-resolved' || 'sync, automated' }} draft: false sign-commits: true From f5fc9eeacd2270f755fd1476489914b6c4c385aa Mon Sep 17 00:00:00 2001 From: rcb Date: Thu, 14 Aug 2025 16:14:32 +1000 Subject: [PATCH 15/29] SDPPE-123: Enable build and scan workflows for 7.x branch --- .github/workflows/build-deploy.yml | 37 +++++++++++-------- .../vulnerability-scan-schedule-7x.yml | 27 ++++++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/vulnerability-scan-schedule-7x.yml diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 7dfe7113..3952c857 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -2,13 +2,9 @@ name: build-deploy-bay-images run-name: Build and deploy Bay images on: pull_request: - types: - - closed - - opened - - synchronize + types: [closed, opened, synchronize] issue_comment: - types: - - created + types: [created] schedule: - cron: '23 20 * * 0' workflow_dispatch: @@ -17,11 +13,11 @@ env: REGISTRY: ghcr.io jobs: buildx: - if: |- + if: | github.event.pull_request.merged == true || contains(fromJson('["schedule", "workflow_dispatch"]'), github.event_name) || - github.event_name == 'pull_request' && startsWith(github.head_ref,'build/') || - ( github.event.issue.pull_request && contains(github.event.comment.body, '/build') ) + (github.event_name == 'pull_request' && startsWith(github.head_ref,'build/')) || + (github.event.issue.pull_request && contains(github.event.comment.body, '/build')) runs-on: ubuntu-latest strategy: matrix: @@ -33,19 +29,19 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - uses: actions/checkout@v3 - if: |- + - uses: actions/checkout@v4 + if: | github.event.pull_request.merged == true || contains(fromJson('["schedule", "workflow_dispatch"]'), github.event_name) || - github.event_name == 'pull_request' && startsWith(github.head_ref,'build/') + (github.event_name == 'pull_request' && startsWith(github.head_ref,'build/')) - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ( github.event.issue.pull_request && contains(github.event.comment.body, '/build') ) with: ref: refs/pull/${{ github.event.issue.number }}/head - name: Login to registry ${{ env.REGISTRY }} - uses: docker/login-action@v2.2.0 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ secrets.REGISTRY_USER }} @@ -53,15 +49,24 @@ jobs: - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4.6.0 + uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.images }} tags: | + # PR images (not merged): pr- type=ref,event=pr,enable=${{ github.event.pull_request.merged == false }} + # Tag with the PR base branch name on merge (e.g., 6.x or 7.x) type=raw,value=${{ github.event.pull_request.base.ref }},enable=${{ github.event.pull_request.merged == true }} + # Manual/scheduled runs tag by branch ref (when the run targets 6.x or 7.x) type=ref,event=branch,enable=${{ contains(fromJson('["schedule", "workflow_dispatch"]'), github.event_name) }} + # Comment-triggered builds: pr- type=raw,event=default,value=pr-${{ github.event.issue.number }},enable=${{ github.event.issue.pull_request != null }} + # --- explicit safety rails for branch tags --- + type=raw,value=6.x,enable=${{ github.ref == 'refs/heads/6.x' || github.event.pull_request.base.ref == '6.x' }} + type=raw,value=7.x,enable=${{ github.ref == 'refs/heads/7.x' || github.event.pull_request.base.ref == '7.x' }} + # Always add an immutable sha tag + type=raw,value=sha-${{ github.sha }} labels: | maintainer=Digital Victoria repository=${{ github.repositoryUrl }} @@ -88,7 +93,7 @@ jobs: retention-days: 1 - name: Build and push the images - uses: docker/bake-action@v3.1.0 + uses: docker/bake-action@v5 with: push: true files: | diff --git a/.github/workflows/vulnerability-scan-schedule-7x.yml b/.github/workflows/vulnerability-scan-schedule-7x.yml new file mode 100644 index 00000000..4cdc0aa3 --- /dev/null +++ b/.github/workflows/vulnerability-scan-schedule-7x.yml @@ -0,0 +1,27 @@ +name: vulnerability-scan-schedule-7.x +run-name: Scheduled CVE vulnerability scan of 7.x published images. + +env: + REGISTRY: ghcr.io + +on: + schedule: + - cron: '2 22 * * 3' # same schedule as 6.x; + workflow_dispatch: + inputs: + summary: + description: 'Summary of the scheduled scan.' + required: false + default: 'Trivy CVE scan of 7.x published images.' + tag: + description: 'Tag to scan.' + required: false + default: '7.x' + +jobs: + vulnerability-scan-schedule: + name: Scan for vulnerabilities on 7.x images + uses: "dpc-sdp/bay/.github/workflows/vulnerability-scan.yml@7.x" + with: + tag: "7.x" + summary: "Trivy CVE scan of 7.x published images." From 729f52547a86744ba456d51aa0e5ad586f8ce21f Mon Sep 17 00:00:00 2001 From: rcb Date: Fri, 15 Aug 2025 13:39:51 +1000 Subject: [PATCH 16/29] SDPPE-123: Align tagging with support for 5.x and set 7x schedule scan after 6x is executed --- .github/workflows/build-deploy.yml | 15 ++++++++++----- .../workflows/vulnerability-scan-schedule-7x.yml | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 3952c857..109e51b0 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -2,9 +2,13 @@ name: build-deploy-bay-images run-name: Build and deploy Bay images on: pull_request: - types: [closed, opened, synchronize] + types: + - closed + - opened + - synchronize issue_comment: - types: [created] + types: + - created schedule: - cron: '23 20 * * 0' workflow_dispatch: @@ -56,13 +60,14 @@ jobs: tags: | # PR images (not merged): pr- type=ref,event=pr,enable=${{ github.event.pull_request.merged == false }} - # Tag with the PR base branch name on merge (e.g., 6.x or 7.x) + # Tag with the PR base branch name on merge type=raw,value=${{ github.event.pull_request.base.ref }},enable=${{ github.event.pull_request.merged == true }} - # Manual/scheduled runs tag by branch ref (when the run targets 6.x or 7.x) + # Manual/scheduled runs tag by branch ref (for whichever release branch the run targets) type=ref,event=branch,enable=${{ contains(fromJson('["schedule", "workflow_dispatch"]'), github.event_name) }} # Comment-triggered builds: pr- type=raw,event=default,value=pr-${{ github.event.issue.number }},enable=${{ github.event.issue.pull_request != null }} - # --- explicit safety rails for branch tags --- + # Supported release branches + type=raw,value=5.x,enable=${{ github.ref == 'refs/heads/5.x' || github.event.pull_request.base.ref == '5.x' }} type=raw,value=6.x,enable=${{ github.ref == 'refs/heads/6.x' || github.event.pull_request.base.ref == '6.x' }} type=raw,value=7.x,enable=${{ github.ref == 'refs/heads/7.x' || github.event.pull_request.base.ref == '7.x' }} # Always add an immutable sha tag diff --git a/.github/workflows/vulnerability-scan-schedule-7x.yml b/.github/workflows/vulnerability-scan-schedule-7x.yml index 4cdc0aa3..bded75fb 100644 --- a/.github/workflows/vulnerability-scan-schedule-7x.yml +++ b/.github/workflows/vulnerability-scan-schedule-7x.yml @@ -6,7 +6,7 @@ env: on: schedule: - - cron: '2 22 * * 3' # same schedule as 6.x; + - cron: '12 22 * * 3' workflow_dispatch: inputs: summary: From 733f7d7791e224d06493a6e74cddde89de9b7a05 Mon Sep 17 00:00:00 2001 From: Jayson Date: Tue, 19 Aug 2025 16:27:58 +1000 Subject: [PATCH 17/29] Update lagoon.api collection version to 2.2.0 (#384) --- images/awx-ee/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/awx-ee/requirements.yml b/images/awx-ee/requirements.yml index f66a4eb7..e84c0ed7 100644 --- a/images/awx-ee/requirements.yml +++ b/images/awx-ee/requirements.yml @@ -8,7 +8,7 @@ collections: - kubernetes.core - name: lagoon.api source: https://github.com/salsadigitalauorg/lagoon_ansible_collection.git - version: 2.1.0 + version: 2.2.0 type: git - name: section.api source: https://github.com/salsadigitalauorg/section_ansible_collection.git From 6c52786e46a5fa53414634fbea962833309e0977 Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Wed, 8 Oct 2025 09:42:07 +1100 Subject: [PATCH 18/29] [SD-1322] Prevent scheduled_transition crashes caused by missing node revisions --- images/php/mtk/drupal.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/images/php/mtk/drupal.conf b/images/php/mtk/drupal.conf index 937e906d..3607994c 100644 --- a/images/php/mtk/drupal.conf +++ b/images/php/mtk/drupal.conf @@ -63,3 +63,4 @@ nodata: - router - sessions - webform_* + - scheduled_transition* From 0f4ea22ce5de481afb532c4f5bd7ad25b54d58a4 Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 10 Oct 2025 16:57:58 +1100 Subject: [PATCH 19/29] [SDPPE-126] Added opensearch image (#379) --- gh-actions-bake.hcl | 9 ++++++++- images/elasticsearch/Dockerfile | 6 ------ images/opensearch/Dockerfile | 9 +++++++++ images/opensearch/opensearch.yml | 7 +++++++ 4 files changed, 24 insertions(+), 7 deletions(-) delete mode 100644 images/elasticsearch/Dockerfile create mode 100644 images/opensearch/Dockerfile create mode 100644 images/opensearch/opensearch.yml diff --git a/gh-actions-bake.hcl b/gh-actions-bake.hcl index 2133fa73..3831f2d6 100755 --- a/gh-actions-bake.hcl +++ b/gh-actions-bake.hcl @@ -11,9 +11,16 @@ target "ci-builder" { platforms = ["linux/amd64", "linux/arm64"] } +target "opensearch" { + inherits = ["docker-metadata-action"] + context = "${CONTEXT}/opensearch" + dockerfile = "Dockerfile" + + platforms = ["linux/amd64", "linux/arm64"] +} target "elasticsearch" { inherits = ["docker-metadata-action"] - context = "${CONTEXT}/elasticsearch" + context = "${CONTEXT}/opensearch" dockerfile = "Dockerfile" platforms = ["linux/amd64", "linux/arm64"] diff --git a/images/elasticsearch/Dockerfile b/images/elasticsearch/Dockerfile deleted file mode 100644 index b15b586e..00000000 --- a/images/elasticsearch/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM elasticsearch:8.16.1 - -ENV ES_PATH=/usr/share/elasticsearch - -RUN $ES_PATH/bin/elasticsearch-plugin install analysis-kuromoji \ - && $ES_PATH/bin/elasticsearch-plugin install analysis-icu diff --git a/images/opensearch/Dockerfile b/images/opensearch/Dockerfile new file mode 100644 index 00000000..93219ca1 --- /dev/null +++ b/images/opensearch/Dockerfile @@ -0,0 +1,9 @@ +FROM uselagoon/opensearch-2:latest + +RUN for plugin in \ + analysis-kuromoji \ + analysis-icu; do \ + /usr/share/opensearch/bin/opensearch-plugin install $plugin; \ + done + +COPY opensearch.yml /usr/share/opensearch/config \ No newline at end of file diff --git a/images/opensearch/opensearch.yml b/images/opensearch/opensearch.yml new file mode 100644 index 00000000..6ebe8cd8 --- /dev/null +++ b/images/opensearch/opensearch.yml @@ -0,0 +1,7 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +discovery.type: single-node +plugins.security.disabled: true + +path.repo: ["/usr/share/elasticsearch/data/snapshots"] \ No newline at end of file From 290249e20272a7b975955ef20af6b05e98ff02f0 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Mon, 13 Oct 2025 07:27:18 +1100 Subject: [PATCH 20/29] [SDPPE-169] Removed deprecated class from awx-ee (#386) * [SDPPE-169] Removed deprecated class from awx-ee * [SDPPE-169] Updated lagoon.api version constraint * [SDPPE-169] Cleanup whitespace --- images/awx-ee/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/awx-ee/requirements.yml b/images/awx-ee/requirements.yml index e84c0ed7..1a87fa5c 100644 --- a/images/awx-ee/requirements.yml +++ b/images/awx-ee/requirements.yml @@ -8,7 +8,7 @@ collections: - kubernetes.core - name: lagoon.api source: https://github.com/salsadigitalauorg/lagoon_ansible_collection.git - version: 2.2.0 + version: "2.2.4" type: git - name: section.api source: https://github.com/salsadigitalauorg/section_ansible_collection.git From 681d671952ff9021a032fd4773217420a95bbc7d Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Thu, 16 Oct 2025 16:39:06 +1100 Subject: [PATCH 21/29] Update aws-es-proxy go packages (#387) --- images/aws-es-proxy/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/images/aws-es-proxy/Dockerfile b/images/aws-es-proxy/Dockerfile index a2ff799d..3bf8c4c3 100644 --- a/images/aws-es-proxy/Dockerfile +++ b/images/aws-es-proxy/Dockerfile @@ -4,6 +4,7 @@ RUN apk add --no-cache git RUN git clone https://github.com/abutaha/aws-es-proxy.git /go/src/github.com/abutaha/aws-es-proxy WORKDIR /go/src/github.com/abutaha/aws-es-proxy +RUN go get -u && go mod tidy RUN CGO_ENABLED=0 GOOS=linux go build -o aws-es-proxy FROM alpine:latest From abb484e4c0a27d108d1fcc272c79b4d35f706b02 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Fri, 17 Oct 2025 15:21:03 +1100 Subject: [PATCH 22/29] [SDPPE-167] Removed install steps for Hub cli. --- images/awx-ee/execution-environment.yml | 3 --- images/ci-builder/Dockerfile | 7 ------- 2 files changed, 10 deletions(-) diff --git a/images/awx-ee/execution-environment.yml b/images/awx-ee/execution-environment.yml index b704bb66..b0f45a66 100644 --- a/images/awx-ee/execution-environment.yml +++ b/images/awx-ee/execution-environment.yml @@ -29,7 +29,6 @@ additional_build_steps: - ARG NVM_DIR="/runner/.nvm" - ARG PHP_VERSION="8.3" - ARG COMPOSER_VERSION="2.7.7" - - ARG HUB_VERSION="2.14.2" - ARG GOJQ_VERSION="0.12.17" - ARG HELM_VERSION="3.18.3" - ARG YAMLFMT_VERSION="0.17.2" @@ -67,8 +66,6 @@ additional_build_steps: - RUN curl -L "https://github.com/uselagoon/lagoon-cli/releases/download/${LAGOON_CLI_VERSION}/lagoon-cli-${LAGOON_CLI_VERSION}-linux-amd64" -o /usr/local/bin/lagoon - RUN chmod +x /usr/local/bin/lagoon - - RUN curl -L "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -o /tmp/hub && tar -xvf /tmp/hub -C /tmp && mv /tmp/hub-linux-amd64-${HUB_VERSION}/bin/hub /usr/local/bin - - RUN chmod +x /usr/local/bin/hub - RUN lagoon config feature --enable-local-dir-check false --force - RUN curl -sS "https://getcomposer.org/download/${COMPOSER_VERSION}/composer.phar" --output composer.phar - RUN chmod +x composer.phar diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index 39aba327..dafe448d 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -3,7 +3,6 @@ FROM ghcr.io/dpc-sdp/sumocli:v0.11.1 AS sumocli FROM php:8.3-cli-alpine ARG AHOY_VERSION=2.4.0 ARG GOJQ_VERSION=0.12.17 -ARG HUB_VERSION=2.14.2 ARG LAGOON_CLI_VERSION=0.32.0 ARG SHIPSHAPE_VERSION=1.0.0-alpha.1.5.1 @@ -37,12 +36,6 @@ RUN apk add --update --no-cache \ python3 \ python3-dev -## Install GitHub CLI tool. -RUN curl -sL "https://github.com/mislav/hub/releases/download/v${HUB_VERSION}/hub-$(echo ${TARGETPLATFORM:-linux/amd64} | tr '/' '-')-${HUB_VERSION}.tgz" -o /tmp/hub.tgz && \ - tar -C /tmp -xzvf /tmp/hub.tgz && \ - mv /tmp/hub-$(echo ${TARGETPLATFORM:-linux/amd64} | tr '/' '-')-${HUB_VERSION}/bin/hub /usr/local/bin && \ - chmod 755 /usr/local/bin - ## Install required PHP extensions for Drupal and python packages. RUN apk add --no-cache \ py3-flake8 \ From 9e5d08cc3454e3beaf79b13b6e19257fc2e93f6e Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Fri, 17 Oct 2025 16:54:03 +1100 Subject: [PATCH 23/29] Removed redundant opensearch config file. (#389) --- images/opensearch/Dockerfile | 2 -- images/opensearch/opensearch.yml | 7 ------- 2 files changed, 9 deletions(-) delete mode 100644 images/opensearch/opensearch.yml diff --git a/images/opensearch/Dockerfile b/images/opensearch/Dockerfile index 93219ca1..8406edc3 100644 --- a/images/opensearch/Dockerfile +++ b/images/opensearch/Dockerfile @@ -5,5 +5,3 @@ RUN for plugin in \ analysis-icu; do \ /usr/share/opensearch/bin/opensearch-plugin install $plugin; \ done - -COPY opensearch.yml /usr/share/opensearch/config \ No newline at end of file diff --git a/images/opensearch/opensearch.yml b/images/opensearch/opensearch.yml deleted file mode 100644 index 6ebe8cd8..00000000 --- a/images/opensearch/opensearch.yml +++ /dev/null @@ -1,7 +0,0 @@ -cluster.name: "docker-cluster" -network.host: 0.0.0.0 - -discovery.type: single-node -plugins.security.disabled: true - -path.repo: ["/usr/share/elasticsearch/data/snapshots"] \ No newline at end of file From ec5197aa08c1499c281d3864fef7359f2cfe942c Mon Sep 17 00:00:00 2001 From: Nick Santamaria Date: Tue, 28 Oct 2025 11:17:56 +1100 Subject: [PATCH 24/29] [SDPPE-121] Added support for BAY_OPENSEARCH_ environment variables and aws-es-proxy integration (#373) --- images/php/settings.php | 67 +++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/images/php/settings.php b/images/php/settings.php index 785dc882..abb99657 100755 --- a/images/php/settings.php +++ b/images/php/settings.php @@ -314,37 +314,60 @@ $config['clamav.settings']['mode_daemon_tcpip']['hostname'] = $clamav_host; $config['clamav.settings']['mode_daemon_tcpip']['port'] = $clamav_port; -// Configure elasticsearch connections from environment variables. -if (getenv('SEARCH_HASH') && getenv('SEARCH_URL')) { - $config['elasticsearch_connector.cluster.elasticsearch_bay']['url'] = sprintf('http://%s.%s', getenv('SEARCH_HASH'), getenv('SEARCH_URL')); +$opensearch_profile = getenv('BAY_OPENSEARCH_PROFILE') ?: 'sdp-elastic'; +if ($opensearch_profile == 'sdp-elastic') { + // Configuration for legacy sdp-elastic integration. + // @todo remove this sdp-elastic block when all applications migrated to opensearch. + if (getenv('SEARCH_HASH') && getenv('SEARCH_URL')) { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['url'] = sprintf('http://%s.%s', getenv('SEARCH_HASH'), getenv('SEARCH_URL')); + } else { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['url'] = "http://elasticsearch:9200"; + } + + if (getenv('SEARCH_INDEX')) { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['rewrite_index'] = 1; + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['index'] = [ + 'prefix' => getenv('SEARCH_INDEX'), + 'suffix' => '', + ]; + } else { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['index'] = [ + 'prefix' => 'elasticsearch_index_default_', + 'suffix' => '', + ]; + } + + if (getenv('SEARCH_AUTH_USERNAME') && getenv('SEARCH_AUTH_PASSWORD')) { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['username'] = getenv('SEARCH_AUTH_USERNAME'); + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['password'] = getenv('SEARCH_AUTH_PASSWORD'); + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['use_authentication'] = 1; + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['authentication_type'] = 'Basic'; + } else { + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['use_authentication'] = 0; + } + // Override data_pipelines url. + $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['url'] = (getenv('SEARCH_HASH') && getenv('SEARCH_URL')) ? sprintf('http://%s.%s', getenv('SEARCH_HASH'), getenv('SEARCH_URL')) : "http://elasticsearch:9200"; } else { - $config['elasticsearch_connector.cluster.elasticsearch_bay']['url'] = "http://elasticsearch:9200"; -} + // Configuration for bay opensearch integration. + + // Connect to a proxy service that handles AWS IAM auth. + $endpoint = "http://aws-es-proxy:9200"; -if (getenv('SEARCH_INDEX')) { + // Default index prefix looks like "${PROJECT}__${ENVIRONMENT}__". This can be overridden with BAY_OPENSEARCH_PREFIX. + $environment = getenv('LAGOON_ENVIRONMENT') ?: 'default'; + $index_prefix = getenv('BAY_OPENSEARCH_PREFIX') ?: sprintf('%s__%s', getenv('LAGOON_PROJECT'), $environment); + $config['elasticsearch_connector.cluster.elasticsearch_bay']['url'] = $endpoint; + $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['use_authentication'] = FALSE; $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['rewrite_index'] = 1; $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['index'] = [ - 'prefix' => getenv('SEARCH_INDEX'), - 'suffix' => '', - ]; -} else { - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['rewrite']['index'] = [ - 'prefix' => 'elasticsearch_index_default_', + 'prefix' => sprintf('%s__%s_', $index_prefix, "sapi"), 'suffix' => '', ]; + $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['url'] = $endpoint; + $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['prefix'] = sprintf('%s__pipeline_', $index_prefix); } -if (getenv('SEARCH_AUTH_USERNAME') && getenv('SEARCH_AUTH_PASSWORD')) { - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['username'] = getenv('SEARCH_AUTH_USERNAME'); - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['password'] = getenv('SEARCH_AUTH_PASSWORD'); - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['use_authentication'] = 1; - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['authentication_type'] = 'Basic'; -} else { - $config['elasticsearch_connector.cluster.elasticsearch_bay']['options']['use_authentication'] = 0; -} -// Override data_pipelines url. -$config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['url'] = (getenv('SEARCH_HASH') && getenv('SEARCH_URL')) ? sprintf('http://%s.%s', getenv('SEARCH_HASH'), getenv('SEARCH_URL')) : "http://elasticsearch:9200"; // Configure tide_logs. if (getenv('TIDE_LOGS_UDPLOG_HOST')) { From ee204a1bc15926b14913ca5005172429a8da7cdc Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Tue, 28 Oct 2025 14:18:42 +1100 Subject: [PATCH 25/29] [SD-1111] Update prefix for data pipelines --- images/php/settings.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/php/settings.php b/images/php/settings.php index abb99657..e01c7833 100755 --- a/images/php/settings.php +++ b/images/php/settings.php @@ -364,7 +364,7 @@ 'suffix' => '', ]; $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['url'] = $endpoint; - $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['prefix'] = sprintf('%s__pipeline_', $index_prefix); + $config['data_pipelines.dataset_destination.sdp_elasticsearch']['destinationSettings']['prefix'] = sprintf('%s__sdp_data_pipelines_', $index_prefix); } From f011cd41ba21eb067cad6df2074a3d03f637541e Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Mon, 17 Nov 2025 12:19:08 +1100 Subject: [PATCH 26/29] Backport GHA 7.x branch sync updates. --- .github/workflows/branch-sync.yml | 37 +++++++++---------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml index 3e57ee4c..5386428e 100644 --- a/.github/workflows/branch-sync.yml +++ b/.github/workflows/branch-sync.yml @@ -25,35 +25,24 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" - name: Prepare sync branch - shell: bash run: | - set -euo pipefail - + # Fetch all branches git fetch origin # Create a new sync branch from 7.x (target) git checkout -b temp-sync-branch origin/7.x - HAD_CONFLICTS=false - - # Merge 6.x into temp branch, preferring 7.x on conflicts - if ! git merge --no-ff -X ours origin/6.x -m "Sync 6.x into 7.x (prefer 7.x on conflicts)"; then - echo "Merge reported conflicts; force-resolving all to 7.x (ours)..." - HAD_CONFLICTS=true - # Accept 'ours' version for all remaining conflicts - git checkout --ours . - git add -A - git commit -m "Sync 6.x into 7.x (auto-resolved conflicts preferring 7.x)" - fi - - # If the merge results in no net change, don't open a PR - if git diff --quiet origin/7.x..HEAD; then + # Find all 6.x commits not on 7.x and cherry-pick them + COMMITS=$(git log --reverse --pretty=format:"%H" origin/7.x..origin/6.x) + if [ -z "$COMMITS" ]; then echo "NO_CHANGES=true" >> "${GITHUB_ENV}" + exit 0 fi - if $HAD_CONFLICTS; then - echo "HAD_CONFLICTS=true" >> "${GITHUB_ENV}" - fi + for COMMIT in $COMMITS; do + git log -1 --format=%s "${COMMIT}" + git cherry-pick "${COMMIT}" || { echo "Cherry-pick failed"; exit 1; } + done - name: Create a pull request from sync branch to 7.x if: env.NO_CHANGES != 'true' @@ -65,13 +54,9 @@ jobs: title: "Sync changes from 6.x" body: | This pull request syncs relevant changes from the `6.x` branch into the `7.x` branch. - - - Merge performed with `-X ours` to prefer `7.x` on conflicts. - - Any remaining conflicts were auto-resolved to `ours` to keep CI green; reviewers can adjust in this PR if needed. - - Non-conflicting changes from `6.x` are preserved. - + This PR was automatically generated by the CI workflow. - labels: ${{ env.HAD_CONFLICTS == 'true' && 'sync, automated, merge-conflicts-resolved' || 'sync, automated' }} + labels: sync, automated draft: false sign-commits: true From 2c1d57e826b594650db264bbe94e9fd93de02123 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Mon, 17 Nov 2025 13:45:04 +1100 Subject: [PATCH 27/29] Revert "Backport GHA 7.x branch sync updates." This reverts commit f011cd41ba21eb067cad6df2074a3d03f637541e. --- .github/workflows/branch-sync.yml | 37 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/branch-sync.yml b/.github/workflows/branch-sync.yml index 5386428e..3e57ee4c 100644 --- a/.github/workflows/branch-sync.yml +++ b/.github/workflows/branch-sync.yml @@ -25,24 +25,35 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" - name: Prepare sync branch + shell: bash run: | - # Fetch all branches + set -euo pipefail + git fetch origin # Create a new sync branch from 7.x (target) git checkout -b temp-sync-branch origin/7.x - # Find all 6.x commits not on 7.x and cherry-pick them - COMMITS=$(git log --reverse --pretty=format:"%H" origin/7.x..origin/6.x) - if [ -z "$COMMITS" ]; then + HAD_CONFLICTS=false + + # Merge 6.x into temp branch, preferring 7.x on conflicts + if ! git merge --no-ff -X ours origin/6.x -m "Sync 6.x into 7.x (prefer 7.x on conflicts)"; then + echo "Merge reported conflicts; force-resolving all to 7.x (ours)..." + HAD_CONFLICTS=true + # Accept 'ours' version for all remaining conflicts + git checkout --ours . + git add -A + git commit -m "Sync 6.x into 7.x (auto-resolved conflicts preferring 7.x)" + fi + + # If the merge results in no net change, don't open a PR + if git diff --quiet origin/7.x..HEAD; then echo "NO_CHANGES=true" >> "${GITHUB_ENV}" - exit 0 fi - for COMMIT in $COMMITS; do - git log -1 --format=%s "${COMMIT}" - git cherry-pick "${COMMIT}" || { echo "Cherry-pick failed"; exit 1; } - done + if $HAD_CONFLICTS; then + echo "HAD_CONFLICTS=true" >> "${GITHUB_ENV}" + fi - name: Create a pull request from sync branch to 7.x if: env.NO_CHANGES != 'true' @@ -54,9 +65,13 @@ jobs: title: "Sync changes from 6.x" body: | This pull request syncs relevant changes from the `6.x` branch into the `7.x` branch. - + + - Merge performed with `-X ours` to prefer `7.x` on conflicts. + - Any remaining conflicts were auto-resolved to `ours` to keep CI green; reviewers can adjust in this PR if needed. + - Non-conflicting changes from `6.x` are preserved. + This PR was automatically generated by the CI workflow. - labels: sync, automated + labels: ${{ env.HAD_CONFLICTS == 'true' && 'sync, automated, merge-conflicts-resolved' || 'sync, automated' }} draft: false sign-commits: true From f29fa488d49e2ba34ff0c97469d43734310625bc Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Tue, 18 Nov 2025 17:52:32 +1100 Subject: [PATCH 28/29] [SDPPE-167] Refactored gh cli install. --- images/ci-builder/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index dafe448d..07a1c3bd 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -25,6 +25,7 @@ RUN apk add --update --no-cache \ docker \ docker-compose \ git \ + github-cli libffi-dev \ musl-dev \ ncurses \ From 29ddf433c0db0ed8cabf0016cc016a88e6c00f83 Mon Sep 17 00:00:00 2001 From: Guy Owen Date: Tue, 18 Nov 2025 17:57:17 +1100 Subject: [PATCH 29/29] [SDPPE-167] Fixed syntax error. --- images/ci-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/ci-builder/Dockerfile b/images/ci-builder/Dockerfile index 07a1c3bd..ee4cc3a6 100644 --- a/images/ci-builder/Dockerfile +++ b/images/ci-builder/Dockerfile @@ -25,7 +25,7 @@ RUN apk add --update --no-cache \ docker \ docker-compose \ git \ - github-cli + github-cli \ libffi-dev \ musl-dev \ ncurses \