From 4acd8d0d8266d9b1763e38f50a951ea8fa86c7d7 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 12:59:07 +1100 Subject: [PATCH 01/12] Add new GATK image built from nightly snapshot 2026-03-01 --- images/gatk_nightly/Dockerfile | 61 +++++++++++++++++++ .../copy_gatk_nightly_snapshot_2026-03-01.sh | 8 +++ 2 files changed, 69 insertions(+) create mode 100644 images/gatk_nightly/Dockerfile create mode 100644 scripts/copy_gatk_nightly_snapshot_2026-03-01.sh diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile new file mode 100644 index 00000000..9f80e363 --- /dev/null +++ b/images/gatk_nightly/Dockerfile @@ -0,0 +1,61 @@ +ARG VERSION=4.6.2.0 + +FROM gatk:4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT AS base + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + apt-transport-https \ + bzip2 \ + ca-certificates \ + curl \ + git \ + gnupg \ + jq \ + libgomp1 \ + procps \ + rsync \ + software-properties-common \ + unzip \ + wget \ + zip && \ + rm -r /var/lib/apt/lists/* && \ + rm -r /var/cache/apt/* + +# build bcftools from source +FROM base AS compiler + +ARG BCFTOOLS_VERSION=1.22 +RUN apt-get update && apt-get install -y \ + bzip2 \ + gcc \ + libbz2-dev \ + libcurl4-openssl-dev \ + liblzma-dev \ + libssl-dev \ + make \ + wget \ + zlib1g-dev && \ + # Install bcftools + wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ + tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ + cd bcftools-${BCFTOOLS_VERSION} && \ + ./configure --enable-libcurl --enable-s3 --enable-gcs && \ + make && \ + strip bcftools plugins/*.so && \ + make DESTDIR=/bcftools_install install && \ + make -C htslib-$BCFTOOLS_VERSION DESTDIR=/bcftools_install install + + +FROM base + +COPY --from=compiler /bcftools_install/usr/local/bin/* /usr/local/bin/ +COPY --from=compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/ + +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \ + tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + wget -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ + apt-get update -y && \ + apt-get install google-cloud-cli --no-install-recommends -y && \ + rm -r /var/lib/apt/lists/* && \ + rm -r /var/cache/apt/* diff --git a/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh b/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh new file mode 100644 index 00000000..264a3350 --- /dev/null +++ b/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -ex +SOURCE_IMAGE="docker://broadinstitute/gatk-nightly:2026-03-01-4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT" +IMAGE_NAME="gatk" +IMAGE_TAG="4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT" + +gcloud auth configure-docker australia-southeast1-docker.pkg.dev +skopeo copy ${SOURCE_IMAGE} docker://australia-southeast1-docker.pkg.dev/cpg-common/images/${IMAGE_NAME}:${IMAGE_TAG} From db3fe3fbb92ca604cc687529eb72114548fed4b4 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:00:06 +1100 Subject: [PATCH 02/12] rm extra file --- scripts/copy_gatk_nightly_snapshot_2026-03-01.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 scripts/copy_gatk_nightly_snapshot_2026-03-01.sh diff --git a/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh b/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh deleted file mode 100644 index 264a3350..00000000 --- a/scripts/copy_gatk_nightly_snapshot_2026-03-01.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -ex -SOURCE_IMAGE="docker://broadinstitute/gatk-nightly:2026-03-01-4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT" -IMAGE_NAME="gatk" -IMAGE_TAG="4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT" - -gcloud auth configure-docker australia-southeast1-docker.pkg.dev -skopeo copy ${SOURCE_IMAGE} docker://australia-southeast1-docker.pkg.dev/cpg-common/images/${IMAGE_NAME}:${IMAGE_TAG} From deb3ea878cb0690bdd46c554ceb69f8317105a6a Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:02:49 +1100 Subject: [PATCH 03/12] Add cpg-common artifact repo to FROM statement --- images/gatk_nightly/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index 9f80e363..6b439738 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -1,6 +1,6 @@ ARG VERSION=4.6.2.0 -FROM gatk:4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT AS base +FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT AS base RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ From e2a3954bce327e97d81547e0b63d8fe0e27f2827 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:10:01 +1100 Subject: [PATCH 04/12] Build bcftools with zlib-devel not zlib1g-dev --- images/gatk_nightly/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index 6b439738..d717022f 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y \ libssl-dev \ make \ wget \ - zlib1g-dev && \ + zlib-devel && \ # Install bcftools wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ From c7abdeab3290628baefb11e9ab9fce1ae3eae941 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:17:23 +1100 Subject: [PATCH 05/12] Revert change to zlib dev install --- images/gatk_nightly/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index d717022f..6b439738 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y \ libssl-dev \ make \ wget \ - zlib-devel && \ + zlib1g-dev && \ # Install bcftools wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ From 84c658bfc5908b878442f1d16409f14ba0e34df5 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:25:27 +1100 Subject: [PATCH 06/12] Unset path variables in compiler stage --- images/gatk_nightly/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index 6b439738..ff15af18 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -39,11 +39,15 @@ RUN apt-get update && apt-get install -y \ wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ cd bcftools-${BCFTOOLS_VERSION} && \ + # UNSET conflicting paths so the compiler looks in /usr/include + unset CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH LIBRARY_PATH && \ ./configure --enable-libcurl --enable-s3 --enable-gcs && \ make && \ strip bcftools plugins/*.so && \ make DESTDIR=/bcftools_install install && \ - make -C htslib-$BCFTOOLS_VERSION DESTDIR=/bcftools_install install + # The internal htslib directory name in the release tarball + # is usually just 'htslib' or 'htslib-' + make -C htslib-${BCFTOOLS_VERSION} DESTDIR=/bcftools_install install FROM base From 8289a3a9675b27632cc357aca1536f277e1da574 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 13:36:58 +1100 Subject: [PATCH 07/12] Reset path to exclude conda --- images/gatk_nightly/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index ff15af18..423515dd 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -39,14 +39,15 @@ RUN apt-get update && apt-get install -y \ wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ cd bcftools-${BCFTOOLS_VERSION} && \ - # UNSET conflicting paths so the compiler looks in /usr/include + # Reset PATH to exclude Conda, and unset the include variables + export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ unset CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH LIBRARY_PATH && \ - ./configure --enable-libcurl --enable-s3 --enable-gcs && \ + # Run configure with explicit help from pkg-config + PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-libcurl --enable-s3 --enable-gcs && \ make && \ strip bcftools plugins/*.so && \ make DESTDIR=/bcftools_install install && \ - # The internal htslib directory name in the release tarball - # is usually just 'htslib' or 'htslib-' + # Building HTSlib explicitly to ensure headers are moved make -C htslib-${BCFTOOLS_VERSION} DESTDIR=/bcftools_install install From 5700d75009f1f839861b9bafbb17c7bd6a254a55 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 14:04:29 +1100 Subject: [PATCH 08/12] Rewrite gcloud-sdk install command --- images/gatk_nightly/Dockerfile | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index 423515dd..b71d274e 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -56,11 +56,23 @@ FROM base COPY --from=compiler /bcftools_install/usr/local/bin/* /usr/local/bin/ COPY --from=compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/ -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \ - tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ +# Install Google Cloud CLI and runtime libraries for BCFtools +RUN apt-get update && \ + # 1. Add Google Cloud repo + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + > /etc/apt/sources.list.d/google-cloud-sdk.list && \ + # 2. Add GPG key with --batch to avoid TTY errors wget -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ - apt-get update -y && \ - apt-get install google-cloud-cli --no-install-recommends -y && \ - rm -r /var/lib/apt/lists/* && \ - rm -r /var/cache/apt/* + gpg --batch --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ + # 3. Install CLI and runtime libs + apt-get update && \ + apt-get install -y --no-install-recommends \ + google-cloud-cli \ + libcurl4 \ + libgomp1 \ + zlib1g \ + libbz2-1.0 \ + liblzma5 && \ + # 4. Clean up to reduce image size + apt-get clean && \ + rm -rf /var/lib/apt/lists/* From 0f67fa3facf8fdbedb9f280cb316a9d1ca20a6a6 Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 14:13:49 +1100 Subject: [PATCH 09/12] Use curl instead of wget for google-cloud sdk --- images/gatk_nightly/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index b71d274e..b9a25a6b 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -58,13 +58,15 @@ COPY --from=compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/l # Install Google Cloud CLI and runtime libraries for BCFtools RUN apt-get update && \ - # 1. Add Google Cloud repo + # 1. Ensure the keyrings directory exists + mkdir -p /usr/share/keyrings && \ + # 2. Add Google Cloud repo echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - # 2. Add GPG key with --batch to avoid TTY errors - wget -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ - gpg --batch --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ - # 3. Install CLI and runtime libs + # 3. Use curl + gpg with --yes (to overwrite) and --batch + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + gpg --batch --yes --dearmor -o /usr/share/keyrings/cloud.google.gpg && \ + # 4. Final update and install apt-get update && \ apt-get install -y --no-install-recommends \ google-cloud-cli \ @@ -73,6 +75,5 @@ RUN apt-get update && \ zlib1g \ libbz2-1.0 \ liblzma5 && \ - # 4. Clean up to reduce image size apt-get clean && \ rm -rf /var/lib/apt/lists/* From 4a6a431f657d33296b66d8fece410253ef9a96ee Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 15:19:47 +1100 Subject: [PATCH 10/12] Refactor the whole Dockerfile with cleaned up layers and comments --- images/gatk_nightly/Dockerfile | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index b9a25a6b..dc02d1f0 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -1,9 +1,13 @@ -ARG VERSION=4.6.2.0 +# Build from the Nightly Snapshot of GATK from 2026-03-01 +ARG VERSION=4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT -FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:4.6.2.0-21-ge8c49f600-NIGHTLY-SNAPSHOT AS base +# Stage 1: Base image with common dependencies +FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:${VERSION} AS base + +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + apt-get install --no-install-recommends -y \ apt-transport-https \ bzip2 \ ca-certificates \ @@ -18,40 +22,37 @@ RUN apt-get update && \ unzip \ wget \ zip && \ - rm -r /var/lib/apt/lists/* && \ - rm -r /var/cache/apt/* + rm -rf /var/lib/apt/lists/* -# build bcftools from source +# Stage 2: Compiler stage (Build BCFtools) FROM base AS compiler ARG BCFTOOLS_VERSION=1.22 RUN apt-get update && apt-get install -y \ - bzip2 \ gcc \ libbz2-dev \ libcurl4-openssl-dev \ liblzma-dev \ libssl-dev \ make \ - wget \ + pkg-config \ zlib1g-dev && \ - # Install bcftools wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \ cd bcftools-${BCFTOOLS_VERSION} && \ - # Reset PATH to exclude Conda, and unset the include variables + # Neutralize Conda paths for clean compilation export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && \ unset CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH LIBRARY_PATH && \ - # Run configure with explicit help from pkg-config PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig ./configure --enable-libcurl --enable-s3 --enable-gcs && \ make && \ strip bcftools plugins/*.so && \ - make DESTDIR=/bcftools_install install && \ - # Building HTSlib explicitly to ensure headers are moved make -C htslib-${BCFTOOLS_VERSION} DESTDIR=/bcftools_install install +# Stage 3: Final Runtime Image +FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:${VERSION} -FROM base +# Metadata for better maintainability +LABEL description="GATK with BCFtools and Google Cloud CLI" COPY --from=compiler /bcftools_install/usr/local/bin/* /usr/local/bin/ COPY --from=compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/ From a8ff669b1c3830b1194402f87458f8bed48634fd Mon Sep 17 00:00:00 2001 From: EddieLF Date: Tue, 3 Mar 2026 15:31:41 +1100 Subject: [PATCH 11/12] Update COPY command to copy the whole /usr/local --- images/gatk_nightly/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index dc02d1f0..7f12e823 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -54,8 +54,8 @@ FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:${VERSION} # Metadata for better maintainability LABEL description="GATK with BCFtools and Google Cloud CLI" -COPY --from=compiler /bcftools_install/usr/local/bin/* /usr/local/bin/ -COPY --from=compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/ +# Copy everything installed under /usr/local in one go +COPY --from=compiler /bcftools_install/usr/local/ /usr/local/ # Install Google Cloud CLI and runtime libraries for BCFtools RUN apt-get update && \ From 47a2d1730498cd29e28dfb0529c7b7bf465a54b7 Mon Sep 17 00:00:00 2001 From: EddieLF <34049565+EddieLF@users.noreply.github.com> Date: Tue, 3 Mar 2026 16:11:00 +1100 Subject: [PATCH 12/12] Update images/gatk_nightly/Dockerfile Build final stage from base Co-authored-by: Matt Welland --- images/gatk_nightly/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/gatk_nightly/Dockerfile b/images/gatk_nightly/Dockerfile index 7f12e823..aa272ce7 100644 --- a/images/gatk_nightly/Dockerfile +++ b/images/gatk_nightly/Dockerfile @@ -49,7 +49,7 @@ RUN apt-get update && apt-get install -y \ make -C htslib-${BCFTOOLS_VERSION} DESTDIR=/bcftools_install install # Stage 3: Final Runtime Image -FROM australia-southeast1-docker.pkg.dev/cpg-common/images/gatk:${VERSION} +FROM base # Metadata for better maintainability LABEL description="GATK with BCFtools and Google Cloud CLI"