diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b850f59..2654e41 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,10 @@ name: CI -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: + - "main" + pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -24,12 +29,12 @@ jobs: steps: - name: Get the code - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Install/cache OpenJDK - name: Cache OpenJDK id: cache-openjdk - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /usr/lib/jvm/java-11-openjdk-amd64 key: ${{ runner.os }}-openjdk-11 @@ -42,7 +47,7 @@ jobs: # Install/cache Nextflow - name: Cache Nextflow id: cache-nextflow - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /usr/local/bin/nextflow key: ${{ runner.os }}-nextflow @@ -51,22 +56,32 @@ jobs: - name: Install Nextflow if: ${{ steps.cache-nextflow.outputs.cache-hit != 'true' }} run: cat .github/workflows/install_nextflow_v24.10.5.sh | bash && mv nextflow /usr/local/bin && chmod +x /usr/local/bin/nextflow + + # Check-out the repo under $GITHUB_WORKSPACE so that the job can access it + - uses: actions/checkout@v4 - # Build/cache containers - - name: Cache containers - id: cache-containers - uses: actions/cache@v3 + - name: Cache Dockerfiles and TAR + id: cache-dockerfiles + uses: actions/cache@v4 with: - key: ${{ runner.os }}-containers - path: /var/lib/docker/overlay2 - restore-keys: | - ${{ runner.os }}-containers - - name: Build Containers - if: ${{ steps.cache-containers.outputs.cache-hit != 'true'}} - run: bash build_all.sh + path: docker-images.tar + key: ${{ runner.os }}-dockerfiles-${{ hashFiles('docker/**/*') }} - # Check-out the repo under $GITHUB_WORKSPACE so that the job can access it - - uses: actions/checkout@v3 + - name: Load cached Docker images + id: load-cache + run: | + if [ -f docker-images.tar ]; then + echo "Loading cached Docker images..." + docker load -i docker-images.tar || true + echo "build=false" >> $GITHUB_ENV + else + echo "No Docker cache found" + echo "build=true" >> $GITHUB_ENV + fi + + - name: Build images + if: env.build == 'true' || steps.cache-dockerfiles.outputs.cache-hit != 'true' + run: bash build_images.sh github_action # Run test(s) - name: test short single diff --git a/build_all.sh b/build_all.sh deleted file mode 100755 index 08285b2..0000000 --- a/build_all.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env sh - -# get architecture -arch=$(uname -m) -# set architecture to docker buildx -docker_arch_option="" - -# save original working directory -wd=$(pwd) -for dir in docker/* -do - cd ${dir} - imgname=$(echo $dir | rev | cut -d/ -f1 | rev) - - # Reditools2 does not compile on arm64, force using amd64 compilation - if [[ $dir =~ "reditools2" ]];then - if [[ "$arch" == arm* || "$arch" == "aarch64" ]]; then - echo "Reditools2 does not compile on arm64, force using amd64 compilation" - docker_arch_option=" --platform linux/amd64" - fi - fi - - docker build ${docker_arch_option} -t ${imgname} . - - # back to the original working directory - cd $wd -done diff --git a/build_images.sh b/build_images.sh new file mode 100755 index 0000000..da28da7 --- /dev/null +++ b/build_images.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env sh + +# Run this script with the argument "github_action" in order to save images in an archive for caching +# Pass no arguments to run the script in "normal" build mode suitable for a local machine + +# Read first argument as the "build mode" +# The build is used, for instance, for special build commands +# for Github actions. +build_mode=$1 + +# get architecture +arch=$(uname -m) +# set architecture to docker buildx +docker_arch_option="" + +# save original working directory +wd=$(pwd) + +# list of image names +image_list=( ) + +for dir in docker/* +do + cd ${dir} + imgname=$(echo $dir | rev | cut -d/ -f1 | rev) + image_list+=(${imgname}) + + echo ██████████████████▓▒░ Building ${imgname} ░▒▓██████████████████ + + # Reditools2 does not compile on arm64, force using amd64 compilation + if [[ $dir =~ "reditools2" ]];then + if [[ "$arch" == arm* || "$arch" == "aarch64" ]]; then + echo "Reditools2 does not compile on arm64, force using amd64 compilation" + docker_arch_option=" --platform linux/amd64" + fi + fi + + docker build ${docker_arch_option} -t ${imgname} . + + # back to the original working directory + cd $wd +done + +if [[ ${build_mode} == 'github_action' ]]; then + echo "Saving docker images to cache..." + docker save ${image_list[@]} -o docker-images.tar + echo Archive size: $(stat --printf="%s" docker-images.tar) +fi diff --git a/nextflow.config b/nextflow.config index b438f20..35bfc51 100644 --- a/nextflow.config +++ b/nextflow.config @@ -55,7 +55,7 @@ profiles { params.aligner = "STAR" params.reads = "${baseDir}/data/chr21/chr21_small_R1.fastq.gz " params.genome = "${baseDir}/data/chr21/chr21_small.fasta.gz" - params.annotation = "${baseDir}/data/chr21/chr21_small_filtered.gff3" + params.annotation = "${baseDir}/data/chr21/chr21_small_filtered.gff3.gz" params.library_type = "ISR" params.read_type = "short_single" } @@ -64,7 +64,7 @@ profiles { params.aligner = "STAR" params.reads = "${baseDir}/data/chr21/" params.genome = "${baseDir}/data/chr21/chr21_small.fasta.gz" - params.annotation = "${baseDir}/data/chr21/chr21_small_filtered.gff3" + params.annotation = "${baseDir}/data/chr21/chr21_small_filtered.gff3.gz" params.library_type = "ISR" params.read_type = "short_paired" }