Merge pull request #11 from Juke34/continuous-integration

Fix caching and set up push trigger only for the main branch

Author: eascarrunz

.github/workflows/main.yml

@@ -1,5 +1,10 @@
 name: CI
-on: [push, pull_request, workflow_dispatch]
+on: 
+  push:
+    branches:
+      - "main"
+  pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -24,12 +29,12 @@ jobs:
     steps:
 
       - name: Get the code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Install/cache OpenJDK
       - name: Cache OpenJDK
         id: cache-openjdk
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /usr/lib/jvm/java-11-openjdk-amd64
           key: ${{ runner.os }}-openjdk-11
@@ -42,7 +47,7 @@ jobs:
       # Install/cache Nextflow
       - name: Cache Nextflow
         id: cache-nextflow
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /usr/local/bin/nextflow
           key: ${{ runner.os }}-nextflow
@@ -51,22 +56,32 @@ jobs:
       - name: Install Nextflow
         if: ${{ steps.cache-nextflow.outputs.cache-hit != 'true' }}
         run: cat .github/workflows/install_nextflow_v24.10.5.sh | bash && mv nextflow /usr/local/bin && chmod +x /usr/local/bin/nextflow
+        
+      # Check-out the repo under $GITHUB_WORKSPACE so that the job can access it
+      - uses: actions/checkout@v4
 
-      # Build/cache containers
-      - name: Cache containers
-        id: cache-containers
-        uses: actions/cache@v3
+      - name: Cache Dockerfiles and TAR
+        id: cache-dockerfiles
+        uses: actions/cache@v4
         with:
-          key: ${{ runner.os }}-containers
-          path: /var/lib/docker/overlay2
-          restore-keys: |
-            ${{ runner.os }}-containers
-      - name: Build Containers
-        if: ${{ steps.cache-containers.outputs.cache-hit != 'true'}}
-        run: bash build_all.sh
+          path: docker-images.tar
+          key: ${{ runner.os }}-dockerfiles-${{ hashFiles('docker/**/*') }}
 
-      # Check-out the repo under $GITHUB_WORKSPACE so that the job can access it
-      - uses: actions/checkout@v3
+      - name: Load cached Docker images
+        id: load-cache
+        run: |
+          if [ -f docker-images.tar ]; then
+            echo "Loading cached Docker images..."
+            docker load -i docker-images.tar || true
+            echo "build=false" >> $GITHUB_ENV
+          else
+            echo "No Docker cache found"
+            echo "build=true" >> $GITHUB_ENV
+          fi
+
+      - name: Build images
+        if: env.build == 'true' || steps.cache-dockerfiles.outputs.cache-hit != 'true'
+        run: bash build_images.sh github_action
 
       # Run test(s)
       - name: test short single

build_all.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env sh
+
+# Run this script with the argument "github_action" in order to save images in an archive for caching
+# Pass no arguments to run the script in "normal" build mode suitable for a local machine
+
+# Read first argument as the "build mode"
+# The build is used, for instance, for special build commands
+# for Github actions.
+build_mode=$1
+
+# get architecture
+arch=$(uname -m)
+# set architecture to docker buildx
+docker_arch_option=""
+
+# save original working directory
+wd=$(pwd)
+
+# list of image names
+image_list=( )
+
+for dir in docker/*
+do 
+    cd ${dir}
+    imgname=$(echo $dir | rev | cut -d/ -f1 | rev)
+    image_list+=(${imgname})
+
+    echo ██████████████████▓▒░   Building ${imgname}   ░▒▓██████████████████
+    
+    # Reditools2 does not compile on arm64, force using amd64 compilation
+    if [[ $dir =~ "reditools2" ]];then
+        if [[ "$arch" == arm* || "$arch" == "aarch64" ]]; then
+            echo "Reditools2 does not compile on arm64, force using amd64 compilation"
+            docker_arch_option=" --platform linux/amd64"
+        fi
+    fi
+
+    docker build ${docker_arch_option} -t ${imgname} .
+    
+    # back to the original working directory
+    cd $wd
+done
+
+if [[ ${build_mode} == 'github_action' ]]; then
+    echo "Saving docker images to cache..."
+    docker save ${image_list[@]} -o docker-images.tar
+    echo Archive size: $(stat --printf="%s" docker-images.tar)
+fi

build_images.sh

@@ -55,7 +55,7 @@ profiles {
         params.aligner        = "STAR" 
         params.reads          = "${baseDir}/data/chr21/chr21_small_R1.fastq.gz "
         params.genome         = "${baseDir}/data/chr21/chr21_small.fasta.gz"
-        params.annotation     = "${baseDir}/data/chr21/chr21_small_filtered.gff3"
+        params.annotation     = "${baseDir}/data/chr21/chr21_small_filtered.gff3.gz"
         params.library_type   = "ISR" 
         params.read_type      = "short_single"
     }
@@ -64,7 +64,7 @@ profiles {
         params.aligner        = "STAR" 
         params.reads          = "${baseDir}/data/chr21/"
         params.genome         = "${baseDir}/data/chr21/chr21_small.fasta.gz" 
-        params.annotation     = "${baseDir}/data/chr21/chr21_small_filtered.gff3"
+        params.annotation     = "${baseDir}/data/chr21/chr21_small_filtered.gff3.gz"
         params.library_type   = "ISR" 
         params.read_type      = "short_paired"
     }