From 8e9d0782db9a3f2a81aa8f47409820e619676d8e Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 02:44:27 +0200 Subject: [PATCH 01/12] feat: enable image scanning with Trivy --- .github/workflows/release.yml | 2 +- .gitignore | 1 + default.nix | 4 +- flake.lock | 4 +- flake.nix | 72 ++++++++++++++++++++++------------ lib/default.nix | 61 ++++++++++++++-------------- scripts/_template/default.nix | 14 +++++++ scripts/_template/script.sh | 30 ++++++++++++++ scripts/default.nix | 41 +++++++++++++++++++ scripts/trivy-scan/default.nix | 56 ++++++++++++++++++++++++++ scripts/trivy-scan/script.sh | 23 +++++++++++ shell.nix | 40 +++++++++++++++---- 12 files changed, 281 insertions(+), 67 deletions(-) create mode 100644 scripts/_template/default.nix create mode 100755 scripts/_template/script.sh create mode 100644 scripts/default.nix create mode 100644 scripts/trivy-scan/default.nix create mode 100755 scripts/trivy-scan/script.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a8282e4..0bda429 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: - id: get-images run: | nix flake show --json --quiet --quiet \ - | jq -r '.packages["x86_64-linux"] | delpaths([["default"],["geolite2"]]) | keys | "images=" + (. | tostring)' \ + | jq -r '.packages["x86_64-linux"] | delpaths([["default"],["geolite2"],["trivy-scan"]]) | keys | "images=" + (. | tostring)' \ >> $GITHUB_OUTPUT build-and-push: diff --git a/.gitignore b/.gitignore index 2efaad0..088b12c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .direnv result* +.trivy diff --git a/default.nix b/default.nix index 6e05844..1da6571 100644 --- a/default.nix +++ b/default.nix @@ -23,8 +23,8 @@ let xlib = import ./lib { inherit nix2container pkgs xpkgs; }; in -xlib.mkAllImages +pkgs.lib.mapAttrs (_: value: value.image) xlib.mkAllImages // xpkgs // { - default = xlib.mkImage { }; + default = (xlib.mkImage { }).image; } diff --git a/flake.lock b/flake.lock index 9a1151d..bd0c20b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,6 @@ { "nodes": { - "nix2container": { + "n2c": { "inputs": { "nixpkgs": [ "nixpkgs" @@ -38,7 +38,7 @@ }, "root": { "inputs": { - "nix2container": "nix2container", + "n2c": "n2c", "nixpkgs": "nixpkgs", "treefmt-nix": "treefmt-nix" } diff --git a/flake.nix b/flake.nix index d5303a4..ba2614d 100644 --- a/flake.nix +++ b/flake.nix @@ -4,7 +4,7 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/25.05"; - nix2container = { + n2c = { url = "github:nlewo/nix2container"; inputs.nixpkgs.follows = "nixpkgs"; }; @@ -19,7 +19,7 @@ { self, nixpkgs, - nix2container, + n2c, treefmt-nix, ... }: @@ -34,8 +34,8 @@ f: nixpkgs.lib.genAttrs systems ( system: - f ( - import nixpkgs { + f rec { + pkgs = import nixpkgs { inherit system; config.allowUnfreePredicate = @@ -43,38 +43,60 @@ builtins.elem (nixpkgs.lib.getName pkg) [ "geolite2" ]; - } - ) + }; + + xpkgs = pkgs.lib.packagesFromDirectoryRecursive { + inherit (pkgs) callPackage; + + directory = ./pkgs; + }; + + nix2container = n2c.packages.${pkgs.system}; + } ); - treefmtEval = eachSystem (pkgs: treefmt-nix.lib.evalModule pkgs ./treefmt.nix); + treefmtEval = eachSystem ({ pkgs, ... }: treefmt-nix.lib.evalModule pkgs ./treefmt.nix); in { # nix flake check - checks = eachSystem (pkgs: { - formatting = treefmtEval.${pkgs.system}.config.build.check self; - }); + checks = eachSystem ( + { pkgs, ... }: + { + formatting = treefmtEval.${pkgs.system}.config.build.check self; + } + ); # nix fmt - formatter = eachSystem (pkgs: treefmtEval.${pkgs.system}.config.build.wrapper); + formatter = eachSystem ({ pkgs, ... }: treefmtEval.${pkgs.system}.config.build.wrapper); # Development environment with tools available in PATH - devShells = eachSystem (pkgs: { - default = pkgs.callPackage ./shell.nix { }; - }); + devShells = eachSystem ( + { + pkgs, + xpkgs, + nix2container, + ... + }: + { + default = pkgs.callPackage ./shell.nix { + inherit xpkgs nix2container; + }; + } + ); packages = eachSystem ( - pkgs: - let - xpkgs = pkgs.lib.packagesFromDirectoryRecursive { - inherit (pkgs) callPackage; - - directory = ./pkgs; - }; - in - import ./default.nix { + { + pkgs, + xpkgs, + nix2container, + ... + }: + (import ./default.nix { inherit pkgs xpkgs; - inherit (nix2container.packages.${pkgs.system}) nix2container; - } + inherit (nix2container) nix2container; + }) + // (import ./scripts { + inherit pkgs xpkgs nix2container; + }) ); }; } diff --git a/lib/default.nix b/lib/default.nix index 5fd127b..8a3ecd0 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -87,10 +87,6 @@ rec { } ); - in - nix2container.buildImage { - name = "goaccess"; - # Generate tag based on version and enabled features tag = lib.concatStrings [ goaccess.version @@ -98,33 +94,40 @@ rec { (lib.optionalString (!withGeolite2 && withGeolocation) "-geoip") (lib.optionalString (baseImage != "") "-${baseImage}") ]; + in + { + inherit tag; - # Set the base image to build from (empty string means scratch/no base) - fromImage = - if (baseImage != "") then + image = nix2container.buildImage { + inherit tag; + + name = "goaccess"; + + # Set the base image to build from + fromImage = lib.optionalString (baseImage != "") ( nix2container.pullImage (distros.${baseImage} or throwDistro) - else - baseImage; - - # Build the root filesystem environment - copyToRoot = buildEnv { - name = "root"; - - # Packages to include in the container - paths = - [ goaccessBuild ] - # Include GeoLite2 database if requested - ++ lib.optional withGeolite2 geolite2; - - # Directories to symlink into the container root - pathsToLink = [ - "/bin" - "/etc" - "/share" - ]; + ); + + # Build the root filesystem environment + copyToRoot = buildEnv { + name = "root"; + + # Packages to include in the container + paths = + [ goaccessBuild ] + # Include GeoLite2 database if requested + ++ lib.optional withGeolite2 geolite2; + + # Directories to symlink into the container root + pathsToLink = [ + "/bin" + "/etc" + "/share" + ]; + }; + + # Default command to run when container starts + config.Entrypoint = [ "goaccess" ]; }; - - # Default command to run when container starts - config.Entrypoint = [ "goaccess" ]; }; } diff --git a/scripts/_template/default.nix b/scripts/_template/default.nix new file mode 100644 index 0000000..f9a88cf --- /dev/null +++ b/scripts/_template/default.nix @@ -0,0 +1,14 @@ +{ writeShellApplication, coreutils, ... }: + +let + name = builtins.baseNameOf (builtins.toString ./.); +in +writeShellApplication { + inherit name; + + text = builtins.readFile ./script.sh; + + runtimeInputs = [ + coreutils + ]; +} diff --git a/scripts/_template/script.sh b/scripts/_template/script.sh new file mode 100755 index 0000000..101a63a --- /dev/null +++ b/scripts/_template/script.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# set +e # Do not exit on error +set -e # Exit on error +set +u # Allow unset variables +# set -u # Exit on unset variable +# set +o pipefail # Disable pipefail +set -o pipefail # Enable pipefail + +nc="\e[0m" # Unset styles +red="\e[31m" # Red foreground + +error() { + >&2 echo -e " ${red}×${nc} ${*}" +} + +# shellcheck disable=SC2120 +die() { + if [ "${#}" -gt 0 ]; then + error "${*}" + fi + + exit 1 +} + +main() { + echo "This is a template." +} + +main "$@" diff --git a/scripts/default.nix b/scripts/default.nix new file mode 100644 index 0000000..acce183 --- /dev/null +++ b/scripts/default.nix @@ -0,0 +1,41 @@ +{ + pkgs ? import (fetchTarball "https://github.com/nixos/nixpkgs/archive/25.05.tar.gz") { + config.allowUnfreePredicate = + pkg: + builtins.elem (pkgs.lib.getName pkg) [ + "geolite2" + ]; + }, + nix2container ? ( + import "${fetchTarball "https://github.com/nlewo/nix2container/archive/master.tar.gz"}/default.nix" + { + inherit pkgs; + inherit (pkgs) system; + } + ), + xpkgs ? pkgs.lib.packagesFromDirectoryRecursive { + inherit (pkgs) callPackage; + + directory = ../pkgs; + }, +}: + +let + xlib = import ../lib { + inherit pkgs xpkgs; + inherit (nix2container) nix2container; + }; + lib = pkgs.lib; + currentDir = ./.; + packages = lib.filterAttrs ( + name: type: + (type == "directory" && name != "_template") || (lib.hasSuffix ".nix" name && name != "default.nix") + ) (builtins.readDir currentDir); +in +lib.mapAttrs ( + name: type: + pkgs.callPackage (currentDir + "/${name}") { + inherit xlib; + inherit (nix2container) skopeo-nix2container; + } +) packages diff --git a/scripts/trivy-scan/default.nix b/scripts/trivy-scan/default.nix new file mode 100644 index 0000000..659dcb6 --- /dev/null +++ b/scripts/trivy-scan/default.nix @@ -0,0 +1,56 @@ +{ + lib, + xlib, + writeShellApplication, + coreutils, + jq, + skopeo-nix2container, + trivy, +}: + +let + name = builtins.baseNameOf (builtins.toString ./.); + allImages = xlib.mkAllImages; + allImagesImages = lib.mapAttrs (_: value: value.image) allImages; +in +writeShellApplication { + inherit name; + + text = builtins.readFile ./script.sh; + + runtimeInputs = + [ + coreutils + jq + skopeo-nix2container + trivy + ] + # Add all built images to runtime environment + ++ (lib.map (img: img.value) (lib.attrsToList allImagesImages)); + + runtimeEnv = + let + fixImgNameForEnv = name: lib.replaceStrings [ "-" ] [ "_" ] name; + + # Attribute set of all images derivation paths and tags + # Will add the following environment variables: + # - `_drv`: image store path + # - `_tag`: image tag + imagesDrvs = lib.foldlAttrs ( + acc: name: value: + let + nameInEnv = fixImgNameForEnv name; + in + acc + // { + "${nameInEnv}_drv" = value.image; + "${nameInEnv}_tag" = value.tag; + } + ) { } allImages; + in + { + # List of all images to scan + images = lib.map fixImgNameForEnv (lib.attrNames allImages); + } + // imagesDrvs; +} diff --git a/scripts/trivy-scan/script.sh b/scripts/trivy-scan/script.sh new file mode 100755 index 0000000..27772bb --- /dev/null +++ b/scripts/trivy-scan/script.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +tmp_dir=".trivy" +scans_dir="$tmp_dir/scans" + +mkdir -p "$scans_dir" + +# shellcheck disable=SC2154 +for img in "${images[@]}"; do + img_name=${img//_/-} + drv_varname="${img}_drv" + tag_varname="${img}_tag" + destoci=".trivy/$img_name" + destsarif="$scans_dir/$img_name.sarif" + skopeo --insecure-policy copy nix:"${!drv_varname}" oci:"$destoci" + trivy image --input "$destoci" --scanners vuln -f sarif -o "$destsarif" + sed -i "s|$destoci|${!tag_varname}|" "$destsarif" + sed -i 's/"repoTags": null/"repoTags": ["'"${!tag_varname}"'"]/' "$destsarif" +done + +jq -s '{ "$schema": "https://json.schemastore.org/sarif-2.1.0", "version": "2.1.0", "runs": map(.runs) | add }' $scans_dir/*.sarif > $tmp_dir/all.sarif + +find $tmp_dir -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \; diff --git a/shell.nix b/shell.nix index d338837..51780f9 100644 --- a/shell.nix +++ b/shell.nix @@ -1,17 +1,41 @@ { - pkgs ? import (fetchTarball "https://github.com/nixos/nixpkgs/archive/25.05.tar.gz") { }, + pkgs ? import (fetchTarball "https://github.com/nixos/nixpkgs/archive/25.05.tar.gz") { + config.allowUnfreePredicate = + pkg: + builtins.elem (pkgs.lib.getName pkg) [ + "geolite2" + ]; + }, + nix2container ? ( + import "${fetchTarball "https://github.com/nlewo/nix2container/archive/master.tar.gz"}/default.nix" + { + inherit pkgs; + inherit (pkgs) system; + } + ), + xpkgs ? pkgs.lib.packagesFromDirectoryRecursive { + inherit (pkgs) callPackage; + + directory = ./pkgs; + }, }: let - inherit (pkgs) mkShellNoCC; + inherit (pkgs) mkShellNoCC lib; + + scripts = lib.map (img: img.value) ( + lib.attrsToList (import ./scripts { inherit pkgs xpkgs nix2container; }) + ); in mkShellNoCC { - nativeBuildInputs = with pkgs; [ - bash - coreutils - findutils - mmdbinspect - ]; + nativeBuildInputs = + (with pkgs; [ + bash + coreutils + findutils + mmdbinspect + ]) + ++ scripts; shellHook = '' find .hooks \ From ed47bc0c89766ab43cdcd3eea1169c7e31cb798f Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 03:10:58 +0200 Subject: [PATCH 02/12] ci: add code scanning workflow --- .github/workflows/code-scanning.yml | 33 ++++++++++++++++++++++++++++ .github/workflows/housekeeping.yml | 1 - .github/workflows/release.yml | 4 ++-- .github/workflows/update-geolite.yml | 1 - 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/code-scanning.yml diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml new file mode 100644 index 0000000..e54b35d --- /dev/null +++ b/.github/workflows/code-scanning.yml @@ -0,0 +1,33 @@ +name: Code scanning + +on: + workflow_dispatch: + workflow_run: + workflows: [Release] + types: [completed] + pull_request: + types: + - opened + - synchronize + branches: [main] + paths: + - "lib/*.nix" + - "pkgs/*.nix" + - "scripts/trivy-scan/**/*" + +jobs: + trivy: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.pull_request.id }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Nix + uses: cachix/install-nix-action@v31 + - name: Build SARIF file + run: nix run '.#trivy-scan' + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: .trivy/all.sarif diff --git a/.github/workflows/housekeeping.yml b/.github/workflows/housekeeping.yml index c5e9464..6ef5edc 100644 --- a/.github/workflows/housekeeping.yml +++ b/.github/workflows/housekeeping.yml @@ -1,7 +1,6 @@ name: Cleanup on: - repository_dispatch: workflow_dispatch: schedule: # Run every day diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0bda429..2c626bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,12 +2,12 @@ name: Release on: push: - branches: - - main + branches: [main] paths: - .github/workflows/release.yml - "**.nix" - flake.lock + - "!scripts/**/*" env: REGISTRY: ghcr.io diff --git a/.github/workflows/update-geolite.yml b/.github/workflows/update-geolite.yml index db4801f..3de9aa4 100644 --- a/.github/workflows/update-geolite.yml +++ b/.github/workflows/update-geolite.yml @@ -1,7 +1,6 @@ name: Update GeoLite on: - repository_dispatch: workflow_dispatch: schedule: # Run every day From 6b9a23dce152831f0cd14e91febef4bbb76d96dc Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:14:17 +0200 Subject: [PATCH 03/12] refactor(scripts): improve trivy-scan - add error handling with trap - add input validation - add better error reporting - move code to functions - use readonly vars for config - add colored logging - add progress indicators - report successful vs failed scans - validate files before processing - add comprehensive comments and function documentation --- scripts/trivy-scan/script.sh | 209 ++++++++++++++++++++++++++++++++--- 1 file changed, 191 insertions(+), 18 deletions(-) diff --git a/scripts/trivy-scan/script.sh b/scripts/trivy-scan/script.sh index 27772bb..43eedf2 100755 --- a/scripts/trivy-scan/script.sh +++ b/scripts/trivy-scan/script.sh @@ -1,23 +1,196 @@ -#!/usr/bin/env bash +# shellcheck shell=bash -tmp_dir=".trivy" -scans_dir="$tmp_dir/scans" +# Global variables +readonly tmp_dir=".trivy" +readonly scans_dir="$tmp_dir/scan_results" -mkdir -p "$scans_dir" +# Color codes +readonly nc="\e[0m" # Unset styles +readonly red="\e[31m" # Red foreground +readonly green="\e[32m" # Green foreground +readonly yellow="\e[33m" # Yellow foreground +readonly blue="\e[34m" # Blue foreground -# shellcheck disable=SC2154 -for img in "${images[@]}"; do - img_name=${img//_/-} - drv_varname="${img}_drv" - tag_varname="${img}_tag" - destoci=".trivy/$img_name" - destsarif="$scans_dir/$img_name.sarif" - skopeo --insecure-policy copy nix:"${!drv_varname}" oci:"$destoci" - trivy image --input "$destoci" --scanners vuln -f sarif -o "$destsarif" - sed -i "s|$destoci|${!tag_varname}|" "$destsarif" - sed -i 's/"repoTags": null/"repoTags": ["'"${!tag_varname}"'"]/' "$destsarif" -done +# Logging functions +info() { + echo -e " ${blue}i${nc} ${*}" +} -jq -s '{ "$schema": "https://json.schemastore.org/sarif-2.1.0", "version": "2.1.0", "runs": map(.runs) | add }' $scans_dir/*.sarif > $tmp_dir/all.sarif +success() { + echo -e " ${green}✔${nc} ${*}" +} -find $tmp_dir -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \; +warn() { + echo -e " ${yellow}⚠${nc} ${*}" +} + +error() { + >&2 echo -e " ${red}×${nc} ${*}" +} + +# Cleanup function for error handling +cleanup() { + local exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + error "Script failed with exit code $exit_code" + info "Cleaning up temporary files..." + rm -rf "$tmp_dir" 2>/dev/null || true + fi + + exit $exit_code +} + +# Set up error handling +trap cleanup EXIT + +# Initialize scan environment +init_scan_env() { + info "Initializing scan environment..." + + # Remove existing directory if it exists to ensure clean state + [[ -d "$tmp_dir" ]] && rm -rf "$tmp_dir" + + # Create scan directory + mkdir -p "$scans_dir" + + success "Scan directory created: $scans_dir" +} + +# Scan individual container image +scan_img() { + local img="$1" + local img_name=${img//_/-} # Replace underscores with hyphens + local drv_varname="${img}_drv" + local tag_varname="${img}_tag" + local destoci="$tmp_dir/$img_name" + local destsarif="$scans_dir/$img_name.sarif" + + info "Scanning image: $img" + + # Validate that required variables are set + if [[ -z "${!drv_varname:-}" ]]; then + error "Derivation variable $drv_varname is not set for image $img" + return 1 + fi + + if [[ -z "${!tag_varname:-}" ]]; then + error "Tag variable $tag_varname is not set for image $img" + return 1 + fi + + # Copy image derivation to OCI layout + info "Copying image derivation to OCI layout..." + if ! skopeo --insecure-policy copy nix:"${!drv_varname}" oci:"$destoci"; then + error "Failed to copy image derivation for $img" + return 1 + fi + + # Scan image for vulnerabilities and generate SARIF report + info "Scanning for vulnerabilities..." + if ! trivy image --input "$destoci" --scanners vuln -f sarif -o "$destsarif"; then + error "Trivy scan failed for $img" + return 1 + fi + + # Update SARIF report with actual image tag information + info "Updating SARIF report with image tag information..." + if [[ -f "$destsarif" ]]; then + # Replace OCI path with actual tag name in scan results + sed -i "s|$destoci|${!tag_varname}|" "$destsarif" + # Set proper repository tags in SARIF metadata + sed -i 's/"repoTags": null/"repoTags": ["'"${!tag_varname}"'"]/' "$destsarif" + + success "Scan completed for $img -> $destsarif" + else + error "SARIF report not generated for $img" + return 1 + fi +} + +# Function to merge multiple SARIF files into a single consolidated report +# This combines multiple scan results while preserving the SARIF format structure +# +# Arguments: +# $1 - Glob pattern of SARIF files to merge +# $2 - Destination file path for the merged report +merge_sarif_files() { + local glob_pattern="$1" + local dest_file="$2" + + info "Merging SARIF files: $glob_pattern -> $dest_file" + + # Check if any files match the pattern + # shellcheck disable=SC2206 + local files=($glob_pattern) + if [[ ! -f "${files[0]}" ]]; then + warn "No files found matching pattern: $glob_pattern" + return 0 + fi + + # Merge SARIF files using jq to combine runs from multiple files + # shellcheck disable=SC2086 + if ! jq -s '{ + "$schema": "https://json.schemastore.org/sarif-2.1.0", + "version": "2.1.0", + "runs": map(.runs) | add + }' $glob_pattern > "$dest_file"; then + error "Failed to merge SARIF files for pattern: $glob_pattern" + return 1 + fi + + success "Successfully merged ${#files[@]} files into $dest_file" +} + +main() { + info "Starting container vulnerability scanning process..." + + # Check if images array is defined and not empty + if [[ -z "${images:-}" ]] || [[ ${#images[@]} -eq 0 ]]; then + error "No images defined in 'images' array" + warn "Please define an array of image names before running this script" + exit 1 + fi + + # Initialize scanning environment + init_scan_env + + # Scan all images in the array + info "Scanning ${#images[@]} images..." + local failed_scans=0 + + # shellcheck disable=SC2154 + for img in "${images[@]}"; do + if ! scan_img "$img"; then + ((failed_scans++)) + error "Failed to scan image: $img" + fi + done + + # Report scan summary + local successful_scans=$((${#images[@]} - failed_scans)) + info "Scan summary: $successful_scans successful, $failed_scans failed" + + if [[ $failed_scans -gt 0 ]]; then + warn "Some scans failed. Check the logs above for details." + fi + + # Merge SARIF files by distribution type + info "Merging SARIF reports by distribution..." + merge_sarif_files "$scans_dir/alpine*.sarif" "$tmp_dir/alpine.sarif" + merge_sarif_files "$scans_dir/debian*.sarif" "$tmp_dir/debian.sarif" + merge_sarif_files "$scans_dir/distroless*.sarif" "$tmp_dir/distroless.sarif" + merge_sarif_files "$scans_dir/ubuntu*.sarif" "$tmp_dir/ubuntu.sarif" + + # Cleanup intermediate OCI directories (keep SARIF files) + info "Cleaning up intermediate files..." + find "$tmp_dir" -mindepth 1 -maxdepth 1 -type d -not -path "$scans_dir" -exec rm -rf {} \; 2>/dev/null || true + + success "Vulnerability scanning completed successfully!" + success "Results available in: $tmp_dir" +} + +# Execute main function if script is run directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi From 095c231f596b4ee493f28e430c92ccf86cfe027d Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:15:34 +0200 Subject: [PATCH 04/12] ci(code-scanning): upload multiple SARIF files --- .github/workflows/code-scanning.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index e54b35d..264c2f8 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -16,7 +16,7 @@ on: - "scripts/trivy-scan/**/*" jobs: - trivy: + trivy-scan: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.pull_request.id }} steps: @@ -27,7 +27,24 @@ jobs: uses: cachix/install-nix-action@v31 - name: Build SARIF file run: nix run '.#trivy-scan' - - name: Upload SARIF file + - name: Get SARIF files + id: sarif_files + run: | + sarifs=(.trivy/*.sarif) + json_array=$(printf '%s\n' "${sarifs[@]}" | jq -R . | jq -sc .) + echo "sarif_files=$json_array" >> $GITHUB_OUTPUT + + upload-sarifs: + needs: trivy-scan + runs-on: ubuntu-latest + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sarif_file: ${{ fromJson(needs.upload-sarifs.outputs.sarif_files) }} + steps: + - name: "Upload sarif file: ${{ matrix.sarif_file }}" uses: github/codeql-action/upload-sarif@v3 with: - sarif_file: .trivy/all.sarif + sarif_file: ${{ matrix.sarif_file }} From 81fe6a5c62e1144b3b2faed8d88be1e5295a6e7d Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:23:51 +0200 Subject: [PATCH 05/12] ci(update-geolite): update commit message and PR title --- .github/workflows/update-geolite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-geolite.yml b/.github/workflows/update-geolite.yml index 3de9aa4..85a8ccc 100644 --- a/.github/workflows/update-geolite.yml +++ b/.github/workflows/update-geolite.yml @@ -33,8 +33,8 @@ jobs: uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} - commit-message: "refactor(pkgs/geolite2): update database" - title: "refactor(pkgs/geolite2): update database" + commit-message: "refactor(pkgs): update geolite2" + title: "refactor(pkgs): update geolite2" body: | Automated update of GeoLite2 database. From 94a770d3caba2756483220a6d0022b3974e6486d Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:28:27 +0200 Subject: [PATCH 06/12] ci(code-scanning): fix invalid matrix fromJson --- .github/workflows/code-scanning.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 264c2f8..e4a5e60 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - sarif_file: ${{ fromJson(needs.upload-sarifs.outputs.sarif_files) }} + sarif_file: ${{ fromJson(needs.trivy-scan.outputs.sarif_files) }} steps: - name: "Upload sarif file: ${{ matrix.sarif_file }}" uses: github/codeql-action/upload-sarif@v3 From 1e264410eed6c4f46b56f50f1f6ef0db1052bde8 Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:39:33 +0200 Subject: [PATCH 07/12] ci(code-scanning): add missing outputs to trivy-scan job --- .github/workflows/code-scanning.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index e4a5e60..f476e0b 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -19,6 +19,8 @@ jobs: trivy-scan: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.pull_request.id }} + outputs: + sarif_files: ${{ steps.sarif_files.outputs.sarif_files }} steps: - uses: actions/checkout@v4 with: From 588f784a1626d23bde0bebc739544192ae61a5b6 Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:49:22 +0200 Subject: [PATCH 08/12] ci(code-scanning): upload SARIF as artifacts for sharing between jobs --- .github/workflows/code-scanning.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index f476e0b..0e1011b 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -35,6 +35,11 @@ jobs: sarifs=(.trivy/*.sarif) json_array=$(printf '%s\n' "${sarifs[@]}" | jq -R . | jq -sc .) echo "sarif_files=$json_array" >> $GITHUB_OUTPUT + - name: Upload SARIF files + uses: actions/upload-artifact@v4 + with: + name: sarif_files + path: .trivy upload-sarifs: needs: trivy-scan @@ -46,6 +51,11 @@ jobs: matrix: sarif_file: ${{ fromJson(needs.trivy-scan.outputs.sarif_files) }} steps: + - name: Download SARIF files + uses: actions/download-artifact@v4 + with: + name: sarif_files + path: .trivy - name: "Upload sarif file: ${{ matrix.sarif_file }}" uses: github/codeql-action/upload-sarif@v3 with: From 27fdfe371f86a8ae02a61ff5c2d1eff0b3754952 Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 10:56:26 +0200 Subject: [PATCH 09/12] ci(code-scanning): update artifact upload source path --- .github/workflows/code-scanning.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 0e1011b..55ea7e0 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -39,7 +39,8 @@ jobs: uses: actions/upload-artifact@v4 with: name: sarif_files - path: .trivy + include-hidden-files: true + path: .trivy/*.sarif upload-sarifs: needs: trivy-scan From 53f27571043f2befa607790670a1bb600414f56d Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 11:32:56 +0200 Subject: [PATCH 10/12] ci(code-scanning): set SARIF category from report filename --- .github/workflows/code-scanning.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 55ea7e0..5a5e087 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -57,7 +57,14 @@ jobs: with: name: sarif_files path: .trivy + - name: Extract category from filename + id: category + run: | + filename=$(basename "${{ matrix.sarif_file }}") + category="${filename%.sarif}" + echo "category=$category" >> $GITHUB_OUTPUT - name: "Upload sarif file: ${{ matrix.sarif_file }}" uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ${{ matrix.sarif_file }} + category: ${{ steps.category.outputs.category }} From 5bb309eda02d4d109ffab82fd5cee361a1c85b0e Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 14:26:10 +0200 Subject: [PATCH 11/12] ci(code-scanning): do not use merged SARIF files for code scanning --- .github/workflows/code-scanning.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 5a5e087..0fdb615 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -32,7 +32,7 @@ jobs: - name: Get SARIF files id: sarif_files run: | - sarifs=(.trivy/*.sarif) + sarifs=(.trivy/scan_results/*.sarif) json_array=$(printf '%s\n' "${sarifs[@]}" | jq -R . | jq -sc .) echo "sarif_files=$json_array" >> $GITHUB_OUTPUT - name: Upload SARIF files @@ -40,7 +40,7 @@ jobs: with: name: sarif_files include-hidden-files: true - path: .trivy/*.sarif + path: .trivy/scan_results/*.sarif upload-sarifs: needs: trivy-scan @@ -56,7 +56,7 @@ jobs: uses: actions/download-artifact@v4 with: name: sarif_files - path: .trivy + path: .trivy/scan_results - name: Extract category from filename id: category run: | From 6482b2ea41fd868af9cea85a97214c125fccadd5 Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Mon, 4 Aug 2025 14:33:04 +0200 Subject: [PATCH 12/12] refactor(scripts/trivy-scan): remove merge feature --- scripts/trivy-scan/script.sh | 41 ------------------------------------ 1 file changed, 41 deletions(-) diff --git a/scripts/trivy-scan/script.sh b/scripts/trivy-scan/script.sh index 43eedf2..fbc2939 100755 --- a/scripts/trivy-scan/script.sh +++ b/scripts/trivy-scan/script.sh @@ -108,40 +108,6 @@ scan_img() { fi } -# Function to merge multiple SARIF files into a single consolidated report -# This combines multiple scan results while preserving the SARIF format structure -# -# Arguments: -# $1 - Glob pattern of SARIF files to merge -# $2 - Destination file path for the merged report -merge_sarif_files() { - local glob_pattern="$1" - local dest_file="$2" - - info "Merging SARIF files: $glob_pattern -> $dest_file" - - # Check if any files match the pattern - # shellcheck disable=SC2206 - local files=($glob_pattern) - if [[ ! -f "${files[0]}" ]]; then - warn "No files found matching pattern: $glob_pattern" - return 0 - fi - - # Merge SARIF files using jq to combine runs from multiple files - # shellcheck disable=SC2086 - if ! jq -s '{ - "$schema": "https://json.schemastore.org/sarif-2.1.0", - "version": "2.1.0", - "runs": map(.runs) | add - }' $glob_pattern > "$dest_file"; then - error "Failed to merge SARIF files for pattern: $glob_pattern" - return 1 - fi - - success "Successfully merged ${#files[@]} files into $dest_file" -} - main() { info "Starting container vulnerability scanning process..." @@ -175,13 +141,6 @@ main() { warn "Some scans failed. Check the logs above for details." fi - # Merge SARIF files by distribution type - info "Merging SARIF reports by distribution..." - merge_sarif_files "$scans_dir/alpine*.sarif" "$tmp_dir/alpine.sarif" - merge_sarif_files "$scans_dir/debian*.sarif" "$tmp_dir/debian.sarif" - merge_sarif_files "$scans_dir/distroless*.sarif" "$tmp_dir/distroless.sarif" - merge_sarif_files "$scans_dir/ubuntu*.sarif" "$tmp_dir/ubuntu.sarif" - # Cleanup intermediate OCI directories (keep SARIF files) info "Cleaning up intermediate files..." find "$tmp_dir" -mindepth 1 -maxdepth 1 -type d -not -path "$scans_dir" -exec rm -rf {} \; 2>/dev/null || true