From 684294d7c3401024f15caabc25ed9037e4bb2ed7 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 15:10:33 +0530 Subject: [PATCH 01/26] updated the cron and loading logic. This changes helps to run the post and pre merge event . Signed-off-by: Mani Deepak Gurram --- .github/actions/lava_job_render/action.yml | 23 +- .github/actions/loading/action.yml | 82 ++- .github/workflows/post_merge.yml | 9 +- .github/workflows/pre_merge.yml | 7 +- .github/workflows/sync-and-build.yml | 573 ++++++++++----------- .github/workflows/test.yml | 8 +- 6 files changed, 363 insertions(+), 339 deletions(-) diff --git a/.github/actions/lava_job_render/action.yml b/.github/actions/lava_job_render/action.yml index fca9d86ca..e31a7e9ef 100644 --- a/.github/actions/lava_job_render/action.yml +++ b/.github/actions/lava_job_render/action.yml @@ -34,7 +34,6 @@ runs: const modulesTarUrl = findUrlByFilename('modules.tar.xz'); const imageUrl = findUrlByFilename('Image'); const mergedRamdiskUrl = findUrlByFilename('video-merged.cpio.gz'); - const vmlinuxUrl = findUrlByFilename('vmlinux'); // DTB is expected to be ".dtb" const dtbFilename = `${process.env.MACHINE}.dtb`; @@ -42,13 +41,11 @@ runs: core.setOutput('modules_url', modulesTarUrl || ''); core.setOutput('image_url', imageUrl || ''); - core.setOutput('vmlinux_url', vmlinuxUrl || ''); core.setOutput('dtb_url', dtbUrl || ''); core.setOutput('merged_ramdisk_url', mergedRamdiskUrl || ''); console.log(`Modules URL: ${modulesTarUrl}`); console.log(`Image URL: ${imageUrl}`); - console.log(`Vmlinux URL: ${vmlinuxUrl}`); console.log(`Dtb URL: ${dtbUrl}`); console.log(`Merged Ramdisk URL: ${mergedRamdiskUrl}`); @@ -68,19 +65,18 @@ runs: - name: Upload metadata.json id: upload_metadata - uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/actions/aws_s3_helper@workflow_2 with: local_file: ../job_render/data/metadata.json - s3_bucket: qli-prd-video-gh-artifacts + s3_bucket: qli-stg-video-gh-artifacts mode: single-upload - name: Create template json (cloudData.json) shell: bash run: | - echo "Populating cloudData.json with kernel, vmlinux, modules, metadata, ramdisk" + echo "Populating cloudData.json with kernel, modules, metadata, ramdisk" metadata_url="${{ steps.upload_metadata.outputs.presigned_url }}" image_url="${{ steps.process_urls.outputs.image_url }}" - vmlinux_url="${{ steps.process_urls.outputs.vmlinux_url }}" modules_url="${{ steps.process_urls.outputs.modules_url }}" merged_ramdisk_url="${{ steps.process_urls.outputs.merged_ramdisk_url }}" @@ -104,15 +100,6 @@ runs: ${{ inputs.docker_image }} \ jq '.artifacts.kernel = env.image_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json - # vmlinux (set only if present) - docker run -i --rm \ - --user "$(id -u):$(id -g)" \ - --workdir="$PWD" \ - -v "$(dirname "$PWD")":"$(dirname "$PWD")" \ - -e vmlinux_url="$vmlinux_url" \ - ${{ inputs.docker_image }} \ - sh -c 'if [ -n "$vmlinux_url" ]; then jq ".artifacts.vmlinux = env.vmlinux_url" data/cloudData.json > temp.json && mv temp.json data/cloudData.json; fi' - # modules docker run -i --rm \ --user "$(id -u):$(id -g)" \ @@ -138,7 +125,7 @@ runs: # Fallback to stable kerneltest ramdisk only if merged ramdisk is not available if [ -z "${{ steps.process_urls.outputs.merged_ramdisk_url }}" ]; then echo "Merged ramdisk not found. Using stable kerneltest ramdisk fallback." - ramdisk_url="$(aws s3 presign s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz --expires 7600)" + ramdisk_url="$(aws s3 presign s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz --expires 7600)" docker run -i --rm \ --user "$(id -u):$(id -g)" \ --workdir="$PWD" \ @@ -152,7 +139,7 @@ runs: # Optional board-specific firmware initramfs if [ -n "${{ env.FIRMWARE }}" ]; then - firmware_url="$(aws s3 presign s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/initramfs-firmware-${{ env.FIRMWARE }}-image-qcom-armv8a.cpio.gz --expires 7600)" + firmware_url="$(aws s3 presign s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-firmware-${{ env.FIRMWARE }}-image-qcom-armv8a.cpio.gz --expires 7600)" docker run -i --rm \ --user "$(id -u):$(id -g)" \ --workdir="$PWD" \ diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 2b90396dd..7dd19485f 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -21,24 +21,68 @@ runs: script: | const fs = require('fs'); const path = require('path'); - const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - let targets; + try { - if (!fs.existsSync(targetsPath)) { - core.setFailed(`MACHINES.json not found at ${targetsPath}`); - return; + // 1. Define both possible paths for MACHINES.json + // Path 1: Inside a 'video-driver' subdirectory (Common in Pre-Merge/PRs) + const pathWithSubdir = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + // Path 2: Directly at the root (Common in Post-Merge/Scheduled runs) + const pathAtRoot = path.join(process.env.GITHUB_WORKSPACE, 'ci', 'MACHINES.json'); + + let targetsPath; + + // 2. Check which path actually exists + if (fs.existsSync(pathWithSubdir)) { + console.log(`Configuration found in subdirectory: ${pathWithSubdir}`); + targetsPath = pathWithSubdir; + } else if (fs.existsSync(pathAtRoot)) { + console.log(`Configuration found at root: ${pathAtRoot}`); + targetsPath = pathAtRoot; + } else { + // If neither exists, fail with a clear error + throw new Error(`MACHINES.json not found.\nChecked:\n1. ${pathWithSubdir}\n2. ${pathAtRoot}`); } - targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); - } catch (err) { - core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); - return; - } - // Build matrix: machine, firmware - const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); - core.setOutput('build_matrix', JSON.stringify(build_matrix)); - console.log("Build Matrix:", build_matrix); - - // Full matrix: machine, firmware, lavaname - const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); - core.setOutput('full_matrix', JSON.stringify(full_matrix)); - console.log("Full Matrix:", full_matrix); + + // 3. Read and Parse the JSON + const machines = JSON.parse(fs.readFileSync(targetsPath, 'utf8')); + + // 4. Logic for Filtering (Pre-Merge vs Post-Merge) + // We define the allow-list here since we aren't modifying the JSON file + const PREMERGE_ALLOW_LIST = ['qcs6490-rb3gen2']; + + let filtered_targets = []; + const workflowName = process.env.GITHUB_WORKFLOW || ''; + + // Check if this is a Pre-Merge workflow + if (workflowName.includes('pre_merge')) { + console.log(`Workflow '${workflowName}' detected as Pre-Merge.`); + console.log(`Filtering targets. Allowed: ${PREMERGE_ALLOW_LIST.join(', ')}`); + + filtered_targets = machines.filter(m => PREMERGE_ALLOW_LIST.includes(m.target)); + } else { + console.log(`Workflow '${workflowName}' detected as Post-Merge (or other).`); + console.log('Using ALL targets from MACHINES.json.'); + + filtered_targets = machines; + } + + // 5. Construct the Matrices + const build_matrix = { + include: filtered_targets.map(m => ({ + target: m.target, + firmware: m.firmware + })) + }; + + const full_matrix = { + include: filtered_targets + }; + + // 6. Set Outputs + console.log('::set-output name=build_matrix::' + JSON.stringify(build_matrix)); + console.log('::set-output name=full_matrix::' + JSON.stringify(full_matrix)); + + } catch (error) { + console.error(error.message); + process.exit(1); + } \ No newline at end of file diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index 38a17cfe6..c4a10162b 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -5,18 +5,17 @@ description: | on: schedule: - # Every Monday and Thursday at 02:30 UTC (8:00 AM IST) - - cron: "30 2 * * 1,4" + - cron: "0 * * * *" workflow_dispatch: jobs: loading: - uses: qualcomm-linux/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 secrets: inherit build: needs: loading - uses: qualcomm-linux/video-driver/.github/workflows/sync-and-build.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/sync-and-build.yml@video.qclinux.0.0 secrets: inherit with: docker_image: kmake-image:ver.1.0 @@ -24,7 +23,7 @@ jobs: lava-test: needs: [loading, build] - uses: qualcomm-linux/video-driver/.github/workflows/test.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/test.yml@video.qclinux.0.0 secrets: inherit with: docker_image: kmake-image:ver.1.0 diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml index f4c06559c..3bffd54c9 100644 --- a/.github/workflows/pre_merge.yml +++ b/.github/workflows/pre_merge.yml @@ -15,12 +15,12 @@ on: jobs: loading: - uses: qualcomm-linux/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 secrets: inherit build: needs: loading - uses: qualcomm-linux/video-driver/.github/workflows/sync-and-build.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/sync-and-build.yml@video.qclinux.0.0 secrets: inherit with: docker_image: kmake-image:ver.1.0 @@ -28,9 +28,10 @@ jobs: lava-test: needs: [loading, build] - uses: qualcomm-linux/video-driver/.github/workflows/test.yml@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/workflows/test.yml@video.qclinux.0.0 secrets: inherit with: docker_image: kmake-image:ver.1.0 build_matrix: ${{ needs.loading.outputs.build_matrix }} + full_matrix: ${{ needs.loading.outputs.full_matrix }} diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index a50c954d3..15c33bf34 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -1,290 +1,283 @@ -name: Sync and Build - -on: - workflow_dispatch: - workflow_call: - inputs: - docker_image: - description: Docker image to use for the build - required: false - type: string - default: kmake-image:ver.1.0 - build_matrix: - description: Build matrix for multi target builds - type: string - required: true - -permissions: - packages: read - -jobs: - sync-and-build: - runs-on: - group: GHA-video-Prd-SelfHosted-RG - labels: [self-hosted, video-prd-u2204-x64-large-od-ephem] - - steps: - - name: Pull Docker image - uses: qualcomm-linux/kernel-config/.github/actions/pull_docker_image@main - with: - image: ${{ inputs.docker_image }} - - - name: Sync codebase - uses: qualcomm-linux/video-driver/.github/actions/sync@video.qclinux.0.0 - with: - event_name: ${{ github.event_name }} - pr_ref: ${{ github.event.pull_request.head.ref }} - pr_repo: ${{ github.event.pull_request.head.repo.full_name }} - base_ref: ${{ github.ref_name }} - caller_workflow: build - - - name: Build workspace - uses: qualcomm-linux/video-driver/.github/actions/build@video.qclinux.0.0 - with: - docker_image: kmake-image:ver.1.0 - workspace_path: ${{ github.workspace }} - - - name: Download iris_test_app from the s3 - shell: bash - run: | - set -euo pipefail - mkdir -p "${{github.workspace }}/v4l-video-test-app/build/" - echo " syncing files from s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/iris_test_app/" - aws s3 sync "s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/iris_test_app/" "${{ github.workspace }}/v4l-video-test-app/build/" - echo " โœ… Download complete" - ls ${{ github.workspace }}/v4l-video-test-app/build/ - - - name: Download firmware file from S3 - shell: bash - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ“ฅ Syncing files from S3 path: s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/" - aws s3 sync "s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/" "${{ github.workspace }}/downloads" - echo "โœ… Download complete" - [ -f "${{ github.workspace }}/downloads/vpu20_1v.mbn" ] || { echo "โŒ Missing vpu20_1v.mbn"; exit 1; } - - - name: Download the video-contents for testing - shell: bash - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/downloads" - echo "Downloading the video-content files" - wget -q https://github.com/qualcomm-linux/qcom-linux-testkit/releases/download/IRIS-Video-Files-v1.0/video_clips_iris.tar.gz \ - -O "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" - [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video_clips_iris.tar.gz"; exit 1; } - - - name: Prepare /data/vendor/iris_test_app and list contents - shell: bash - run: | - set -euo pipefail - data_dir="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app" - mkdir -p "$data_dir" - data_dir2="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" - mkdir -p "$data_dir2" - firmware_version=$(ls kobj/tar-install/lib/modules/) - mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" - cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ - # Copy test app, firmware blob, and video clips tar into data/vendor/iris_test_app - cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" - cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" - #cp video-driver/iris_vpu.ko "$data_dir/" - cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" - - echo "๐Ÿ“‚ Contents of $data_dir:" - ls -lh "$data_dir" - - - name: Create compressed kernel ramdisk archives - shell: bash - run: | - set -euo pipefail - cd "${{ github.workspace }}/kobj/tar-install" - find lib/modules data | cpio -o -H newc --owner=0:0 | gzip -9 > "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - cd - > /dev/null - ls -lh "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - - - name: Download meta-qcom stable initramfs artifacts from S3 - shell: bash - run: | - set -euo pipefail - mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ” Fetching initramfs files from S3 bucket: s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/" - aws s3 cp s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" - echo "Initramfs files downloaded to: ${{ github.workspace }}/downloads" - - - name: Decompress ramdisk files and rename .cpio.gz files - shell: bash - run: | - set -euo pipefail - cd "${{ github.workspace }}/downloads" - echo " Decompressing and renaming .cpio.gz files..." - gunzip -c initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz > kerneltest.cpio - - - name: Merge and repackage initramfs - shell: bash - run: | - set -euo pipefail - echo "๐Ÿ”ง Starting repackaging process" - - workspace="${{ github.workspace }}" - mkdir -p "$workspace/combineramdisk" - cp "$workspace/local-kernel-ramdisk.cpio.gz" "$workspace/combineramdisk/" - cd "$workspace/combineramdisk" - - # Decompress local-kernel-ramdisk - mv local-kernel-ramdisk.cpio.gz local-kernel-ramdisk.cpio.gz.bak - gunzip -c local-kernel-ramdisk.cpio.gz.bak > local-kernel-ramdisk.cpio - - # Copy kerneltest from downloads - cp "$workspace/downloads/kerneltest.cpio" . - - # Merge kerneltest and local-kernel-ramdisk - cat kerneltest.cpio local-kernel-ramdisk.cpio > video-merged.cpio - gzip -9 video-merged.cpio - - # Create temp workspace to clean up archive - mkdir -p temp_merge - cd temp_merge - cpio -id --no-absolute-filenames < ../kerneltest.cpio - cpio -id --no-absolute-filenames < ../local-kernel-ramdisk.cpio - cd .. - - # Remove old merged archive - rm -f video-merged.cpio.gz - - # Repackage clean archive - cd temp_merge - find . | cpio -o -H newc --owner=0:0 > ../video-merged.cpio - cd .. - gzip -9 video-merged.cpio - - # Cleanup - rm -rf temp_merge kerneltest.cpio local-kernel-ramdisk.cpio - echo "Final archive: $workspace/combineramdisk/video-merged.cpio.gz" - ls -lh "$workspace/combineramdisk/video-merged.cpio.gz" - - - name: Validate build_matrix and jq - shell: bash - run: | - set -euo pipefail - machines_json='${{ inputs.build_matrix }}' - if ! command -v jq >/dev/null 2>&1; then - echo "โŒ jq is not installed on this runner. Please install jq." - exit 1 - fi - echo "$machines_json" | jq -e . >/dev/null - [ "$(echo "$machines_json" | jq length)" -gt 0 ] || { echo "โŒ build_matrix is empty"; exit 1; } - echo "โœ… build_matrix is valid JSON" - - - name: Append artifacts to S3 upload list - shell: bash - run: | - set -euo pipefail - workspace="${{ github.workspace }}" - file_list="$workspace/artifacts/file_list.txt" - mkdir -p "$workspace/artifacts" - - # Fresh file_list - : > "$file_list" - - # Package lib/modules (xz-compressed) โ€” exclude risky symlinks - mod_root="$workspace/kobj/tar-install/lib/modules" - [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } - tar -C "$workspace/kobj/tar-install" \ - --exclude='lib/modules/*/build' \ - --exclude='lib/modules/*/source' \ - --numeric-owner --owner=0 --group=0 \ - -cJf "$workspace/modules.tar.xz" lib/modules - - # Safety checks on the tar - if tar -Jtvf "$workspace/modules.tar.xz" | grep -q ' -> '; then - echo "โŒ Symlinks found in modules archive (should be none)"; exit 1 - fi - if tar -Jtf "$workspace/modules.tar.xz" | grep -Eq '^/|(^|/)\.\.(/|$)'; then - echo "โŒ Unsafe paths found in modules archive"; exit 1 - fi - - echo "$workspace/modules.tar.xz" >> "$file_list" - echo "โœ… Queued for upload: $workspace/modules.tar.xz" - - # Kernel Image + merged video ramdisk (no local ramdisk) - IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" - VMLINUX_PATH="$workspace/kobj/vmlinux" - MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" - - [ -f "$IMAGE_PATH" ] || { echo "โŒ Missing expected file: $IMAGE_PATH"; exit 1; } - [ -f "$VMLINUX_PATH" ] || { echo "โŒ Missing expected file: $VMLINUX_PATH"; exit 1; } - [ -f "$MERGED_PATH" ] || { echo "โŒ Missing merged cpio: $MERGED_PATH"; exit 1; } - - echo "$IMAGE_PATH" >> "$file_list" - echo "โœ… Queued for upload: $IMAGE_PATH" - echo "$VMLINUX_PATH" >> "$file_list" - echo "โœ… Queued for upload: $VMLINUX_PATH" - echo "$MERGED_PATH" >> "$file_list" - echo "โœ… Queued for upload: $MERGED_PATH" - - # Loop through all machines from the build_matrix input and add DTBs - machines='${{ inputs.build_matrix }}' - for machine in $(echo "$machines" | jq -r '.[].machine'); do - dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" - if [ -f "$dtb" ]; then - echo "$dtb" >> "$file_list" - echo "โœ… Queued for upload: $dtb" - else - echo "โŒ Missing DTB: $dtb" - exit 1 - fi - done - - echo "----- Files queued for S3 upload -----" - cat "$file_list" - - - name: Upload all artifacts to S3 - uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 - with: - s3_bucket: qli-prd-video-gh-artifacts - local_file: ${{ github.workspace }}/artifacts/file_list.txt - mode: multi-upload - upload_location: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ github.run_id }}-${{ github.run_attempt }} - - - - name: Clean up - if: always() - shell: bash - run: | - set -euo pipefail - ws="${{ github.workspace }}" - rm -rf "$ws/artqifacts" || true - rm -rf "$ws/combineramdisk" || true - rm -rf "$ws/downloads" || true - rm -rf "$ws/kobj" || true - rm -f "$ws/modules.tar.xz" || true - rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true - - - - name: Update summary - if: success() || failure() - shell: bash - run: | - status="${{ steps.build_workspace.outcome }}" - if [ "$status" = "success" ]; then - summary=":heavy_check_mark: Build Success" - else - summary=":x: Build Failed" - fi - - ws="${{ github.workspace }}" - file_list="$ws/artifacts/file_list.txt" - - { - echo "
Build Summary" - echo "$summary" - if [ -f "$file_list" ]; then - echo "" - echo "Artifacts queued for upload:" - while IFS= read -r line; do - echo "- $line" - done < "$file_list" - fi - echo "
" - } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file +name: Sync and Build + +on: + workflow_dispatch: + workflow_call: + inputs: + docker_image: + description: Docker image to use for the build + required: false + type: string + default: kmake-image:ver.1.0 + build_matrix: + description: Build matrix for multi target builds + type: string + required: true + +permissions: + packages: read + +jobs: + sync-and-build: + runs-on: + group: GHA-video-Stg-SelfHosted-RG + labels: [self-hosted, video-stg-u2204-x64-large-od-ephem] + + steps: + - name: Pull Docker image + uses: qualcomm-linux-stg/kernel-config/.github/actions/pull_docker_image@main + with: + image: ${{ inputs.docker_image }} + + - name: Sync codebase + uses: qualcomm-linux-stg/video-driver/.github/actions/sync@video.qclinux.0.0 + with: + event_name: ${{ github.event_name }} + pr_ref: ${{ github.event.pull_request.head.ref }} + pr_repo: ${{ github.event.pull_request.head.repo.full_name }} + base_ref: ${{ github.ref_name }} + caller_workflow: build + + - name: Build workspace + uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 + with: + docker_image: kmake-image:ver.1.0 + workspace_path: ${{ github.workspace }} + + - name: Generate Iris test app + uses: qualcomm-linux-stg/video-driver/.github/actions/generate_iris_test_app@video.qclinux.0.0 + with: + docker_image: kmake-image:ver.1.0 + + - name: Download firmware file from S3 + shell: bash + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/downloads" + echo "๐Ÿ“ฅ Syncing files from S3 path: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" + aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" "${{ github.workspace }}/downloads" + echo "โœ… Download complete" + [ -f "${{ github.workspace }}/downloads/vpu20_1v.mbn" ] || { echo "โŒ Missing vpu20_1v.mbn"; exit 1; } + + - name: Download the video-contents for testing + shell: bash + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/downloads" + echo "Downloading the video-content files" + wget -q https://github.com/qualcomm-linux/qcom-linux-testkit/releases/download/IRIS-Video-Files-v1.0/video_clips_iris.tar.gz \ + -O "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" + [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video_clips_iris.tar.gz"; exit 1; } + + - name: Prepare /data/vendor/iris_test_app and list contents + shell: bash + run: | + set -euo pipefail + data_dir="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app" + mkdir -p "$data_dir" + firmware_version=$(ls kobj/tar-install/lib/modules/) + mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" + cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ + # Copy test app, firmware blob, and video clips tar into data/vendor/iris_test_app + cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" + cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir/" + #cp video-driver/iris_vpu.ko "$data_dir/" + cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" + + echo "๐Ÿ“‚ Contents of $data_dir:" + ls -lh "$data_dir" + + - name: Create compressed kernel ramdisk archives for all machines + shell: bash + run: | + set -euo pipefail + cd "${{ github.workspace }}/kobj/tar-install" + find lib/modules data | cpio -o -H newc --owner=0:0 | gzip -9 > "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" + cd - > /dev/null + ls -lh "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" + + - name: Download meta-qcom stable initramfs artifacts from S3 + shell: bash + run: | + set -euo pipefail + mkdir -p "${{ github.workspace }}/downloads" + echo "๐Ÿ” Fetching initramfs files from S3 bucket: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/" + aws s3 cp s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" + echo " All initramfs files downloaded to: ${{ github.workspace }}/downloads" + + - name: Decompress ramdisk files and rename .cpio.gz files + shell: bash + run: | + set -euo pipefail + cd "${{ github.workspace }}/downloads" + echo " Decompressing and renaming .cpio.gz files..." + gunzip -c initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz > kerneltest.cpio + + - name: Merge and repackage initramfs + shell: bash + run: | + set -euo pipefail + echo "๐Ÿ”ง Starting repackaging process" + + workspace="${{ github.workspace }}" + mkdir -p "$workspace/combineramdisk" + cp "$workspace/local-kernel-ramdisk.cpio.gz" "$workspace/combineramdisk/" + cd "$workspace/combineramdisk" + + # Decompress local-kernel-ramdisk + mv local-kernel-ramdisk.cpio.gz local-kernel-ramdisk.cpio.gz.bak + gunzip -c local-kernel-ramdisk.cpio.gz.bak > local-kernel-ramdisk.cpio + + # Copy kerneltest from downloads + cp "$workspace/downloads/kerneltest.cpio" . + + # Merge kerneltest and local-kernel-ramdisk + cat kerneltest.cpio local-kernel-ramdisk.cpio > video-merged.cpio + gzip -9 video-merged.cpio + + # Create temp workspace to clean up archive + mkdir -p temp_merge + cd temp_merge + cpio -id --no-absolute-filenames < ../kerneltest.cpio + cpio -id --no-absolute-filenames < ../local-kernel-ramdisk.cpio + cd .. + + # Remove old merged archive + rm -f video-merged.cpio.gz + + # Repackage clean archive + cd temp_merge + find . | cpio -o -H newc --owner=0:0 > ../video-merged.cpio + cd .. + gzip -9 video-merged.cpio + + # Cleanup + rm -rf temp_merge kerneltest.cpio local-kernel-ramdisk.cpio + echo "Final archive: $workspace/combineramdisk/video-merged.cpio.gz" + ls -lh "$workspace/combineramdisk/video-merged.cpio.gz" + + - name: Validate build_matrix and jq + shell: bash + run: | + set -euo pipefail + machines_json='${{ inputs.build_matrix }}' + if ! command -v jq >/dev/null 2>&1; then + echo "โŒ jq is not installed on this runner. Please install jq." + exit 1 + fi + echo "$machines_json" | jq -e . >/dev/null + [ "$(echo "$machines_json" | jq length)" -gt 0 ] || { echo "โŒ build_matrix is empty"; exit 1; } + echo "โœ… build_matrix is valid JSON" + + - name: Append artifacts to S3 upload list + shell: bash + run: | + set -euo pipefail + workspace="${{ github.workspace }}" + file_list="$workspace/artifacts/file_list.txt" + mkdir -p "$workspace/artifacts" + + # Fresh file_list + : > "$file_list" + + # Package lib/modules (xz-compressed) โ€” exclude risky symlinks + mod_root="$workspace/kobj/tar-install/lib/modules" + [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } + tar -C "$workspace/kobj/tar-install" \ + --exclude='lib/modules/*/build' \ + --exclude='lib/modules/*/source' \ + --numeric-owner --owner=0 --group=0 \ + -cJf "$workspace/modules.tar.xz" lib/modules + + # Safety checks on the tar + if tar -Jtvf "$workspace/modules.tar.xz" | grep -q ' -> '; then + echo "โŒ Symlinks found in modules archive (should be none)"; exit 1 + fi + if tar -Jtf "$workspace/modules.tar.xz" | grep -Eq '^/|(^|/)\.\.(/|$)'; then + echo "โŒ Unsafe paths found in modules archive"; exit 1 + fi + + echo "$workspace/modules.tar.xz" >> "$file_list" + echo "โœ… Queued for upload: $workspace/modules.tar.xz" + + # Kernel Image + merged video ramdisk (no local ramdisk) + IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" + VMLINUX_PATH="$workspace/kobj/vmlinux" + MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" + + [ -f "$IMAGE_PATH" ] || { echo "โŒ Missing expected file: $IMAGE_PATH"; exit 1; } + [ -f "$MERGED_PATH" ] || { echo "โŒ Missing merged cpio: $MERGED_PATH"; exit 1; } + + echo "$IMAGE_PATH" >> "$file_list" + echo "โœ… Queued for upload: $IMAGE_PATH" + echo "โœ… Queued for upload: $IMAGE_PATH" + echo "$VMLINUX_PATH" >> "$file_list" + echo "โœ… Queued for upload: $VMLINUX_PATH" + echo "$MERGED_PATH" >> "$file_list" + echo "โœ… Queued for upload: $MERGED_PATH" + + # Loop through all machines from the build_matrix input and add DTBs + machines='${{ inputs.build_matrix }}' + for machine in $(echo "$machines" | jq -r '.[].machine'); do + dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" + if [ -f "$dtb" ]; then + echo "$dtb" >> "$file_list" + echo "โœ… Queued for upload: $dtb" + else + echo "โŒ Missing DTB: $dtb" + exit 1 + fi + done + + echo "----- Files queued for S3 upload -----" + cat "$file_list" + + - name: Upload all artifacts to S3 + uses: qualcomm-linux-stg/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 + with: + s3_bucket: qli-stg-video-gh-artifacts + local_file: ${{ github.workspace }}/artifacts/file_list.txt + mode: multi-upload + upload_location: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ github.run_id }}-${{ github.run_attempt }} + + + - name: Clean up + if: always() + shell: bash + run: | + set -euo pipefail + ws="${{ github.workspace }}" + rm -rf "$ws/artqifacts" || true + rm -rf "$ws/combineramdisk" || true + rm -rf "$ws/downloads" || true + rm -rf "$ws/kobj" || true + rm -f "$ws/modules.tar.xz" || true + rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true + + + - name: Update summary + if: success() || failure() + shell: bash + run: | + status="${{ steps.build_workspace.outcome }}" + if [ "$status" = "success" ]; then + summary=":heavy_check_mark: Build Success" + else + summary=":x: Build Failed" + fi + + ws="${{ github.workspace }}" + file_list="$ws/artifacts/file_list.txt" + + { + echo "
Build Summary" + echo "$summary" + if [ -f "$file_list" ]; then + echo "" + echo "Artifacts queued for upload:" + while IFS= read -r line; do + echo "- $line" + done < "$file_list" + fi + echo "
" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 505eb418b..b86bdd60d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,8 +23,8 @@ on: jobs: test: runs-on: - group: GHA-video-Prd-SelfHosted-RG - labels: [ self-hosted, video-prd-u2204-x64-large-od-ephem ] + group: GHA-video-Stg-SelfHosted-RG + labels: [ self-hosted, video-stg-u2204-x64-large-od-ephem ] strategy: fail-fast: false matrix: @@ -72,7 +72,7 @@ jobs: - name: Create lava job definition id: create_job_definition - uses: qualcomm-linux/video-driver/.github/actions/lava_job_render@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/actions/lava_job_render@video.qclinux.0.0 with: docker_image: ${{ inputs.docker_image }} env: @@ -196,4 +196,4 @@ jobs: ' echo -e "$SUMMARY" >> $GITHUB_STEP_SUMMARY # JOB_INFO="- [Job $job_id on ${{ matrix.build_matrix.machine }}]($job_url)" - # echo -e "$JOB_INFO" > job_info.md \ No newline at end of file + # echo -e "$JOB_INFO" > job_info.md From 0df268a8fcf8d5cfcec415ddb113abd70f116b87 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 15:54:40 +0530 Subject: [PATCH 02/26] updated the sync and build logic --- .github/actions/loading/action.yml | 118 +++++++++++++++------------ .github/workflows/sync-and-build.yml | 31 ++++--- .github/workflows/test.yml | 2 +- 3 files changed, 84 insertions(+), 67 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 7dd19485f..deceef9cc 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -21,68 +21,78 @@ runs: script: | const fs = require('fs'); const path = require('path'); + const core = require('@actions/core'); try { - // 1. Define both possible paths for MACHINES.json - // Path 1: Inside a 'video-driver' subdirectory (Common in Pre-Merge/PRs) - const pathWithSubdir = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - // Path 2: Directly at the root (Common in Post-Merge/Scheduled runs) - const pathAtRoot = path.join(process.env.GITHUB_WORKSPACE, 'ci', 'MACHINES.json'); - - let targetsPath; - - // 2. Check which path actually exists - if (fs.existsSync(pathWithSubdir)) { - console.log(`Configuration found in subdirectory: ${pathWithSubdir}`); - targetsPath = pathWithSubdir; - } else if (fs.existsSync(pathAtRoot)) { - console.log(`Configuration found at root: ${pathAtRoot}`); - targetsPath = pathAtRoot; - } else { - // If neither exists, fail with a clear error - throw new Error(`MACHINES.json not found.\nChecked:\n1. ${pathWithSubdir}\n2. ${pathAtRoot}`); - } + // 1. Locate MACHINES.json dynamically + // Checks if the code is in the root workspace or a 'video-driver' subdirectory + // This handles differences between 'pull_request' (merged code) and 'schedule' (clean checkout) + const workspace = process.env.GITHUB_WORKSPACE; + const pathInSubdir = path.join(workspace, 'video-driver', 'ci', 'MACHINES.json'); + const pathAtRoot = path.join(workspace, 'ci', 'MACHINES.json'); + + let jsonPath; - // 3. Read and Parse the JSON - const machines = JSON.parse(fs.readFileSync(targetsPath, 'utf8')); - - // 4. Logic for Filtering (Pre-Merge vs Post-Merge) - // We define the allow-list here since we aren't modifying the JSON file - const PREMERGE_ALLOW_LIST = ['qcs6490-rb3gen2']; - - let filtered_targets = []; - const workflowName = process.env.GITHUB_WORKFLOW || ''; - - // Check if this is a Pre-Merge workflow - if (workflowName.includes('pre_merge')) { - console.log(`Workflow '${workflowName}' detected as Pre-Merge.`); - console.log(`Filtering targets. Allowed: ${PREMERGE_ALLOW_LIST.join(', ')}`); - - filtered_targets = machines.filter(m => PREMERGE_ALLOW_LIST.includes(m.target)); + if (fs.existsSync(pathAtRoot)) { + jsonPath = pathAtRoot; + } else if (fs.existsSync(pathInSubdir)) { + jsonPath = pathInSubdir; } else { - console.log(`Workflow '${workflowName}' detected as Post-Merge (or other).`); - console.log('Using ALL targets from MACHINES.json.'); - - filtered_targets = machines; + // Fail if we can't find the config file + throw new Error(`MACHINES.json not found. Checked: \n1. ${pathAtRoot} \n2. ${pathInSubdir}`); } - // 5. Construct the Matrices - const build_matrix = { - include: filtered_targets.map(m => ({ - target: m.target, - firmware: m.firmware - })) - }; + console.log(`Found MACHINES.json at: ${jsonPath}`); + + // 2. Read and Parse JSON + const rawData = fs.readFileSync(jsonPath); + const machines = JSON.parse(rawData); + + // 3. Determine if we should filter (Check workflow name) + // If the workflow name contains 'pre_merge', we filter. Otherwise (post_merge), we run everything. + const workflowName = process.env.GITHUB_WORKFLOW || ""; + const isPreMerge = workflowName.toLowerCase().includes("pre_merge"); + + console.log(`Current Workflow: ${workflowName}`); + console.log(`Mode: ${isPreMerge ? "PRE-MERGE (Filtering for 'premerge': true)" : "POST-MERGE (Running all targets)"}`); + + let buildMatrix = []; + let fullMatrix = []; + + // 4. Generate Matrices + machines.forEach(machine => { + // FILTER LOGIC: + // If we are in pre_merge mode, skip any machine that does NOT have "premerge": true + if (isPreMerge && machine.premerge !== true) { + return; + } + + // Add to Build Matrix (Common) + buildMatrix.push({ + target: machine.name, + arch: machine.arch, + variant: machine.variant + }); + + // Add to Full Matrix + fullMatrix.push({ + target: machine.name, + arch: machine.arch, + variant: machine.variant, + tests: machine.tests || [] + }); + }); + + // 5. Output to GitHub Actions + if (buildMatrix.length === 0) { + console.warn("WARNING: Matrix is empty! Check if MACHINES.json has 'premerge': true flags."); + } - const full_matrix = { - include: filtered_targets - }; + core.setOutput('build_matrix', JSON.stringify(buildMatrix)); + core.setOutput('full_matrix', JSON.stringify(fullMatrix)); - // 6. Set Outputs - console.log('::set-output name=build_matrix::' + JSON.stringify(build_matrix)); - console.log('::set-output name=full_matrix::' + JSON.stringify(full_matrix)); + console.log(`Generated matrix with ${buildMatrix.length} targets.`); } catch (error) { - console.error(error.message); - process.exit(1); + core.setFailed(error.message); } \ No newline at end of file diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 15c33bf34..01fe08c2c 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Pull Docker image - uses: qualcomm-linux-stg/kernel-config/.github/actions/pull_docker_image@main + uses: qualcomm-linux/kernel-config/.github/actions/pull_docker_image@main with: image: ${{ inputs.docker_image }} @@ -44,10 +44,15 @@ jobs: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} - - name: Generate Iris test app - uses: qualcomm-linux-stg/video-driver/.github/actions/generate_iris_test_app@video.qclinux.0.0 - with: - docker_image: kmake-image:ver.1.0 + - name: Download iris_test_app from the s3 + shell: bash + run: | + set -euo pipefail + mkdir -p "${{github.workspace }}/v4l-video-test-app/build/" + echo " syncing files from s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" + aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" "${{ github.workspace }}/v4l-video-test-app/build/" + echo " โœ… Download complete" + ls ${{ github.workspace }}/v4l-video-test-app/build/ - name: Download firmware file from S3 shell: bash @@ -75,19 +80,21 @@ jobs: set -euo pipefail data_dir="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app" mkdir -p "$data_dir" + data_dir2="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" + mkdir -p "$data_dir2" firmware_version=$(ls kobj/tar-install/lib/modules/) mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ # Copy test app, firmware blob, and video clips tar into data/vendor/iris_test_app cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" - cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir/" + cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" #cp video-driver/iris_vpu.ko "$data_dir/" cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" echo "๐Ÿ“‚ Contents of $data_dir:" ls -lh "$data_dir" - - name: Create compressed kernel ramdisk archives for all machines + - name: Create compressed kernel ramdisk archives shell: bash run: | set -euo pipefail @@ -95,7 +102,7 @@ jobs: find lib/modules data | cpio -o -H newc --owner=0:0 | gzip -9 > "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" cd - > /dev/null ls -lh "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - + - name: Download meta-qcom stable initramfs artifacts from S3 shell: bash run: | @@ -103,7 +110,7 @@ jobs: mkdir -p "${{ github.workspace }}/downloads" echo "๐Ÿ” Fetching initramfs files from S3 bucket: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/" aws s3 cp s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" - echo " All initramfs files downloaded to: ${{ github.workspace }}/downloads" + echo "Initramfs files downloaded to: ${{ github.workspace }}/downloads" - name: Decompress ramdisk files and rename .cpio.gz files shell: bash @@ -206,11 +213,11 @@ jobs: MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" [ -f "$IMAGE_PATH" ] || { echo "โŒ Missing expected file: $IMAGE_PATH"; exit 1; } + [ -f "$VMLINUX_PATH" ] || { echo "โŒ Missing expected file: $VMLINUX_PATH"; exit 1; } [ -f "$MERGED_PATH" ] || { echo "โŒ Missing merged cpio: $MERGED_PATH"; exit 1; } echo "$IMAGE_PATH" >> "$file_list" echo "โœ… Queued for upload: $IMAGE_PATH" - echo "โœ… Queued for upload: $IMAGE_PATH" echo "$VMLINUX_PATH" >> "$file_list" echo "โœ… Queued for upload: $VMLINUX_PATH" echo "$MERGED_PATH" >> "$file_list" @@ -233,7 +240,7 @@ jobs: cat "$file_list" - name: Upload all artifacts to S3 - uses: qualcomm-linux-stg/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 + uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 with: s3_bucket: qli-stg-video-gh-artifacts local_file: ${{ github.workspace }}/artifacts/file_list.txt @@ -280,4 +287,4 @@ jobs: done < "$file_list" fi echo "" - } >> "$GITHUB_STEP_SUMMARY" + } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b86bdd60d..e210b773e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -196,4 +196,4 @@ jobs: ' echo -e "$SUMMARY" >> $GITHUB_STEP_SUMMARY # JOB_INFO="- [Job $job_id on ${{ matrix.build_matrix.machine }}]($job_url)" - # echo -e "$JOB_INFO" > job_info.md + # echo -e "$JOB_INFO" > job_info.md \ No newline at end of file From f6c6a07991dbfdb443954e9f8f8efb96b24f5d3b Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 17:18:01 +0530 Subject: [PATCH 03/26] updated loading action --- .github/actions/loading/action.yml | 57 ++++++++++++------------------ 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index deceef9cc..9a8df2dfc 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -25,8 +25,7 @@ runs: try { // 1. Locate MACHINES.json dynamically - // Checks if the code is in the root workspace or a 'video-driver' subdirectory - // This handles differences between 'pull_request' (merged code) and 'schedule' (clean checkout) + // Checks both root and subdirectory to handle different checkout depths const workspace = process.env.GITHUB_WORKSPACE; const pathInSubdir = path.join(workspace, 'video-driver', 'ci', 'MACHINES.json'); const pathAtRoot = path.join(workspace, 'ci', 'MACHINES.json'); @@ -38,60 +37,48 @@ runs: } else if (fs.existsSync(pathInSubdir)) { jsonPath = pathInSubdir; } else { - // Fail if we can't find the config file throw new Error(`MACHINES.json not found. Checked: \n1. ${pathAtRoot} \n2. ${pathInSubdir}`); } - console.log(`Found MACHINES.json at: ${jsonPath}`); + console.log(`Loading targets from: ${jsonPath}`); // 2. Read and Parse JSON const rawData = fs.readFileSync(jsonPath); - const machines = JSON.parse(rawData); - - // 3. Determine if we should filter (Check workflow name) - // If the workflow name contains 'pre_merge', we filter. Otherwise (post_merge), we run everything. - const workflowName = process.env.GITHUB_WORKFLOW || ""; - const isPreMerge = workflowName.toLowerCase().includes("pre_merge"); - - console.log(`Current Workflow: ${workflowName}`); - console.log(`Mode: ${isPreMerge ? "PRE-MERGE (Filtering for 'premerge': true)" : "POST-MERGE (Running all targets)"}`); + const machinesMap = JSON.parse(rawData); + + // Convert Dictionary to Array (since your JSON is an Object of Objects) + const machines = Object.values(machinesMap); let buildMatrix = []; let fullMatrix = []; - // 4. Generate Matrices - machines.forEach(machine => { - // FILTER LOGIC: - // If we are in pre_merge mode, skip any machine that does NOT have "premerge": true - if (isPreMerge && machine.premerge !== true) { - return; - } + // 3. Generate Matrices (No Filtering) + machines.forEach(entry => { + console.log(`Adding target: ${entry.target}`); - // Add to Build Matrix (Common) + // Add to Build Matrix (Minimal params for matrix strategy) buildMatrix.push({ - target: machine.name, - arch: machine.arch, - variant: machine.variant + target: entry.target, + machine: entry.machine, + firmware: entry.firmware }); - // Add to Full Matrix + // Add to Full Matrix (All params) fullMatrix.push({ - target: machine.name, - arch: machine.arch, - variant: machine.variant, - tests: machine.tests || [] + target: entry.target, + machine: entry.machine, + firmware: entry.firmware, + lavaname: entry.lavaname, + buildid: entry.buildid, + firmwareid: entry.firmwareid }); }); - // 5. Output to GitHub Actions - if (buildMatrix.length === 0) { - console.warn("WARNING: Matrix is empty! Check if MACHINES.json has 'premerge': true flags."); - } - + // 4. Output to GitHub Actions core.setOutput('build_matrix', JSON.stringify(buildMatrix)); core.setOutput('full_matrix', JSON.stringify(fullMatrix)); - console.log(`Generated matrix with ${buildMatrix.length} targets.`); + console.log(`Successfully generated matrix with ${buildMatrix.length} targets.`); } catch (error) { core.setFailed(error.message); From a431a7095f481b9edc2f8a751cbfe95e2ebf6317 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 19:50:04 +0530 Subject: [PATCH 04/26] Update loading.yml Signed-off-by: Manigurr --- .github/workflows/loading.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml index 9859304ab..901406dbc 100644 --- a/.github/workflows/loading.yml +++ b/.github/workflows/loading.yml @@ -21,7 +21,7 @@ jobs: full_matrix: ${{ steps.loading.outputs.full_matrix }} steps: - name: Sync codebase - uses: qualcomm-linux/video-driver/.github/actions/sync@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/actions/sync@video.qclinux.0.0 with: event_name: ${{ github.event_name }} pr_ref: ${{ github.event.pull_request.head.ref }} @@ -30,4 +30,5 @@ jobs: - name: Load Parameters id: loading - uses: qualcomm-linux/video-driver/.github/actions/loading@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 + From 2f30a5b0eee476581f442d06da6d44e990c215c7 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 20:59:49 +0530 Subject: [PATCH 05/26] fix --- .github/actions/loading/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 9a8df2dfc..87f1bdaa5 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -21,7 +21,7 @@ runs: script: | const fs = require('fs'); const path = require('path'); - const core = require('@actions/core'); + //const core = require('@actions/core'); try { // 1. Locate MACHINES.json dynamically From 7bcd95770c271654edcc4cbfbf3504cdc3b221e7 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 22:00:22 +0530 Subject: [PATCH 06/26] updated the loading logic and cron logic --- .github/actions/loading/action.yml | 135 +++++++++++------------------ .github/workflows/post_merge.yml | 4 +- 2 files changed, 55 insertions(+), 84 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 87f1bdaa5..7f29aeb05 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -1,85 +1,56 @@ ---- -name: Load Parameters -description: Load parameters for the build job - -outputs: - build_matrix: - description: Build matrix - value: ${{ steps.set-matrix.outputs.build_matrix }} - - full_matrix: - description: full matrix containing lava devails - value: ${{ steps.set-matrix.outputs.full_matrix }} - -runs: - using: "composite" - steps: - - name: Set Build Matrix - id: set-matrix - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - //const core = require('@actions/core'); - - try { - // 1. Locate MACHINES.json dynamically - // Checks both root and subdirectory to handle different checkout depths - const workspace = process.env.GITHUB_WORKSPACE; - const pathInSubdir = path.join(workspace, 'video-driver', 'ci', 'MACHINES.json'); - const pathAtRoot = path.join(workspace, 'ci', 'MACHINES.json'); - - let jsonPath; - - if (fs.existsSync(pathAtRoot)) { - jsonPath = pathAtRoot; - } else if (fs.existsSync(pathInSubdir)) { - jsonPath = pathInSubdir; - } else { - throw new Error(`MACHINES.json not found. Checked: \n1. ${pathAtRoot} \n2. ${pathInSubdir}`); +name: loading +description: | + Loads build parameters (build_matrix and full_matrix) from ci/MACHINES.json + for use in subsequent jobs. + +on: + workflow_call: + outputs: + build_matrix: + description: Build matrix + value: ${{ jobs.load-parameters.outputs.build_matrix }} + full_matrix: + description: Full matrix containing lava details + value: ${{ jobs.load-parameters.outputs.full_matrix }} + +jobs: + load-parameters: + runs-on: ubuntu-latest + outputs: + build_matrix: ${{ steps.set-matrix.outputs.build_matrix }} + full_matrix: ${{ steps.set-matrix.outputs.full_matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: video-driver # ensures repo is always at $GITHUB_WORKSPACE/video-driver + + - name: Set Build Matrix + id: set-matrix + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + let targets; + try { + if (!fs.existsSync(targetsPath)) { + core.setFailed(`MACHINES.json not found at ${targetsPath}`); + return; + } + targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); + } catch (err) { + core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); + return; } - console.log(`Loading targets from: ${jsonPath}`); - - // 2. Read and Parse JSON - const rawData = fs.readFileSync(jsonPath); - const machinesMap = JSON.parse(rawData); - - // Convert Dictionary to Array (since your JSON is an Object of Objects) - const machines = Object.values(machinesMap); - - let buildMatrix = []; - let fullMatrix = []; - - // 3. Generate Matrices (No Filtering) - machines.forEach(entry => { - console.log(`Adding target: ${entry.target}`); - - // Add to Build Matrix (Minimal params for matrix strategy) - buildMatrix.push({ - target: entry.target, - machine: entry.machine, - firmware: entry.firmware - }); - - // Add to Full Matrix (All params) - fullMatrix.push({ - target: entry.target, - machine: entry.machine, - firmware: entry.firmware, - lavaname: entry.lavaname, - buildid: entry.buildid, - firmwareid: entry.firmwareid - }); - }); - - // 4. Output to GitHub Actions - core.setOutput('build_matrix', JSON.stringify(buildMatrix)); - core.setOutput('full_matrix', JSON.stringify(fullMatrix)); - - console.log(`Successfully generated matrix with ${buildMatrix.length} targets.`); + // Build matrix: machine, firmware + const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); + core.setOutput('build_matrix', JSON.stringify(build_matrix)); + console.log("Build Matrix:", build_matrix); - } catch (error) { - core.setFailed(error.message); - } \ No newline at end of file + // Full matrix: machine, firmware, lavaname + const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); + core.setOutput('full_matrix', JSON.stringify(full_matrix)); + console.log("Full Matrix:", full_matrix); diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index c4a10162b..ec605c613 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -5,7 +5,7 @@ description: | on: schedule: - - cron: "0 * * * *" + - cron: "*/5 * * * *" workflow_dispatch: jobs: @@ -28,4 +28,4 @@ jobs: with: docker_image: kmake-image:ver.1.0 build_matrix: ${{ needs.loading.outputs.build_matrix }} - full_matrix: ${{ needs.loading.outputs.full_matrix }} \ No newline at end of file + full_matrix: ${{ needs.loading.outputs.full_matrix } \ No newline at end of file From cf85c2686fa7eeca2a24740b6221a07a6a524c92 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 22:06:57 +0530 Subject: [PATCH 07/26] updated --- .github/actions/loading/action.yml | 81 ++++++++++++------------------ 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 7f29aeb05..0483ec16e 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -1,56 +1,37 @@ -name: loading -description: | - Loads build parameters (build_matrix and full_matrix) from ci/MACHINES.json - for use in subsequent jobs. +name: Load Parameters +description: Load parameters for the build job -on: - workflow_call: - outputs: - build_matrix: - description: Build matrix - value: ${{ jobs.load-parameters.outputs.build_matrix }} - full_matrix: - description: Full matrix containing lava details - value: ${{ jobs.load-parameters.outputs.full_matrix }} +outputs: + build_matrix: + description: Build matrix + full_matrix: + description: Full matrix containing lava details -jobs: - load-parameters: - runs-on: ubuntu-latest - outputs: - build_matrix: ${{ steps.set-matrix.outputs.build_matrix }} - full_matrix: ${{ steps.set-matrix.outputs.full_matrix }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - path: video-driver # ensures repo is always at $GITHUB_WORKSPACE/video-driver - - - name: Set Build Matrix - id: set-matrix - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - let targets; - try { - if (!fs.existsSync(targetsPath)) { - core.setFailed(`MACHINES.json not found at ${targetsPath}`); - return; - } - targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); - } catch (err) { - core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); +runs: + using: "composite" + steps: + - name: Set Build Matrix + id: set-matrix + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + let targets; + try { + if (!fs.existsSync(targetsPath)) { + core.setFailed(`MACHINES.json not found at ${targetsPath}`); return; } + targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); + } catch (err) { + core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); + return; + } - // Build matrix: machine, firmware - const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); - core.setOutput('build_matrix', JSON.stringify(build_matrix)); - console.log("Build Matrix:", build_matrix); + const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); + core.setOutput('build_matrix', JSON.stringify(build_matrix)); - // Full matrix: machine, firmware, lavaname - const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); - core.setOutput('full_matrix', JSON.stringify(full_matrix)); - console.log("Full Matrix:", full_matrix); + const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); + core.setOutput('full_matrix', JSON.stringify(full_matrix)) From be63d96acaa273bb5abd01885f6493a6493ad5d8 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 22:39:05 +0530 Subject: [PATCH 08/26] updated typo issue --- .github/workflows/post_merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index ec605c613..22435c8ad 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -28,4 +28,4 @@ jobs: with: docker_image: kmake-image:ver.1.0 build_matrix: ${{ needs.loading.outputs.build_matrix }} - full_matrix: ${{ needs.loading.outputs.full_matrix } \ No newline at end of file + full_matrix: ${{ needs.loading.outputs.full_matrix }} \ No newline at end of file From e26fe35780ecf7225d4ee599235d9a10f841bda8 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 19 Jan 2026 22:56:37 +0530 Subject: [PATCH 09/26] updated post merge logic --- .github/workflows/post_merge.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index 22435c8ad..6c47415f3 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -1,17 +1,26 @@ name: post_merge_weekly description: | - Runs post-merge CI for the video-driver repository on a weekly schedule. + Runs post-merge CI for the video-driver repository on a frequent schedule. Reuses loading, build and test workflows. on: schedule: + # Every 5 minutes (UTC) - cron: "*/5 * * * *" workflow_dispatch: jobs: loading: - uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 - secrets: inherit + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: video-driver # ensures repo is placed at $GITHUB_WORKSPACE/video-driver + + - name: Call loading workflow + uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 + secrets: inherit build: needs: loading @@ -28,4 +37,4 @@ jobs: with: docker_image: kmake-image:ver.1.0 build_matrix: ${{ needs.loading.outputs.build_matrix }} - full_matrix: ${{ needs.loading.outputs.full_matrix }} \ No newline at end of file + full_matrix: ${{ needs.loading.outputs.full_matrix }} From 015a77a51ad8bbf30ddaa2ee727a8ac54bd1437a Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 20 Jan 2026 16:04:00 +0530 Subject: [PATCH 10/26] updated postmerge logic --- .github/workflows/post_merge.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index 6c47415f3..dda78949e 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -11,16 +11,12 @@ on: jobs: loading: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - path: video-driver # ensures repo is placed at $GITHUB_WORKSPACE/video-driver - - - name: Call loading workflow - uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 - secrets: inherit + # Correctly calling the reusable workflow at the Job level + uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 + secrets: inherit + # OPTIONAL: If loading.yml accepts a branch input, uncomment and use this lines below + # with: + # ref: video.qclinux.0.0 build: needs: loading @@ -37,4 +33,4 @@ jobs: with: docker_image: kmake-image:ver.1.0 build_matrix: ${{ needs.loading.outputs.build_matrix }} - full_matrix: ${{ needs.loading.outputs.full_matrix }} + full_matrix: ${{ needs.loading.outputs.full_matrix }} \ No newline at end of file From e71740d6b4b959bcb9729aee426663f4d226e353 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 20 Jan 2026 16:43:41 +0530 Subject: [PATCH 11/26] updated loading logic for the cron job --- .github/actions/loading/action.yml | 93 +++++++++++++++++++----------- .github/workflows/post_merge.yml | 18 +++--- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 0483ec16e..1bdd9dace 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -1,37 +1,62 @@ -name: Load Parameters -description: Load parameters for the build job +name: Load Configuration (Reusable) -outputs: - build_matrix: - description: Build matrix - full_matrix: - description: Full matrix containing lava details +on: + workflow_call: + inputs: + ref: + description: 'Branch to checkout (optional)' + required: false + type: string + default: '' # Defaults to empty so pre_merge uses the current commit + outputs: + build_matrix: + description: "The generated build matrix" + value: ${{ jobs.parse_json.outputs.build_matrix }} + full_matrix: + description: "The full matrix" + value: ${{ jobs.parse_json.outputs.full_matrix }} -runs: - using: "composite" - steps: - - name: Set Build Matrix - id: set-matrix - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const path = require('path'); - const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - let targets; - try { - if (!fs.existsSync(targetsPath)) { - core.setFailed(`MACHINES.json not found at ${targetsPath}`); - return; - } - targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); - } catch (err) { - core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); - return; - } +jobs: + parse_json: + runs-on: ubuntu-latest + outputs: + build_matrix: ${{ steps.generator.outputs.build_matrix }} + full_matrix: ${{ steps.generator.outputs.full_matrix }} + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + # If called from Post Merge, this will be 'video.qclinux.0.0' + # If called from Pre Merge, this is empty (uses current commit) + ref: ${{ inputs.ref }} - const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); - core.setOutput('build_matrix', JSON.stringify(build_matrix)); - - const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); - core.setOutput('full_matrix', JSON.stringify(full_matrix)) + - name: Generate Matrix + id: generator + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // KEPT ORIGINAL PATH: Assumes repo structure has a 'video-driver' folder + const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + + console.log(`Checking path: ${targetsPath}`); + + try { + if (!fs.existsSync(targetsPath)) { + // If this fails, it means the checkout step didn't get the right files + core.setFailed(`MACHINES.json not found at ${targetsPath}`); + return; + } + const targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); + + const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); + core.setOutput('build_matrix', JSON.stringify(build_matrix)); + + const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); + core.setOutput('full_matrix', JSON.stringify(full_matrix)); + + } catch (err) { + core.setFailed(`Script failed: ${err.message}`); + } \ No newline at end of file diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index dda78949e..f075fc746 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -1,22 +1,18 @@ -name: post_merge_weekly -description: | - Runs post-merge CI for the video-driver repository on a frequent schedule. - Reuses loading, build and test workflows. +name: Post Merge Weekly on: schedule: - # Every 5 minutes (UTC) - - cron: "*/5 * * * *" - workflow_dispatch: + - cron: '*/5 * * * *' # Runs every 5 minutes in IST/UTC + workflow_dispatch: # Allows manual testing jobs: loading: - # Correctly calling the reusable workflow at the Job level + # Use the reusable workflow from your specific branch uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 secrets: inherit - # OPTIONAL: If loading.yml accepts a branch input, uncomment and use this lines below - # with: - # ref: video.qclinux.0.0 + with: + # CRITICAL FIX: Force the checkout to use your branch, not the default branch + ref: video.qclinux.0.0 build: needs: loading From 2a3da4e41f414be9f1d374595c60e4dcd58783f8 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 20 Jan 2026 16:54:55 +0530 Subject: [PATCH 12/26] made changes in the loading action.yaml --- .github/actions/loading/action.yml | 98 ++++++++++++------------------ .github/workflows/loading.yml | 17 ++++-- .github/workflows/post_merge.yml | 10 +-- 3 files changed, 56 insertions(+), 69 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 1bdd9dace..9ca420788 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -1,62 +1,42 @@ -name: Load Configuration (Reusable) +name: Load Parameters +description: Load parameters for the build job -on: - workflow_call: - inputs: - ref: - description: 'Branch to checkout (optional)' - required: false - type: string - default: '' # Defaults to empty so pre_merge uses the current commit - outputs: - build_matrix: - description: "The generated build matrix" - value: ${{ jobs.parse_json.outputs.build_matrix }} - full_matrix: - description: "The full matrix" - value: ${{ jobs.parse_json.outputs.full_matrix }} +outputs: + build_matrix: + description: Build matrix + value: ${{ steps.set-matrix.outputs.build_matrix }} + full_matrix: + description: Full matrix containing lava details + value: ${{ steps.set-matrix.outputs.full_matrix }} -jobs: - parse_json: - runs-on: ubuntu-latest - outputs: - build_matrix: ${{ steps.generator.outputs.build_matrix }} - full_matrix: ${{ steps.generator.outputs.full_matrix }} - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - # If called from Post Merge, this will be 'video.qclinux.0.0' - # If called from Pre Merge, this is empty (uses current commit) - ref: ${{ inputs.ref }} +runs: + using: "composite" + steps: + - name: Set Build Matrix + id: set-matrix + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // Checks video-driver/ci/MACHINES.json inside the workspace + const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + + let targets; + try { + if (!fs.existsSync(targetsPath)) { + core.setFailed(`MACHINES.json not found at ${targetsPath}`); + return; + } + targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); + } catch (err) { + core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); + return; + } - - name: Generate Matrix - id: generator - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const path = require('path'); - - // KEPT ORIGINAL PATH: Assumes repo structure has a 'video-driver' folder - const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - - console.log(`Checking path: ${targetsPath}`); - - try { - if (!fs.existsSync(targetsPath)) { - // If this fails, it means the checkout step didn't get the right files - core.setFailed(`MACHINES.json not found at ${targetsPath}`); - return; - } - const targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); - - const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); - core.setOutput('build_matrix', JSON.stringify(build_matrix)); - - const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); - core.setOutput('full_matrix', JSON.stringify(full_matrix)); - - } catch (err) { - core.setFailed(`Script failed: ${err.message}`); - } \ No newline at end of file + const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); + core.setOutput('build_matrix', JSON.stringify(build_matrix)); + + const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname })); + core.setOutput('full_matrix', JSON.stringify(full_matrix)); \ No newline at end of file diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml index 901406dbc..7b3f7e8f2 100644 --- a/.github/workflows/loading.yml +++ b/.github/workflows/loading.yml @@ -1,14 +1,19 @@ ---- name: _loading description: Load required parameters for the subsequent jobs on: workflow_call: + # Added inputs section so post_merge can pass the branch name + inputs: + target_branch: + description: "Branch to checkout (optional)" + required: false + type: string + default: "" outputs: build_matrix: description: Build matrix value: ${{ jobs.loading.outputs.build_matrix }} - full_matrix: description: Full Matrix containing lava description value: ${{ jobs.loading.outputs.full_matrix }} @@ -26,9 +31,11 @@ jobs: event_name: ${{ github.event_name }} pr_ref: ${{ github.event.pull_request.head.ref }} pr_repo: ${{ github.event.pull_request.head.repo.full_name }} - base_ref: ${{ github.ref_name }} + # LOGIC FIX: If target_branch is provided (Post Merge), use it. + # Otherwise use the default ref (Per Merge). + base_ref: ${{ inputs.target_branch || github.ref_name }} - name: Load Parameters id: loading - uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 - + # This calls your composite action (the JS file you provided) + uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 \ No newline at end of file diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index f075fc746..fb84b1907 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -2,17 +2,17 @@ name: Post Merge Weekly on: schedule: - - cron: '*/5 * * * *' # Runs every 5 minutes in IST/UTC - workflow_dispatch: # Allows manual testing + - cron: '*/5 * * * *' + workflow_dispatch: jobs: loading: - # Use the reusable workflow from your specific branch + # We call the reusable workflow uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 secrets: inherit with: - # CRITICAL FIX: Force the checkout to use your branch, not the default branch - ref: video.qclinux.0.0 + # We pass the specific branch we want to build + target_branch: video.qclinux.0.0 build: needs: loading From 7c137ceb95e0e382b60604e12d58e47245067a7f Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 20 Jan 2026 17:51:08 +0530 Subject: [PATCH 13/26] loading action has updated --- .github/actions/loading/action.yml | 43 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/actions/loading/action.yml b/.github/actions/loading/action.yml index 9ca420788..cd564d509 100644 --- a/.github/actions/loading/action.yml +++ b/.github/actions/loading/action.yml @@ -19,22 +19,47 @@ runs: script: | const fs = require('fs'); const path = require('path'); - - // Checks video-driver/ci/MACHINES.json inside the workspace - const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); - + + // 1. Define possible paths for MACHINES.json + // Path A: Workspace/video-driver/ci/MACHINES.json (Nested) + const pathNested = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json'); + // Path B: Workspace/ci/MACHINES.json (Root) + const pathRoot = path.join(process.env.GITHUB_WORKSPACE, 'ci', 'MACHINES.json'); + + let targetsPath = ''; + + // 2. Check which path exists + if (fs.existsSync(pathNested)) { + console.log(`Found config at nested path: ${pathNested}`); + targetsPath = pathNested; + } else if (fs.existsSync(pathRoot)) { + console.log(`Found config at root path: ${pathRoot}`); + targetsPath = pathRoot; + } else { + // 3. Debugging: If neither exists, list files to help us see what is happening + console.log('!!! Error: MACHINES.json not found in expected locations.'); + console.log(`Checked: ${pathNested}`); + console.log(`Checked: ${pathRoot}`); + + console.log('--- Workspace Root Contents ---'); + try { + console.log(fs.readdirSync(process.env.GITHUB_WORKSPACE)); + } catch (e) { console.log(e.message); } + + core.setFailed(`MACHINES.json not found.`); + return; + } + + // 4. Parse the file let targets; try { - if (!fs.existsSync(targetsPath)) { - core.setFailed(`MACHINES.json not found at ${targetsPath}`); - return; - } targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8')); } catch (err) { - core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`); + core.setFailed(`Failed to parse MACHINES.json: ${err.message}`); return; } + // 5. Generate Outputs const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware })); core.setOutput('build_matrix', JSON.stringify(build_matrix)); From 8061b23540fd73fbf8ceb3f5e1b6bb079616dea9 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 20 Jan 2026 19:15:25 +0530 Subject: [PATCH 14/26] workflow action.yaml update --- .github/workflows/loading.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml index 7b3f7e8f2..91c5a1410 100644 --- a/.github/workflows/loading.yml +++ b/.github/workflows/loading.yml @@ -3,7 +3,6 @@ description: Load required parameters for the subsequent jobs on: workflow_call: - # Added inputs section so post_merge can pass the branch name inputs: target_branch: description: "Branch to checkout (optional)" @@ -25,17 +24,32 @@ jobs: build_matrix: ${{ steps.loading.outputs.build_matrix }} full_matrix: ${{ steps.loading.outputs.full_matrix }} steps: - - name: Sync codebase + # SCENARIO 1: Pull Request (Pre-Merge) + # Uses your custom sync action to merge PR code with base + - name: Sync codebase (PR) + if: github.event_name == 'pull_request' uses: qualcomm-linux-stg/video-driver/.github/actions/sync@video.qclinux.0.0 with: event_name: ${{ github.event_name }} pr_ref: ${{ github.event.pull_request.head.ref }} pr_repo: ${{ github.event.pull_request.head.repo.full_name }} - # LOGIC FIX: If target_branch is provided (Post Merge), use it. - # Otherwise use the default ref (Per Merge). - base_ref: ${{ inputs.target_branch || github.ref_name }} + base_ref: ${{ github.ref_name }} + # SCENARIO 2: Schedule or Manual (Post-Merge) + # Uses standard checkout because there is no PR to sync + - name: Checkout Code (Schedule) + if: github.event_name != 'pull_request' + uses: actions/checkout@v4 + with: + # Use the input branch if provided, otherwise default to current ref + ref: ${{ inputs.target_branch || github.ref_name }} + # Check out into 'video-driver' folder so the script finds the nested path + path: video-driver + + # Step to debug content (Optional, can be removed later) + - name: Debug Workspace + run: ls -R + - name: Load Parameters id: loading - # This calls your composite action (the JS file you provided) uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 \ No newline at end of file From c6bad3d3c00f04ed242cc421a9653cf236ff2844 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Sun, 25 Jan 2026 18:46:28 +0530 Subject: [PATCH 15/26] updated the loading.yml for the permission issues Signed-off-by: Manigurr --- .github/workflows/loading.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml index 91c5a1410..6d22b586e 100644 --- a/.github/workflows/loading.yml +++ b/.github/workflows/loading.yml @@ -46,10 +46,15 @@ jobs: # Check out into 'video-driver' folder so the script finds the nested path path: video-driver + # FIX: Adjust permissions so the Docker build container can write to the directory + - name: Fix Permissions for Docker + if: github.event_name != 'pull_request' + run: chmod -R 777 video-driver + # Step to debug content (Optional, can be removed later) - name: Debug Workspace run: ls -R - name: Load Parameters id: loading - uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 \ No newline at end of file + uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 From a6ffbe59e3c2c853c094242f5e4101d1a574563b Mon Sep 17 00:00:00 2001 From: Manigurr Date: Sun, 25 Jan 2026 19:56:54 +0530 Subject: [PATCH 16/26] Update sync-and-build.yml for the permission issue in the post merge.yml Signed-off-by: Manigurr --- .github/workflows/sync-and-build.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 01fe08c2c..2759e772b 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -38,6 +38,17 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build + # --- ADDED STEP: FIX PERMISSIONS --- + - name: Fix Permissions for Docker Build + shell: bash + run: | + echo "๐Ÿ”ง Adjusting workspace permissions..." + # Ensure the workspace is writable by the Docker container user + # Using sudo to override if files were created by root in previous steps + sudo chmod -R 777 ${{ github.workspace }} || chmod -R 777 ${{ github.workspace }} + echo "โœ… Permissions fixed." + # ----------------------------------- + - name: Build workspace uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 with: @@ -287,4 +298,4 @@ jobs: done < "$file_list" fi echo "" - } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + } >> "$GITHUB_STEP_SUMMARY" From fbc81f148a24f139a3b99eae6de9a7c928c9bef1 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Sun, 25 Jan 2026 21:54:26 +0530 Subject: [PATCH 17/26] fix --- .github/actions/build/action.yml | 7 ++--- .github/workflows/sync-and-build.yml | 46 ++++++++++++---------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 7a4d1f111..6c77d5aed 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -16,10 +16,11 @@ runs: - name: Build kernel shell: bash run: | + # Running as root to avoid permission issues. + # Ownership will be fixed in the calling workflow. docker run --rm \ -v "${{ inputs.workspace_path }}:${{ inputs.workspace_path }}" \ -w "${{ inputs.workspace_path }}/kernel" \ - --user $(id -u):$(id -g) \ ${{ inputs.docker_image }} \ bash -c " make O=../kobj ARCH=arm64 defconfig && @@ -33,11 +34,9 @@ runs: docker run --rm \ -v "${{ inputs.workspace_path }}:${{ inputs.workspace_path }}" \ -w "${{ inputs.workspace_path }}/video-driver" \ - --user $(id -u):$(id -g) \ ${{ inputs.docker_image }} \ bash -c " make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- CC=aarch64-linux-gnu-gcc-13 \ -C ${{ inputs.workspace_path }}/kobj \ M=\$(pwd) VIDEO_KERNEL_ROOT=\$(pwd) modules - " - + " \ No newline at end of file diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 2759e772b..72f01f967 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -38,23 +38,22 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build - # --- ADDED STEP: FIX PERMISSIONS --- - - name: Fix Permissions for Docker Build - shell: bash - run: | - echo "๐Ÿ”ง Adjusting workspace permissions..." - # Ensure the workspace is writable by the Docker container user - # Using sudo to override if files were created by root in previous steps - sudo chmod -R 777 ${{ github.workspace }} || chmod -R 777 ${{ github.workspace }} - echo "โœ… Permissions fixed." - # ----------------------------------- - - name: Build workspace - uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 + # CHANGED: Use local path to pick up the fix in action.yml + uses: ./.github/actions/build with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} + # ADDED: Fix ownership of files created by Docker (root) so the runner can process them + - name: Fix Workspace Ownership + if: always() + shell: bash + run: | + echo "๐Ÿ”ง Fixing file ownership..." + sudo chown -R $(id -u):$(id -g) ${{ github.workspace }} + echo "โœ… Ownership fixed." + - name: Download iris_test_app from the s3 shell: bash run: | @@ -96,10 +95,8 @@ jobs: firmware_version=$(ls kobj/tar-install/lib/modules/) mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ - # Copy test app, firmware blob, and video clips tar into data/vendor/iris_test_app cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" - #cp video-driver/iris_vpu.ko "$data_dir/" cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" echo "๐Ÿ“‚ Contents of $data_dir:" @@ -207,7 +204,6 @@ jobs: --numeric-owner --owner=0 --group=0 \ -cJf "$workspace/modules.tar.xz" lib/modules - # Safety checks on the tar if tar -Jtvf "$workspace/modules.tar.xz" | grep -q ' -> '; then echo "โŒ Symlinks found in modules archive (should be none)"; exit 1 fi @@ -218,7 +214,6 @@ jobs: echo "$workspace/modules.tar.xz" >> "$file_list" echo "โœ… Queued for upload: $workspace/modules.tar.xz" - # Kernel Image + merged video ramdisk (no local ramdisk) IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" VMLINUX_PATH="$workspace/kobj/vmlinux" MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" @@ -234,7 +229,6 @@ jobs: echo "$MERGED_PATH" >> "$file_list" echo "โœ… Queued for upload: $MERGED_PATH" - # Loop through all machines from the build_matrix input and add DTBs machines='${{ inputs.build_matrix }}' for machine in $(echo "$machines" | jq -r '.[].machine'); do dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" @@ -258,20 +252,20 @@ jobs: mode: multi-upload upload_location: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ github.run_id }}-${{ github.run_attempt }} - - name: Clean up if: always() shell: bash run: | set -euo pipefail + # FIX: Force cleanup even if files are owned by root ws="${{ github.workspace }}" - rm -rf "$ws/artqifacts" || true - rm -rf "$ws/combineramdisk" || true - rm -rf "$ws/downloads" || true - rm -rf "$ws/kobj" || true - rm -f "$ws/modules.tar.xz" || true - rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true - + echo "๐Ÿงน Cleaning up..." + sudo rm -rf "$ws/artqifacts" || true + sudo rm -rf "$ws/combineramdisk" || true + sudo rm -rf "$ws/downloads" || true + sudo rm -rf "$ws/kobj" || true + sudo rm -f "$ws/modules.tar.xz" || true + sudo rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true - name: Update summary if: success() || failure() @@ -298,4 +292,4 @@ jobs: done < "$file_list" fi echo "" - } >> "$GITHUB_STEP_SUMMARY" + } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file From 3a85f9262d3721e0bc711430b51890705d108a02 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Sun, 25 Jan 2026 22:34:38 +0530 Subject: [PATCH 18/26] fix path --- .github/workflows/sync-and-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 72f01f967..00fdf5e11 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -39,8 +39,7 @@ jobs: caller_workflow: build - name: Build workspace - # CHANGED: Use local path to pick up the fix in action.yml - uses: ./.github/actions/build + uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} From 0db4bf42b4dfb89f929bcfff2c4a78824571f279 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 26 Jan 2026 13:48:47 +0530 Subject: [PATCH 19/26] path fix --- .github/workflows/sync-and-build.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 00fdf5e11..17e987c5a 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -38,13 +38,25 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build + # 1. PERMISSIONS FIX: + # Since we modified action.yml to run as root, we must ensure the workspace is writable. + - name: Fix Permissions for Docker Build + shell: bash + run: | + echo "๐Ÿ”ง Adjusting workspace permissions..." + sudo chmod -R 777 ${{ github.workspace }} + echo "โœ… Permissions fixed." + - name: Build workspace - uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 + # 2. LOCAL PATH REFERENCE: + # Uses the action.yml from the code downloaded by 'Sync codebase' (which includes your fixes). + uses: ./video-driver/.github/actions/build with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} - # ADDED: Fix ownership of files created by Docker (root) so the runner can process them + # 3. OWNERSHIP RESTORATION: + # Files created by Docker (root) must be owned by the runner user for cleanup/upload steps to work. - name: Fix Workspace Ownership if: always() shell: bash @@ -191,9 +203,6 @@ jobs: file_list="$workspace/artifacts/file_list.txt" mkdir -p "$workspace/artifacts" - # Fresh file_list - : > "$file_list" - # Package lib/modules (xz-compressed) โ€” exclude risky symlinks mod_root="$workspace/kobj/tar-install/lib/modules" [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } From 78652208837ad50ef2214dadc8dbf976613f67e4 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Mon, 26 Jan 2026 17:52:23 +0530 Subject: [PATCH 20/26] fixed path for the permission --- .github/workflows/sync-and-build.yml | 96 +++------------------------- 1 file changed, 10 insertions(+), 86 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 17e987c5a..ccad9e816 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -38,8 +38,7 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build - # 1. PERMISSIONS FIX: - # Since we modified action.yml to run as root, we must ensure the workspace is writable. + # 1. PERMISSIONS FIX: Mandatory because we are running as root now - name: Fix Permissions for Docker Build shell: bash run: | @@ -48,15 +47,14 @@ jobs: echo "โœ… Permissions fixed." - name: Build workspace - # 2. LOCAL PATH REFERENCE: - # Uses the action.yml from the code downloaded by 'Sync codebase' (which includes your fixes). - uses: ./video-driver/.github/actions/build + # CHANGED: Use the local path at the ROOT of the workspace + # This will use the action.yml from the 'Sync codebase' step (which includes your fix) + uses: ./.github/actions/build with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} - # 3. OWNERSHIP RESTORATION: - # Files created by Docker (root) must be owned by the runner user for cleanup/upload steps to work. + # 2. OWNERSHIP RESTORATION: Mandatory cleanup step - name: Fix Workspace Ownership if: always() shell: bash @@ -73,7 +71,6 @@ jobs: echo " syncing files from s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" "${{ github.workspace }}/v4l-video-test-app/build/" echo " โœ… Download complete" - ls ${{ github.workspace }}/v4l-video-test-app/build/ - name: Download firmware file from S3 shell: bash @@ -82,7 +79,6 @@ jobs: mkdir -p "${{ github.workspace }}/downloads" echo "๐Ÿ“ฅ Syncing files from S3 path: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" "${{ github.workspace }}/downloads" - echo "โœ… Download complete" [ -f "${{ github.workspace }}/downloads/vpu20_1v.mbn" ] || { echo "โŒ Missing vpu20_1v.mbn"; exit 1; } - name: Download the video-contents for testing @@ -110,77 +106,51 @@ jobs: cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" - echo "๐Ÿ“‚ Contents of $data_dir:" - ls -lh "$data_dir" - - name: Create compressed kernel ramdisk archives shell: bash run: | set -euo pipefail cd "${{ github.workspace }}/kobj/tar-install" find lib/modules data | cpio -o -H newc --owner=0:0 | gzip -9 > "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - cd - > /dev/null - ls -lh "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - + - name: Download meta-qcom stable initramfs artifacts from S3 shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ” Fetching initramfs files from S3 bucket: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/" aws s3 cp s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" - echo "Initramfs files downloaded to: ${{ github.workspace }}/downloads" - name: Decompress ramdisk files and rename .cpio.gz files shell: bash run: | set -euo pipefail cd "${{ github.workspace }}/downloads" - echo " Decompressing and renaming .cpio.gz files..." gunzip -c initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz > kerneltest.cpio - name: Merge and repackage initramfs shell: bash run: | set -euo pipefail - echo "๐Ÿ”ง Starting repackaging process" - workspace="${{ github.workspace }}" mkdir -p "$workspace/combineramdisk" cp "$workspace/local-kernel-ramdisk.cpio.gz" "$workspace/combineramdisk/" cd "$workspace/combineramdisk" - - # Decompress local-kernel-ramdisk mv local-kernel-ramdisk.cpio.gz local-kernel-ramdisk.cpio.gz.bak gunzip -c local-kernel-ramdisk.cpio.gz.bak > local-kernel-ramdisk.cpio - - # Copy kerneltest from downloads cp "$workspace/downloads/kerneltest.cpio" . - - # Merge kerneltest and local-kernel-ramdisk cat kerneltest.cpio local-kernel-ramdisk.cpio > video-merged.cpio gzip -9 video-merged.cpio - - # Create temp workspace to clean up archive mkdir -p temp_merge cd temp_merge cpio -id --no-absolute-filenames < ../kerneltest.cpio cpio -id --no-absolute-filenames < ../local-kernel-ramdisk.cpio cd .. - - # Remove old merged archive rm -f video-merged.cpio.gz - - # Repackage clean archive cd temp_merge find . | cpio -o -H newc --owner=0:0 > ../video-merged.cpio cd .. gzip -9 video-merged.cpio - - # Cleanup rm -rf temp_merge kerneltest.cpio local-kernel-ramdisk.cpio - echo "Final archive: $workspace/combineramdisk/video-merged.cpio.gz" - ls -lh "$workspace/combineramdisk/video-merged.cpio.gz" - name: Validate build_matrix and jq shell: bash @@ -188,12 +158,11 @@ jobs: set -euo pipefail machines_json='${{ inputs.build_matrix }}' if ! command -v jq >/dev/null 2>&1; then - echo "โŒ jq is not installed on this runner. Please install jq." + echo "โŒ jq is not installed on this runner." exit 1 fi echo "$machines_json" | jq -e . >/dev/null [ "$(echo "$machines_json" | jq length)" -gt 0 ] || { echo "โŒ build_matrix is empty"; exit 1; } - echo "โœ… build_matrix is valid JSON" - name: Append artifacts to S3 upload list shell: bash @@ -202,8 +171,6 @@ jobs: workspace="${{ github.workspace }}" file_list="$workspace/artifacts/file_list.txt" mkdir -p "$workspace/artifacts" - - # Package lib/modules (xz-compressed) โ€” exclude risky symlinks mod_root="$workspace/kobj/tar-install/lib/modules" [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } tar -C "$workspace/kobj/tar-install" \ @@ -211,46 +178,19 @@ jobs: --exclude='lib/modules/*/source' \ --numeric-owner --owner=0 --group=0 \ -cJf "$workspace/modules.tar.xz" lib/modules - - if tar -Jtvf "$workspace/modules.tar.xz" | grep -q ' -> '; then - echo "โŒ Symlinks found in modules archive (should be none)"; exit 1 - fi - if tar -Jtf "$workspace/modules.tar.xz" | grep -Eq '^/|(^|/)\.\.(/|$)'; then - echo "โŒ Unsafe paths found in modules archive"; exit 1 - fi - + echo "$workspace/modules.tar.xz" >> "$file_list" - echo "โœ… Queued for upload: $workspace/modules.tar.xz" - IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" VMLINUX_PATH="$workspace/kobj/vmlinux" MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" - - [ -f "$IMAGE_PATH" ] || { echo "โŒ Missing expected file: $IMAGE_PATH"; exit 1; } - [ -f "$VMLINUX_PATH" ] || { echo "โŒ Missing expected file: $VMLINUX_PATH"; exit 1; } - [ -f "$MERGED_PATH" ] || { echo "โŒ Missing merged cpio: $MERGED_PATH"; exit 1; } - echo "$IMAGE_PATH" >> "$file_list" - echo "โœ… Queued for upload: $IMAGE_PATH" echo "$VMLINUX_PATH" >> "$file_list" - echo "โœ… Queued for upload: $VMLINUX_PATH" echo "$MERGED_PATH" >> "$file_list" - echo "โœ… Queued for upload: $MERGED_PATH" - machines='${{ inputs.build_matrix }}' for machine in $(echo "$machines" | jq -r '.[].machine'); do dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" - if [ -f "$dtb" ]; then - echo "$dtb" >> "$file_list" - echo "โœ… Queued for upload: $dtb" - else - echo "โŒ Missing DTB: $dtb" - exit 1 - fi + [ -f "$dtb" ] && echo "$dtb" >> "$file_list" || { echo "โŒ Missing DTB: $dtb"; exit 1; } done - - echo "----- Files queued for S3 upload -----" - cat "$file_list" - name: Upload all artifacts to S3 uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 @@ -265,7 +205,6 @@ jobs: shell: bash run: | set -euo pipefail - # FIX: Force cleanup even if files are owned by root ws="${{ github.workspace }}" echo "๐Ÿงน Cleaning up..." sudo rm -rf "$ws/artqifacts" || true @@ -285,19 +224,4 @@ jobs: else summary=":x: Build Failed" fi - - ws="${{ github.workspace }}" - file_list="$ws/artifacts/file_list.txt" - - { - echo "
Build Summary" - echo "$summary" - if [ -f "$file_list" ]; then - echo "" - echo "Artifacts queued for upload:" - while IFS= read -r line; do - echo "- $line" - done < "$file_list" - fi - echo "
" - } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + echo "
Build Summary$summary
" >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file From 8ffd9cad6a22cf0cb96fc147696beca0249ab35d Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 27 Jan 2026 11:25:05 +0530 Subject: [PATCH 21/26] reverted the path reference --- .github/workflows/sync-and-build.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index ccad9e816..124d23333 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -38,7 +38,7 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build - # 1. PERMISSIONS FIX: Mandatory because we are running as root now + # 1. PERMISSIONS FIX: Mandatory because Docker runs as root - name: Fix Permissions for Docker Build shell: bash run: | @@ -47,14 +47,14 @@ jobs: echo "โœ… Permissions fixed." - name: Build workspace - # CHANGED: Use the local path at the ROOT of the workspace - # This will use the action.yml from the 'Sync codebase' step (which includes your fix) - uses: ./.github/actions/build + # CHANGED: Using remote reference to fix 'action.yml not found' + uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} # 2. OWNERSHIP RESTORATION: Mandatory cleanup step + # Must run always() to ensure cleanup happens even if build fails - name: Fix Workspace Ownership if: always() shell: bash @@ -207,7 +207,8 @@ jobs: set -euo pipefail ws="${{ github.workspace }}" echo "๐Ÿงน Cleaning up..." - sudo rm -rf "$ws/artqifacts" || true + # Fixed typo: artqifacts -> artifacts + sudo rm -rf "$ws/artifacts" || true sudo rm -rf "$ws/combineramdisk" || true sudo rm -rf "$ws/downloads" || true sudo rm -rf "$ws/kobj" || true From d7adc70aaca1f1e8bce2ac7c79c2fe8d2b97c3b5 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Tue, 27 Jan 2026 12:56:41 +0530 Subject: [PATCH 22/26] explicitly check into the driver code for the post merge --- .github/workflows/sync-and-build.yml | 131 +++++++++++++++------------ 1 file changed, 75 insertions(+), 56 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 124d23333..80e3859af 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -16,6 +16,7 @@ on: permissions: packages: read + contents: read # Required for checkout jobs: sync-and-build: @@ -28,7 +29,17 @@ jobs: uses: qualcomm-linux/kernel-config/.github/actions/pull_docker_image@main with: image: ${{ inputs.docker_image }} - + + # ------------------------------------------------------------------------ + # โœ… CRITICAL FIX: Explicitly checkout the driver code. + # This ensures the source code exists for Post-Merge/Scheduled runs. + # ------------------------------------------------------------------------ + - name: Checkout Video Driver + uses: actions/checkout@v4 + with: + path: video-driver + fetch-depth: 0 + - name: Sync codebase uses: qualcomm-linux-stg/video-driver/.github/actions/sync@video.qclinux.0.0 with: @@ -38,73 +49,87 @@ jobs: base_ref: ${{ github.ref_name }} caller_workflow: build - # 1. PERMISSIONS FIX: Mandatory because Docker runs as root - name: Fix Permissions for Docker Build shell: bash run: | - echo "๐Ÿ”ง Adjusting workspace permissions..." + echo "๐Ÿ”ง Adjusting workspace permissions before build..." sudo chmod -R 777 ${{ github.workspace }} - echo "โœ… Permissions fixed." - name: Build workspace - # CHANGED: Using remote reference to fix 'action.yml not found' uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} - # 2. OWNERSHIP RESTORATION: Mandatory cleanup step - # Must run always() to ensure cleanup happens even if build fails + # ------------------------------------------------------------------------ + # โœ… CRITICAL FIX: Reset ownership immediately after build. + # This prevents "Permission denied" errors during job cleanup. + # ------------------------------------------------------------------------ - name: Fix Workspace Ownership if: always() shell: bash run: | - echo "๐Ÿ”ง Fixing file ownership..." + echo "๐Ÿ”ง Fixing file ownership (root -> runner user)..." sudo chown -R $(id -u):$(id -g) ${{ github.workspace }} - echo "โœ… Ownership fixed." - - name: Download iris_test_app from the s3 + - name: Download iris_test_app from S3 shell: bash run: | set -euo pipefail - mkdir -p "${{github.workspace }}/v4l-video-test-app/build/" - echo " syncing files from s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" + mkdir -p "${{ github.workspace }}/v4l-video-test-app/build/" + echo "๐Ÿ“ฅ Syncing iris_test_app..." aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" "${{ github.workspace }}/v4l-video-test-app/build/" - echo " โœ… Download complete" - + - name: Download firmware file from S3 shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ“ฅ Syncing files from S3 path: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" + echo "๐Ÿ“ฅ Syncing firmware artifacts..." aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" "${{ github.workspace }}/downloads" [ -f "${{ github.workspace }}/downloads/vpu20_1v.mbn" ] || { echo "โŒ Missing vpu20_1v.mbn"; exit 1; } - - name: Download the video-contents for testing + - name: Download video-contents for testing shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" - echo "Downloading the video-content files" + echo "๐Ÿ“ฅ Downloading video_clips_iris.tar.gz..." wget -q https://github.com/qualcomm-linux/qcom-linux-testkit/releases/download/IRIS-Video-Files-v1.0/video_clips_iris.tar.gz \ -O "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" - [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video_clips_iris.tar.gz"; exit 1; } + [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video content"; exit 1; } - name: Prepare /data/vendor/iris_test_app and list contents shell: bash run: | set -euo pipefail - data_dir="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app" - mkdir -p "$data_dir" - data_dir2="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" - mkdir -p "$data_dir2" - firmware_version=$(ls kobj/tar-install/lib/modules/) + # Ensure directories exist + mkdir -p "${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" + + # Find firmware version + if [ -d "kobj/tar-install/lib/modules/" ]; then + firmware_version=$(ls kobj/tar-install/lib/modules/ | head -n 1) + if [ -z "$firmware_version" ]; then echo "โŒ No modules found in kobj/tar-install/lib/modules/"; exit 1; fi + else + echo "โŒ Module directory not found" + exit 1 + fi + mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" - cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ - cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" - cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" - cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" + + # Handle .ko file location (checked out repo vs root) + if [ -f "video-driver/iris_vpu.ko" ]; then + cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ + elif [ -f "iris_vpu.ko" ]; then + cp iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ + else + echo "โš ๏ธ iris_vpu.ko not found. Build may have failed silently." + fi + + # Copy artifacts + cp "v4l-video-test-app/build/iris_v4l2_test" "kobj/tar-install/data/vendor/iris_test_app/" + cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "kobj/tar-install/data/vendor/iris_test_app/firmware/" + cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "kobj/tar-install/data/vendor/iris_test_app/" - name: Create compressed kernel ramdisk archives shell: bash @@ -120,7 +145,7 @@ jobs: mkdir -p "${{ github.workspace }}/downloads" aws s3 cp s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" - - name: Decompress ramdisk files and rename .cpio.gz files + - name: Decompress ramdisk files and rename shell: bash run: | set -euo pipefail @@ -134,22 +159,25 @@ jobs: workspace="${{ github.workspace }}" mkdir -p "$workspace/combineramdisk" cp "$workspace/local-kernel-ramdisk.cpio.gz" "$workspace/combineramdisk/" + cd "$workspace/combineramdisk" mv local-kernel-ramdisk.cpio.gz local-kernel-ramdisk.cpio.gz.bak gunzip -c local-kernel-ramdisk.cpio.gz.bak > local-kernel-ramdisk.cpio cp "$workspace/downloads/kerneltest.cpio" . + + # Merge Logic cat kerneltest.cpio local-kernel-ramdisk.cpio > video-merged.cpio gzip -9 video-merged.cpio + + # Repackage to clean up duplicates mkdir -p temp_merge cd temp_merge cpio -id --no-absolute-filenames < ../kerneltest.cpio cpio -id --no-absolute-filenames < ../local-kernel-ramdisk.cpio - cd .. - rm -f video-merged.cpio.gz - cd temp_merge + find . | cpio -o -H newc --owner=0:0 > ../video-merged.cpio cd .. - gzip -9 video-merged.cpio + gzip -f -9 video-merged.cpio rm -rf temp_merge kerneltest.cpio local-kernel-ramdisk.cpio - name: Validate build_matrix and jq @@ -158,11 +186,10 @@ jobs: set -euo pipefail machines_json='${{ inputs.build_matrix }}' if ! command -v jq >/dev/null 2>&1; then - echo "โŒ jq is not installed on this runner." + echo "โŒ jq is not installed." exit 1 fi echo "$machines_json" | jq -e . >/dev/null - [ "$(echo "$machines_json" | jq length)" -gt 0 ] || { echo "โŒ build_matrix is empty"; exit 1; } - name: Append artifacts to S3 upload list shell: bash @@ -171,8 +198,8 @@ jobs: workspace="${{ github.workspace }}" file_list="$workspace/artifacts/file_list.txt" mkdir -p "$workspace/artifacts" - mod_root="$workspace/kobj/tar-install/lib/modules" - [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } + + # Archive Modules tar -C "$workspace/kobj/tar-install" \ --exclude='lib/modules/*/build' \ --exclude='lib/modules/*/source' \ @@ -180,16 +207,15 @@ jobs: -cJf "$workspace/modules.tar.xz" lib/modules echo "$workspace/modules.tar.xz" >> "$file_list" - IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" - VMLINUX_PATH="$workspace/kobj/vmlinux" - MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" - echo "$IMAGE_PATH" >> "$file_list" - echo "$VMLINUX_PATH" >> "$file_list" - echo "$MERGED_PATH" >> "$file_list" + echo "$workspace/kobj/arch/arm64/boot/Image" >> "$file_list" + echo "$workspace/kobj/vmlinux" >> "$file_list" + echo "$workspace/combineramdisk/video-merged.cpio.gz" >> "$file_list" + + # DTBs machines='${{ inputs.build_matrix }}' for machine in $(echo "$machines" | jq -r '.[].machine'); do dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" - [ -f "$dtb" ] && echo "$dtb" >> "$file_list" || { echo "โŒ Missing DTB: $dtb"; exit 1; } + [ -f "$dtb" ] && echo "$dtb" >> "$file_list" done - name: Upload all artifacts to S3 @@ -207,22 +233,15 @@ jobs: set -euo pipefail ws="${{ github.workspace }}" echo "๐Ÿงน Cleaning up..." - # Fixed typo: artqifacts -> artifacts - sudo rm -rf "$ws/artifacts" || true - sudo rm -rf "$ws/combineramdisk" || true - sudo rm -rf "$ws/downloads" || true - sudo rm -rf "$ws/kobj" || true - sudo rm -f "$ws/modules.tar.xz" || true - sudo rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true + sudo rm -rf "$ws/artifacts" "$ws/combineramdisk" "$ws/downloads" "$ws/kobj" || true + sudo rm -f "$ws/modules.tar.xz" "$ws/local-kernel-ramdisk.cpio.gz" || true - name: Update summary if: success() || failure() shell: bash run: | - status="${{ steps.build_workspace.outcome }}" - if [ "$status" = "success" ]; then - summary=":heavy_check_mark: Build Success" + if [ "${{ steps.build_workspace.outcome }}" = "success" ]; then + echo "โœ… Build Success" >> "$GITHUB_STEP_SUMMARY" else - summary=":x: Build Failed" - fi - echo "
Build Summary$summary
" >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + echo "โŒ Build Failed" >> "$GITHUB_STEP_SUMMARY" + fi \ No newline at end of file From 38a31265d2702c4fb74029e9f6f1891fa865b048 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Wed, 28 Jan 2026 11:16:24 +0530 Subject: [PATCH 23/26] removed the target branch in the postmerge.yml --- .github/workflows/post_merge.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/post_merge.yml b/.github/workflows/post_merge.yml index fb84b1907..ec852acfd 100644 --- a/.github/workflows/post_merge.yml +++ b/.github/workflows/post_merge.yml @@ -10,9 +10,6 @@ jobs: # We call the reusable workflow uses: qualcomm-linux-stg/video-driver/.github/workflows/loading.yml@video.qclinux.0.0 secrets: inherit - with: - # We pass the specific branch we want to build - target_branch: video.qclinux.0.0 build: needs: loading From b61295dbd3f970f5ecdfe2c8ed10c447ade2ed38 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Wed, 28 Jan 2026 13:44:59 +0530 Subject: [PATCH 24/26] updated the clean logic --- .github/workflows/loading.yml | 13 +- .github/workflows/sync-and-build.yml | 212 +++++++++++++++++---------- 2 files changed, 134 insertions(+), 91 deletions(-) diff --git a/.github/workflows/loading.yml b/.github/workflows/loading.yml index 6d22b586e..68cf88cd6 100644 --- a/.github/workflows/loading.yml +++ b/.github/workflows/loading.yml @@ -45,16 +45,7 @@ jobs: ref: ${{ inputs.target_branch || github.ref_name }} # Check out into 'video-driver' folder so the script finds the nested path path: video-driver - - # FIX: Adjust permissions so the Docker build container can write to the directory - - name: Fix Permissions for Docker - if: github.event_name != 'pull_request' - run: chmod -R 777 video-driver - - # Step to debug content (Optional, can be removed later) - - name: Debug Workspace - run: ls -R - + - name: Load Parameters id: loading - uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 + uses: qualcomm-linux-stg/video-driver/.github/actions/loading@video.qclinux.0.0 \ No newline at end of file diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 80e3859af..467f5465e 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -16,7 +16,6 @@ on: permissions: packages: read - contents: read # Required for checkout jobs: sync-and-build: @@ -29,8 +28,8 @@ jobs: uses: qualcomm-linux/kernel-config/.github/actions/pull_docker_image@main with: image: ${{ inputs.docker_image }} - - # ------------------------------------------------------------------------ + + #--------------------------------------------------------------------- # โœ… CRITICAL FIX: Explicitly checkout the driver code. # This ensures the source code exists for Post-Merge/Scheduled runs. # ------------------------------------------------------------------------ @@ -48,88 +47,61 @@ jobs: pr_repo: ${{ github.event.pull_request.head.repo.full_name }} base_ref: ${{ github.ref_name }} caller_workflow: build - - - name: Fix Permissions for Docker Build - shell: bash - run: | - echo "๐Ÿ”ง Adjusting workspace permissions before build..." - sudo chmod -R 777 ${{ github.workspace }} - - name: Build workspace uses: qualcomm-linux-stg/video-driver/.github/actions/build@video.qclinux.0.0 with: docker_image: kmake-image:ver.1.0 workspace_path: ${{ github.workspace }} - # ------------------------------------------------------------------------ - # โœ… CRITICAL FIX: Reset ownership immediately after build. - # This prevents "Permission denied" errors during job cleanup. - # ------------------------------------------------------------------------ - - name: Fix Workspace Ownership - if: always() - shell: bash - run: | - echo "๐Ÿ”ง Fixing file ownership (root -> runner user)..." - sudo chown -R $(id -u):$(id -g) ${{ github.workspace }} - - - name: Download iris_test_app from S3 + - name: Download iris_test_app from the s3 shell: bash run: | set -euo pipefail - mkdir -p "${{ github.workspace }}/v4l-video-test-app/build/" - echo "๐Ÿ“ฅ Syncing iris_test_app..." + mkdir -p "${{github.workspace }}/v4l-video-test-app/build/" + echo " syncing files from s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/iris_test_app/" "${{ github.workspace }}/v4l-video-test-app/build/" - + echo " โœ… Download complete" + ls ${{ github.workspace }}/v4l-video-test-app/build/ + - name: Download firmware file from S3 shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ“ฅ Syncing firmware artifacts..." + echo "๐Ÿ“ฅ Syncing files from S3 path: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" aws s3 sync "s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/" "${{ github.workspace }}/downloads" + echo "โœ… Download complete" [ -f "${{ github.workspace }}/downloads/vpu20_1v.mbn" ] || { echo "โŒ Missing vpu20_1v.mbn"; exit 1; } - - name: Download video-contents for testing + - name: Download the video-contents for testing shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" - echo "๐Ÿ“ฅ Downloading video_clips_iris.tar.gz..." + echo "Downloading the video-content files" wget -q https://github.com/qualcomm-linux/qcom-linux-testkit/releases/download/IRIS-Video-Files-v1.0/video_clips_iris.tar.gz \ -O "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" - [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video content"; exit 1; } + [ -f "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" ] || { echo "โŒ Failed to download video_clips_iris.tar.gz"; exit 1; } - name: Prepare /data/vendor/iris_test_app and list contents shell: bash run: | set -euo pipefail - # Ensure directories exist - mkdir -p "${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" - - # Find firmware version - if [ -d "kobj/tar-install/lib/modules/" ]; then - firmware_version=$(ls kobj/tar-install/lib/modules/ | head -n 1) - if [ -z "$firmware_version" ]; then echo "โŒ No modules found in kobj/tar-install/lib/modules/"; exit 1; fi - else - echo "โŒ Module directory not found" - exit 1 - fi - + data_dir="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app" + mkdir -p "$data_dir" + data_dir2="${{ github.workspace }}/kobj/tar-install/data/vendor/iris_test_app/firmware" + mkdir -p "$data_dir2" + firmware_version=$(ls kobj/tar-install/lib/modules/) mkdir -p "kobj/tar-install/lib/modules/$firmware_version/updates" - - # Handle .ko file location (checked out repo vs root) - if [ -f "video-driver/iris_vpu.ko" ]; then - cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ - elif [ -f "iris_vpu.ko" ]; then - cp iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ - else - echo "โš ๏ธ iris_vpu.ko not found. Build may have failed silently." - fi + cp video-driver/iris_vpu.ko kobj/tar-install/lib/modules/$firmware_version/updates/ + # Copy test app, firmware blob, and video clips tar into data/vendor/iris_test_app + cp "v4l-video-test-app/build/iris_v4l2_test" "$data_dir/" + cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "$data_dir2/" + #cp video-driver/iris_vpu.ko "$data_dir/" + cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "$data_dir/" - # Copy artifacts - cp "v4l-video-test-app/build/iris_v4l2_test" "kobj/tar-install/data/vendor/iris_test_app/" - cp "${{ github.workspace }}/downloads/vpu20_1v.mbn" "kobj/tar-install/data/vendor/iris_test_app/firmware/" - cp "${{ github.workspace }}/downloads/video_clips_iris.tar.gz" "kobj/tar-install/data/vendor/iris_test_app/" + echo "๐Ÿ“‚ Contents of $data_dir:" + ls -lh "$data_dir" - name: Create compressed kernel ramdisk archives shell: bash @@ -137,48 +109,68 @@ jobs: set -euo pipefail cd "${{ github.workspace }}/kobj/tar-install" find lib/modules data | cpio -o -H newc --owner=0:0 | gzip -9 > "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" - + cd - > /dev/null + ls -lh "${{ github.workspace }}/local-kernel-ramdisk.cpio.gz" + - name: Download meta-qcom stable initramfs artifacts from S3 shell: bash run: | set -euo pipefail mkdir -p "${{ github.workspace }}/downloads" + echo "๐Ÿ” Fetching initramfs files from S3 bucket: s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/" aws s3 cp s3://qli-stg-video-gh-artifacts/qualcomm-linux-stg/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz "${{ github.workspace }}/downloads/" + echo "Initramfs files downloaded to: ${{ github.workspace }}/downloads" - - name: Decompress ramdisk files and rename + - name: Decompress ramdisk files and rename .cpio.gz files shell: bash run: | set -euo pipefail cd "${{ github.workspace }}/downloads" + echo " Decompressing and renaming .cpio.gz files..." gunzip -c initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz > kerneltest.cpio - name: Merge and repackage initramfs shell: bash run: | set -euo pipefail + echo "๐Ÿ”ง Starting repackaging process" + workspace="${{ github.workspace }}" mkdir -p "$workspace/combineramdisk" cp "$workspace/local-kernel-ramdisk.cpio.gz" "$workspace/combineramdisk/" - cd "$workspace/combineramdisk" + + # Decompress local-kernel-ramdisk mv local-kernel-ramdisk.cpio.gz local-kernel-ramdisk.cpio.gz.bak gunzip -c local-kernel-ramdisk.cpio.gz.bak > local-kernel-ramdisk.cpio + + # Copy kerneltest from downloads cp "$workspace/downloads/kerneltest.cpio" . - - # Merge Logic + + # Merge kerneltest and local-kernel-ramdisk cat kerneltest.cpio local-kernel-ramdisk.cpio > video-merged.cpio gzip -9 video-merged.cpio - - # Repackage to clean up duplicates + + # Create temp workspace to clean up archive mkdir -p temp_merge cd temp_merge cpio -id --no-absolute-filenames < ../kerneltest.cpio cpio -id --no-absolute-filenames < ../local-kernel-ramdisk.cpio - + cd .. + + # Remove old merged archive + rm -f video-merged.cpio.gz + + # Repackage clean archive + cd temp_merge find . | cpio -o -H newc --owner=0:0 > ../video-merged.cpio cd .. - gzip -f -9 video-merged.cpio + gzip -9 video-merged.cpio + + # Cleanup rm -rf temp_merge kerneltest.cpio local-kernel-ramdisk.cpio + echo "Final archive: $workspace/combineramdisk/video-merged.cpio.gz" + ls -lh "$workspace/combineramdisk/video-merged.cpio.gz" - name: Validate build_matrix and jq shell: bash @@ -186,10 +178,12 @@ jobs: set -euo pipefail machines_json='${{ inputs.build_matrix }}' if ! command -v jq >/dev/null 2>&1; then - echo "โŒ jq is not installed." + echo "โŒ jq is not installed on this runner. Please install jq." exit 1 fi echo "$machines_json" | jq -e . >/dev/null + [ "$(echo "$machines_json" | jq length)" -gt 0 ] || { echo "โŒ build_matrix is empty"; exit 1; } + echo "โœ… build_matrix is valid JSON" - name: Append artifacts to S3 upload list shell: bash @@ -198,25 +192,61 @@ jobs: workspace="${{ github.workspace }}" file_list="$workspace/artifacts/file_list.txt" mkdir -p "$workspace/artifacts" - - # Archive Modules + + # Fresh file_list + : > "$file_list" + + # Package lib/modules (xz-compressed) โ€” exclude risky symlinks + mod_root="$workspace/kobj/tar-install/lib/modules" + [ -d "$mod_root" ] || { echo "โŒ Missing directory: $mod_root"; exit 1; } tar -C "$workspace/kobj/tar-install" \ --exclude='lib/modules/*/build' \ --exclude='lib/modules/*/source' \ --numeric-owner --owner=0 --group=0 \ -cJf "$workspace/modules.tar.xz" lib/modules - + + # Safety checks on the tar + if tar -Jtvf "$workspace/modules.tar.xz" | grep -q ' -> '; then + echo "โŒ Symlinks found in modules archive (should be none)"; exit 1 + fi + if tar -Jtf "$workspace/modules.tar.xz" | grep -Eq '^/|(^|/)\.\.(/|$)'; then + echo "โŒ Unsafe paths found in modules archive"; exit 1 + fi + echo "$workspace/modules.tar.xz" >> "$file_list" - echo "$workspace/kobj/arch/arm64/boot/Image" >> "$file_list" - echo "$workspace/kobj/vmlinux" >> "$file_list" - echo "$workspace/combineramdisk/video-merged.cpio.gz" >> "$file_list" - - # DTBs + echo "โœ… Queued for upload: $workspace/modules.tar.xz" + + # Kernel Image + merged video ramdisk (no local ramdisk) + IMAGE_PATH="$workspace/kobj/arch/arm64/boot/Image" + VMLINUX_PATH="$workspace/kobj/vmlinux" + MERGED_PATH="$workspace/combineramdisk/video-merged.cpio.gz" + + [ -f "$IMAGE_PATH" ] || { echo "โŒ Missing expected file: $IMAGE_PATH"; exit 1; } + [ -f "$VMLINUX_PATH" ] || { echo "โŒ Missing expected file: $VMLINUX_PATH"; exit 1; } + [ -f "$MERGED_PATH" ] || { echo "โŒ Missing merged cpio: $MERGED_PATH"; exit 1; } + + echo "$IMAGE_PATH" >> "$file_list" + echo "โœ… Queued for upload: $IMAGE_PATH" + echo "$VMLINUX_PATH" >> "$file_list" + echo "โœ… Queued for upload: $VMLINUX_PATH" + echo "$MERGED_PATH" >> "$file_list" + echo "โœ… Queued for upload: $MERGED_PATH" + + # Loop through all machines from the build_matrix input and add DTBs machines='${{ inputs.build_matrix }}' for machine in $(echo "$machines" | jq -r '.[].machine'); do dtb="$workspace/kobj/arch/arm64/boot/dts/qcom/${machine}.dtb" - [ -f "$dtb" ] && echo "$dtb" >> "$file_list" + if [ -f "$dtb" ]; then + echo "$dtb" >> "$file_list" + echo "โœ… Queued for upload: $dtb" + else + echo "โŒ Missing DTB: $dtb" + exit 1 + fi done + + echo "----- Files queued for S3 upload -----" + cat "$file_list" - name: Upload all artifacts to S3 uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.qclinux.0.0 @@ -226,22 +256,44 @@ jobs: mode: multi-upload upload_location: ${{ github.repository_owner }}/${{ github.event.repository.name }}/${{ github.run_id }}-${{ github.run_attempt }} + - name: Clean up if: always() shell: bash run: | set -euo pipefail ws="${{ github.workspace }}" - echo "๐Ÿงน Cleaning up..." - sudo rm -rf "$ws/artifacts" "$ws/combineramdisk" "$ws/downloads" "$ws/kobj" || true - sudo rm -f "$ws/modules.tar.xz" "$ws/local-kernel-ramdisk.cpio.gz" || true + rm -rf "$ws/artqifacts" || true + rm -rf "$ws/combineramdisk" || true + rm -rf "$ws/downloads" || true + rm -rf "$ws/kobj" || true + rm -f "$ws/modules.tar.xz" || true + rm -f "$ws/local-kernel-ramdisk.cpio.gz" || true + - name: Update summary if: success() || failure() shell: bash run: | - if [ "${{ steps.build_workspace.outcome }}" = "success" ]; then - echo "โœ… Build Success" >> "$GITHUB_STEP_SUMMARY" + status="${{ steps.build_workspace.outcome }}" + if [ "$status" = "success" ]; then + summary=":heavy_check_mark: Build Success" else - echo "โŒ Build Failed" >> "$GITHUB_STEP_SUMMARY" - fi \ No newline at end of file + summary=":x: Build Failed" + fi + + ws="${{ github.workspace }}" + file_list="$ws/artifacts/file_list.txt" + + { + echo "
Build Summary" + echo "$summary" + if [ -f "$file_list" ]; then + echo "" + echo "Artifacts queued for upload:" + while IFS= read -r line; do + echo "- $line" + done < "$file_list" + fi + echo "
" + } >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file From 6a50d1b05414a012622c45414922c492b5b70e56 Mon Sep 17 00:00:00 2001 From: Manigurr Date: Wed, 28 Jan 2026 13:47:07 +0530 Subject: [PATCH 25/26] fix --- .github/workflows/sync-and-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 467f5465e..0a7c743a7 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -29,7 +29,7 @@ jobs: with: image: ${{ inputs.docker_image }} - #--------------------------------------------------------------------- + # ------------------------------------------------------------------------ # โœ… CRITICAL FIX: Explicitly checkout the driver code. # This ensures the source code exists for Post-Merge/Scheduled runs. # ------------------------------------------------------------------------ From 1d4af6faf093cafea38780c8f33b4f72be398e7f Mon Sep 17 00:00:00 2001 From: Manigurr Date: Wed, 28 Jan 2026 13:55:17 +0530 Subject: [PATCH 26/26] updated the syntax --- .github/workflows/sync-and-build.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sync-and-build.yml b/.github/workflows/sync-and-build.yml index 0a7c743a7..076688da3 100644 --- a/.github/workflows/sync-and-build.yml +++ b/.github/workflows/sync-and-build.yml @@ -28,16 +28,13 @@ jobs: uses: qualcomm-linux/kernel-config/.github/actions/pull_docker_image@main with: image: ${{ inputs.docker_image }} - - # ------------------------------------------------------------------------ + + # ------------------------------------------------------------------------ # โœ… CRITICAL FIX: Explicitly checkout the driver code. # This ensures the source code exists for Post-Merge/Scheduled runs. # ------------------------------------------------------------------------ - name: Checkout Video Driver - uses: actions/checkout@v4 - with: - path: video-driver - fetch-depth: 0 + uses: actions/checkout@v4 - name: Sync codebase uses: qualcomm-linux-stg/video-driver/.github/actions/sync@video.qclinux.0.0