From fa8ca00467369e212ff311998bd7981c642369a0 Mon Sep 17 00:00:00 2001 From: Thomas Goodwin Date: Thu, 17 Apr 2025 10:16:38 -0400 Subject: [PATCH 1/6] feat(github): port cerbero-action features to cerbero build The goal of these changes is to make cerbero able to build on github-provided runners as well as hosted ones. --- .github/actions/build-gstreamer/action.yml | 66 ++++++++---- .github/actions/install-gstreamer/action.yml | 96 ++++++++++------- .github/actions/setup-gstreamer/action.yml | 108 +++++++++++++------ .github/workflows/windows.yml | 6 +- 4 files changed, 179 insertions(+), 97 deletions(-) diff --git a/.github/actions/build-gstreamer/action.yml b/.github/actions/build-gstreamer/action.yml index 8d8e612a5..234095232 100644 --- a/.github/actions/build-gstreamer/action.yml +++ b/.github/actions/build-gstreamer/action.yml @@ -40,15 +40,15 @@ outputs: value: ${{ steps.cerbero-config.outputs.version }} runtime-installer-path: description: 'Location of the GStreamer runtime installer' - value: ${{ steps.build-packages.outputs.runtime-msi }} + value: ${{ steps.build-packages.outputs.runtime-file }} devel-installer-path: description: 'Location of the GStreamer development installer' - value: ${{ steps.build-packages.outputs.dev-msi }} + value: ${{ steps.build-packages.outputs.dev-file }} runs: using: "composite" steps: - if: ${{ inputs.bootstrap-system == 'true' }} - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} working-directory: cerbero run: | # Bootstrap Windows @@ -56,7 +56,7 @@ runs: - id: cerbero-config working-directory: cerbero - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} env: CERBERO_PATH: cerbero CERBERO_HOME: cerbero-build @@ -73,13 +73,19 @@ runs: echo "cerbero-home=${CERBERO_PATH}\\${CERBERO_HOME}" >> $GITHUB_OUTPUT echo "cerbero-sources=${CERBERO_PATH}\\${CERBERO_SOURCES}" >> $GITHUB_OUTPUT - echo "full-log-path=$(cygpath -au .)/${CERBERO_HOME}/logs" >> $GITHUB_OUTPUT - pwd_native=$(cygpath -am .) + if [ "${{ runner.os }}" == "Windows" ]; then + echo "full-log-path=$(cygpath -au .)/${CERBERO_HOME}/logs" >> $GITHUB_OUTPUT + pwd_native=$(cygpath -am .) + else + echo "full-log-path=$(pwd)/${CERBERO_HOME}/logs" >> $GITHUB_OUTPUT + pwd_native=$(pwd) + fi + echo "home_dir = \"${pwd_native}/${CERBERO_HOME}\"" > localconf.cbc echo "local_sources = \"${pwd_native}/${CERBERO_SOURCES}\"" >> localconf.cbc echo "vs_install_version = \"vs17\"" >> localconf.cbc - echo "num_of_cpus = ${NUMBER_OF_PROCESSORS}" >> localconf.cbc + echo "num_of_cpus = ${NUMBER_OF_PROCESSORS:-$(nproc --all)}" >> localconf.cbc echo "package_origin = \"${CERBERO_PACKAGE_ORIGIN}\"" >> localconf.cbc # Handle custom gst-plugins-rs provider @@ -123,7 +129,7 @@ runs: ${{ runner.os }}-cerbero-deps-${{ steps.cerbero-config.outputs.version }}-${{ steps.cerbero-config.outputs.cerbero-sha }}-${{ steps.cerbero-config.outputs.config-hash }}- - if: ${{ steps.restore-cerbero-deps-cache.outputs.cache-matched-key }} - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} run: | # Restore Cerbero dependencies deps_file='${{ steps.cerbero-config.outputs.cerbero-home }}\cerbero-deps.tar.xz' @@ -138,9 +144,9 @@ runs: ls -l '${{ steps.cerbero-config.outputs.cerbero-home }}' echo "::endgroup::" - - id: bootstrap-cerbero + - id: build-gstreamer-deps working-directory: cerbero - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} env: CI_PROJECT_NAME: cerbero LOG_PATH: ${{ steps.cerbero-config.outputs.full-log-path }} @@ -173,13 +179,13 @@ runs: group_cmd "$CERBERO $CERBERO_ARGS build --offline $more_deps" | tee -a "${LOG_PATH}/3_build-deps.log" group_cmd "$CERBERO $CERBERO_ARGS gen-cache" | tee "${LOG_PATH}/4_cache.log" - - if: ${{ steps.bootstrap-cerbero.outcome == 'success' && ! steps.restore-cerbero-sources-cache.outputs.cache-hit }} + - if: ${{ steps.build-gstreamer-deps.outcome == 'success' && ! steps.restore-cerbero-sources-cache.outputs.cache-hit }} uses: actions/cache/save@v4 with: path: ${{ steps.cerbero-config.outputs.cerbero-sources }} key: ${{ steps.restore-cerbero-sources-cache.outputs.cache-primary-key }} - - if: ${{ steps.bootstrap-cerbero.outcome == 'success' && ! steps.restore-cerbero-deps-cache.outputs.cache-hit }} + - if: ${{ steps.build-gstreamer-deps.outcome == 'success' && ! steps.restore-cerbero-deps-cache.outputs.cache-hit }} uses: actions/cache/save@v4 with: path: ${{ steps.cerbero-config.outputs.cerbero-home }}\cerbero-deps.tar.xz @@ -187,32 +193,43 @@ runs: - id: build-packages working-directory: cerbero - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} env: CI_PROJECT_NAME: cerbero LOG_PATH: ${{ steps.cerbero-config.outputs.full-log-path }} run: | mkdir -p "${LOG_PATH}" - cmd="$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o \"$(cygpath -am .)\" gstreamer-1.0" + + if [ "${{ runner.os }}" == "Windows" ]; then + cmd="$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o \"$(cygpath -am .)\" gstreamer-1.0" + else + cmd="$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o \"$(pwd)\" gstreamer-1.0" + fi + echo "::group::$cmd" - eval $cmd | tee "${LOG_PATH}/package.log" + eval $cmd | tee "${LOG_PATH}/5_package.log" echo "::endgroup::" - find ~+ -maxdepth 1 -name 'gstreamer-1.0-msvc*.msi' -exec sh -c 'echo runtime-msi=$(cygpath -aw "{}") >> $GITHUB_OUTPUT' \; - find ~+ -maxdepth 1 -name 'gstreamer-1.0-devel*.msi' -exec sh -c 'echo dev-msi=$(cygpath -aw "{}") >> $GITHUB_OUTPUT' \; + if [ "${{ runner.os }}" == "Windows" ]; then + find ~+ -maxdepth 1 -name 'gstreamer-1.0-msvc*.msi' -exec sh -c 'echo runtime-file=$(cygpath -aw "{}") >> $GITHUB_OUTPUT' \; + find ~+ -maxdepth 1 -name 'gstreamer-1.0-devel*.msi' -exec sh -c 'echo dev-file=$(cygpath -aw "{}") >> $GITHUB_OUTPUT' \; + else + >&2 echo "Error: This action is only supported on Windows at this time; pull requests welcome!" + exit 1 + fi - name: Annotate workflow run with GStreamer version - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} run: | echo "::notice title=Built GStreamer version ${{ steps.cerbero-config.outputs.version }}::Created installer with Cerbero short sha ${{ steps.cerbero-config.outputs.cerbero-sha }}" - uses: actions/upload-artifact@v4 - if: ${{ steps.build-packages.outputs.runtime-msi || steps.build-packages.outputs.dev-msi }} + if: ${{ steps.build-packages.outputs.runtime-file || steps.build-packages.outputs.dev-file }} with: name: GStreamer Installers path: | - ${{ steps.build-packages.outputs.runtime-msi }} - ${{ steps.build-packages.outputs.dev-msi }} + ${{ steps.build-packages.outputs.runtime-file }} + ${{ steps.build-packages.outputs.dev-file }} - name: Upload logs uses: actions/upload-artifact@v4 @@ -220,3 +237,10 @@ runs: with: name: cerbero-logs path: ${{ steps.cerbero-config.outputs.cerbero-home }}\logs + + - name: Upload builddir on build failure + uses: actions/upload-artifact@v4 + if: ${{ always() && steps.build-packages.outcome != 'success' }} + with: + name: build-gstreamer-deps + path: ${{ steps.cerbero-config.outputs.cerbero-home }} diff --git a/.github/actions/install-gstreamer/action.yml b/.github/actions/install-gstreamer/action.yml index 0827059f7..ee18024a0 100644 --- a/.github/actions/install-gstreamer/action.yml +++ b/.github/actions/install-gstreamer/action.yml @@ -1,5 +1,5 @@ -name: Install GStreamer with Cerbero on Windows -description: Installs the GStreamer MSIs either from the URLs or local paths +name: Install GStreamer from Packages +description: Installs the GStreamer using packages at either the URLs or local paths inputs: runtime-installer-url: description: 'URL of the GStreamer runtime installer' @@ -32,81 +32,91 @@ runs: - if: ${{ inputs.runtime-installer-url && inputs.runtime-installer-path }} shell: msys2 -eo pipefail {0} run: | - echo "Error: Cannot specify both runtime-installer-url and runtime-installer-path" + echo "::error Cannot specify both runtime-installer-url and runtime-installer-path" exit 1 - if: ${{ inputs.development-installer-url && inputs.development-installer-path }} shell: msys2 -eo pipefail {0} run: | - echo "Error: Cannot specify both development-installer-url and development-installer-path" + echo "::error Cannot specify both development-installer-url and development-installer-path" exit 2 - if: ${{ !inputs.runtime-installer-url && !inputs.runtime-installer-path }} shell: msys2 -eo pipefail {0} run: | - echo "Error: Must specify either runtime-installer-url or runtime-installer-path" + echo "::error Must specify either runtime-installer-url or runtime-installer-path" exit 3 - if: ${{ !inputs.development-installer-url && !inputs.development-installer-path }} shell: msys2 -eo pipefail {0} run: | - echo "Error: Must specify either development-installer-url or development-installer-path" + echo "::error Must specify either development-installer-url or development-installer-path" exit 4 - if: ${{ inputs.runtime-installer-path }} shell: msys2 -eo pipefail {0} run: | if [[ ! -f "${{ inputs.runtime-installer-path }}" ]]; then - echo "Error: Runtime installer path does not exist: ${{ inputs.runtime-installer-path }}" + echo "::error Runtime installer path does not exist: ${{ inputs.runtime-installer-path }}" exit 5 fi - if: ${{ inputs.development-installer-path }} shell: msys2 -eo pipefail {0} run: | if [[ ! -f "${{ inputs.development-installer-path }}" ]]; then - echo "Error: Development installer path does not exist: ${{ inputs.development-installer-path }}" + echo "::error Development installer path does not exist: ${{ inputs.development-installer-path }}" exit 6 fi + - if: ${{ runner.os != 'Windows' }} + shell: msys2 -eo pipefail {0} + run: | + echo "::error This action is only supported on Windows at this time; pull requests welcome!" + exit 7 - # If the URL is provided, download the MSI, set path. + # If the URL is provided, download the file, set path. # This will error out if the file is missing. - if: ${{ !inputs.runtime-installer-path }} id: download-runtime - name: Download GStreamer Runtime MSI - shell: powershell + name: Download GStreamer Runtime File + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} run: | - $msi_path = "$(( $pwd ))\gstreamer-runtime.msi" - Invoke-WebRequest -Uri "${{ inputs.runtime-installer-url }}" -OutFile "$msi_path" - echo "msi-path=$msi_path" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 - echo "::notice title=GStreamer runtime installer downloaded::Downloaded GStreamer runtime installer from ${{ inputs.runtime-installer-url }}" + if [ "${{ runner.os }}" == "Windows" ]; then + runtime_path="$(pwd)\gstreamer-runtime.msi" + fi + curl -L -O "${{ inputs.runtime-installer-url }}" -o "$runtime_path" + echo "runtime-path=$runtime_path" | tee -a $GITHUB_OUTPUT + echo "::notice title=GStreamer runtime installer downloaded from ${{ inputs.runtime-installer-url }}" - if: ${{ !inputs.development-installer-path }} id: download-development - name: Download GStreamer Development MSI - shell: powershell + name: Download GStreamer Development File + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} run: | - $msi_path = "$(( $pwd ))\gstreamer-development.msi" - Invoke-WebRequest -Uri "${{ inputs.development-installer-url }}" -OutFile "$msi_path" - echo "msi-path=$msi_path" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 - echo "::notice title=GStreamer development installer downloaded::Downloaded GStreamer development installer from ${{ inputs.development-installer-url }}" + if [ "${{ runner.os }}" == "Windows" ]; then + development_path="$(pwd)\gstreamer-development.msi" + fi + curl -L -O "${{ inputs.development-installer-url }}" -o "$development_path" + echo "development-path=$development_path" | tee -a $GITHUB_OUTPUT + echo "::notice title=GStreamer development installer downloaded from ${{ inputs.development-installer-url }}" # Set the local paths favoring if the download happened - - id: msi-path - shell: powershell + - id: choose-paths + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} run: | # If the download happened, use the downloaded path, else use the input path - $runtime = "${{ steps.download-development.outputs.msi-path }}" - if ($runtime -eq "") { - $runtime = "${{ inputs.runtime-installer-path }}" - } - $development = "${{ steps.download-development.outputs.msi-path }}" - if ($development -eq "") { - $development = "${{ inputs.development-installer-path }}" - } - echo "runtime=$runtime" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 - echo "development=$development" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 + runtime="${{ steps.download-runtime.outputs.runtime-path }}" + if [ -z "$runtime" ]; then + runtime="${{ inputs.runtime-installer-path }}" + fi + development="${{ steps.download-development.outputs.development-path }}" + if [ -z "$development" ]; then + development="${{ inputs.development-installer-path }}" + fi + echo "runtime=$runtime" | tee -a $GITHUB_OUTPUT + echo "development=$development" | tee -a $GITHUB_OUTPUT - - name: Install GStreamer + - if: ${{ runner.os == 'Windows' }} + name: Install GStreamer id: install-gstreamer shell: powershell env: - RUNTIME_MSI: ${{ steps.msi-path.outputs.runtime }} - DEVELOPMENT_MSI: ${{ steps.msi-path.outputs.development }} + RUNTIME_MSI: ${{ steps.choose-paths.outputs.runtime }} + DEVELOPMENT_MSI: ${{ steps.choose-paths.outputs.development }} run: | $GSTREAMER_INSTALL_DIR="${{ runner.temp }}\gstreamer" $GSTREAMER_ROOT="$GSTREAMER_INSTALL_DIR\1.0\msvc_x86_64" @@ -118,12 +128,18 @@ runs: echo "gstreamer-install-dir=$GSTREAMER_INSTALL_DIR" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 echo "gstreamer-plugin-dir=$GSTREAMER_ROOT" | Out-File -FilePath $Env:GITHUB_OUTPUT -Append -Encoding utf8 - if ($RUNTIME_MSI) { - echo "Installing GStreamer Runtime" + Write-Output "::notice Using runtime: $Env:RUNTIME_MSI" + Write-Output "::notice Using development: $Env:DEVELOPMENT_MSI" + + if ("$Env:RUNTIME_MSI" -ne "") { + Write-Output "::notice title=Installing GStreamer Runtime" Start-Process msiexec -ArgumentList '/i `"$RUNTIME_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn' -NoNewWindow -PassThru -Wait - if ($DEVELOPMENT_MSI) { - echo "Installing GStreamer Development" + if ("$Env:DEVELOPMENT_MSI" -ne "") { + Write-Output "::notice title=Installing GStreamer Development" Start-Process msiexec -ArgumentList '/i `"$DEVELOPMENT_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn' -NoNewWindow -PassThru -Wait } + } else { + Write-Output "::error title=GStreamer is not installed." + exit 999 } diff --git a/.github/actions/setup-gstreamer/action.yml b/.github/actions/setup-gstreamer/action.yml index 40102aa0d..70c0b4723 100644 --- a/.github/actions/setup-gstreamer/action.yml +++ b/.github/actions/setup-gstreamer/action.yml @@ -59,13 +59,20 @@ outputs: runs: using: "composite" steps: + - name: Python >= 3.10 + uses: actions/setup-python@v5 + with: + # cerbero requires >=3.10 + python-version: '>=3.10' - - id: setup-msys2 + - if: runner.os == 'Windows' + id: setup-msys2 uses: msys2/setup-msys2@v2 with: msystem: UCRT64 path-type: inherit release: false + cache: ${{ inputs.no-cache != 'true' }} install: >- mingw-w64-ucrt-x86_64-jq m4 @@ -75,13 +82,24 @@ runs: mingw-w64-ucrt-x86_64-diffutils flex mingw-w64-ucrt-x86_64-gperf - mingw-w64-ucrt-x86_64-make + mingw-w64-ucrt-x86_64-autotools mingw-w64-ucrt-x86_64-ninja mingw-w64-ucrt-x86_64-perl - - shell: msys2 -eo pipefail {0} + - if: runner.os == 'Windows' + shell: msys2 -eo pipefail {0} + name: Additional Windows Configuration (MSYS2) run: git config --global core.autocrlf false + - if: runner.os == 'Windows' + name: Additional Windows Configuration (powershell) + shell: powershell + run: | + New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force + Add-Content "C:\msys64\ucrt64.ini" "`nCHERE_INVOKING=1" + Add-Content "C:\msys64\ucrt64.ini" "`nMSYS2_NOSTART=yes" + Add-Content "C:\msys64\ucrt64.ini" "`nMSYS=winsymlinks:nativestrict" + - id: get-cerbero name: Get Cerbero uses: actions/checkout@v4 @@ -90,8 +108,18 @@ runs: path: cerbero ref: ${{ inputs.cerbero-ref }} + - name: Run Bootstrap + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} + working-directory: cerbero + run: | + if [ "${{ runner.os }}" == "Windows" ]; then + powershell.exe ./tools/bootstrap-windows.ps1 + else + ./tools/bootstrap-debian.sh + fi + - id: version-info - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} working-directory: cerbero run: | # Get GStreamer/cerbero version info and installer filenames @@ -101,22 +129,24 @@ runs: cerbero_short_sha=$(git -C cerbero rev-parse --short HEAD) echo "cerbero-short-sha=${cerbero_short_sha}" | tee -a $GITHUB_OUTPUT - echo "runtime-msi-filename=gstreamer-1.0-msvc-x86_64-${gstreamer_version}-${cerbero_short_sha}.msi" | tee -a $GITHUB_OUTPUT - echo "dev-msi-filename=gstreamer-1.0-devel-msvc-x86_64-${gstreamer_version}-${cerbero_short_sha}.msi" | tee -a $GITHUB_OUTPUT + if [ "${{ runner.os }}" == "Windows" ]; then + echo "runtime-filename=gstreamer-1.0-msvc-x86_64-${gstreamer_version}-${cerbero_short_sha}.msi" | tee -a $GITHUB_OUTPUT + echo "dev-filename=gstreamer-1.0-devel-msvc-x86_64-${gstreamer_version}-${cerbero_short_sha}.msi" | tee -a $GITHUB_OUTPUT + fi - id: check-installer-exists if: ${{ inputs.force != 'true' && inputs.s3-download-paths }} - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} continue-on-error: true env: S3_DOWNLOAD_PATHS: ${{ inputs.s3-download-paths }} - RUNTIME_MSI_FILENAME: ${{ steps.version-info.outputs.runtime-msi-filename }} - DEV_MSI_FILENAME: ${{ steps.version-info.outputs.dev-msi-filename }} + RUNTIME_FILENAME: ${{ steps.version-info.outputs.runtime-filename }} + DEV_FILENAME: ${{ steps.version-info.outputs.dev-filename }} run: | # Check installer already exists in S3 - runtime_msi="null" - dev_msi="null" + runtime_path="null" + dev_path="null" for download_path in ${S3_DOWNLOAD_PATHS}; do echo "Checking for installers under ${download_path}" @@ -124,29 +154,29 @@ runs: bucket=$(echo ${download_path} | cut -d/ -f3) prefix=$(echo ${download_path} | cut -d/ -f4-) - if [ "$runtime_msi" = "null" ]; then + if [ "$runtime_path" = "null" ]; then echo "Checking for runtime installer under s3://${bucket}/${prefix}/" - runtime_msi=$(aws s3api list-objects-v2 --bucket "${bucket}" --prefix "${prefix}/" --query 'Contents[].Key' | jq -r 'if type!="array" then [] else . end | sort_by(length) | map(select(endswith("'"${RUNTIME_MSI_FILENAME}"'"))) | .[0]') - if [ "$runtime_msi" != "null" ]; then - s3_uri="s3://${bucket}/${runtime_msi}" - aws s3 cp --no-progress "${s3_uri}" "cerbero/$(basename ${runtime_msi})" - echo "runtime-path=cerbero/$(basename ${runtime_msi})" | tee -a $GITHUB_OUTPUT + runtime_path=$(aws s3api list-objects-v2 --bucket "${bucket}" --prefix "${prefix}/" --query 'Contents[].Key' | jq -r 'if type!="array" then [] else . end | sort_by(length) | map(select(endswith("'"${RUNTIME_path}"'"))) | .[0]') + if [ "$runtime_path" != "null" ]; then + s3_uri="s3://${bucket}/${runtime_path}" + aws s3 cp --no-progress "${s3_uri}" "cerbero/$(basename ${runtime_path})" + echo "runtime-path=cerbero/$(basename ${runtime_path})" | tee -a $GITHUB_OUTPUT echo "::notice title=GStreamer Runtime Installer Downloaded From S3::Location: ${s3_uri}" fi fi - if [ "$dev_msi" = "null" ]; then + if [ "$dev_path" = "null" ]; then echo "Checking for development installer under s3://${bucket}/${prefix}/" - dev_msi=$(aws s3api list-objects-v2 --bucket "${bucket}" --prefix "${prefix}/" --query 'Contents[].Key' | jq -r 'if type!="array" then [] else . end | sort_by(length) | map(select(endswith("'"${DEV_MSI_FILENAME}"'"))) | .[0]') - if [ "$dev_msi" != "null" ]; then - s3_uri="s3://${bucket}/${dev_msi}" - aws s3 cp --no-progress "${s3_uri}" "cerbero/$(basename ${dev_msi})" - echo "dev-path=cerbero/$(basename ${dev_msi})" | tee -a $GITHUB_OUTPUT + dev_path=$(aws s3api list-objects-v2 --bucket "${bucket}" --prefix "${prefix}/" --query 'Contents[].Key' | jq -r 'if type!="array" then [] else . end | sort_by(length) | map(select(endswith("'"${DEV_FILENAME}"'"))) | .[0]') + if [ "$dev_path" != "null" ]; then + s3_uri="s3://${bucket}/${dev_path}" + aws s3 cp --no-progress "${s3_uri}" "cerbero/$(basename ${dev_path})" + echo "dev-path=cerbero/$(basename ${dev_path})" | tee -a $GITHUB_OUTPUT echo "::notice title=GStreamer Development Installer Downloaded From S3::Location: ${s3_uri}" fi fi - if [ "$runtime_msi" != "null" ] && [ "$dev_msi" != "null" ]; then + if [ "$runtime_path" != "null" ] && [ "$dev_path" != "null" ]; then break fi done @@ -170,28 +200,40 @@ runs: - id: upload-packages if: ${{ steps.build-packages.outcome == 'success' && inputs.s3-upload-path }} name: Upload newly-built GStreamer installers to S3 - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} working-directory: cerbero env: - RUNTIME_MSI_FILENAME: ${{ steps.version-info.outputs.runtime-msi-filename }} - DEV_MSI_FILENAME: ${{ steps.version-info.outputs.dev-msi-filename }} + RUNTIME_FILENAME: ${{ steps.version-info.outputs.runtime-filename }} + DEV_FILENAME: ${{ steps.version-info.outputs.dev-filename }} run: | echo "::group::Upload packages" - upload_path="${{ inputs.s3-upload-path }}/${RUNTIME_MSI_FILENAME}" + upload_path="${{ inputs.s3-upload-path }}/${RUNTIME_FILENAME}" aws s3 cp --no-progress "$(cygpath -u '${{ steps.build-packages.outputs.runtime-installer-path }}')" "${upload_path}" echo "::notice title=GStreamer Runtime Installer Uploaded to S3::Location: ${upload_path}" - echo "runtime-msi=${upload_path}" | tee -a $GITHUB_OUTPUT - upload_path="${{ inputs.s3-upload-path }}/${DEV_MSI_FILENAME}" + upload_path="${{ inputs.s3-upload-path }}/${DEV_FILENAME}" aws s3 cp --no-progress "$(cygpath -u '${{ steps.build-packages.outputs.devel-installer-path }}')" "${upload_path}" echo "::notice title=GStreamer Development Installer Uploaded to S3::Location: ${upload_path}" - echo "dev-msi=${upload_path}" | tee -a $GITHUB_OUTPUT echo "::endgroup::" + - name: Upload runtime artifact + if: ${{ steps.build-packages.outcome == 'success' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.version-info.outputs.runtime-filename }} + path: ${{ steps.build-packages.outputs.runtime-installer-path }} + + - name: Upload development artifact + if: ${{ steps.build-packages.outcome == 'success' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.version-info.outputs.dev-filename }} + path: ${{ steps.build-packages.outputs.devel-installer-path }} + - id: select-path - shell: msys2 -eo pipefail {0} + shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} env: RUNTIME_PATH: ${{ steps.build-packages.outputs.runtime-installer-path || steps.check-installer-exists.outputs.runtime-path }} DEV_PATH: ${{ steps.build-packages.outputs.devel-installer-path || steps.check-installer-exists.outputs.dev-path }} @@ -199,7 +241,7 @@ runs: echo "runtime-installer-path=${RUNTIME_PATH}" | tee -a $GITHUB_OUTPUT echo "development-installer-path=${DEV_PATH}" | tee -a $GITHUB_OUTPUT - # Install the gstreamer MSIs + # Install the gstreamer packages - id: install name: Install GStreamer on Windows uses: ./.github/actions/install-gstreamer diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 396db65ea..e967e9508 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -20,7 +20,7 @@ on: options: - '["floor-to-ceiling-windows-latest-runner"]' - '["self-hosted", "Windows", "X64", "cerbero"]' - default: '["self-hosted", "Windows", "X64", "cerbero"]' + default: '["floor-to-ceiling-windows-latest-runner"]' force-build: description: 'Force the build' required: false @@ -34,8 +34,8 @@ on: jobs: build: - name: Build GStreamer Windows - ${{ (contains(inputs.runner, 'self-hosted') || inputs.runner == '') && 'Self-hosted' || 'GitHub-hosted' }} - runs-on: ${{ fromJSON(inputs.runner || '["self-hosted", "Windows", "X64", "cerbero"]') }} + name: Build GStreamer Windows - ${{ (contains(inputs.runner, 'GitHub-hosted') || inputs.runner == '') && 'GitHub-hosted' || 'Self-hosted' }} + runs-on: ${{ fromJSON(inputs.runner || '["floor-to-ceiling-windows-latest-runner"]') }} steps: - name: Get GStreamer installer S3 prefix id: s3-prefix From 73ad0172e03aa5fdebfc5ad9c1c1715374aae0ec Mon Sep 17 00:00:00 2001 From: Thomas Goodwin Date: Tue, 22 Apr 2025 14:19:35 -0400 Subject: [PATCH 2/6] feat(github): make install-gstreamer install by version, version_ref The install action can now run based off version/version-ref inputs as long as the S3 download location is specified. This should help downstream repositories to "call" this action and simply install a version of gstreamer based on its version. --- .github/actions/install-gstreamer/action.yml | 219 ++++++++++++------- 1 file changed, 142 insertions(+), 77 deletions(-) diff --git a/.github/actions/install-gstreamer/action.yml b/.github/actions/install-gstreamer/action.yml index ee18024a0..270c2212f 100644 --- a/.github/actions/install-gstreamer/action.yml +++ b/.github/actions/install-gstreamer/action.yml @@ -1,17 +1,30 @@ name: Install GStreamer from Packages description: Installs the GStreamer using packages at either the URLs or local paths inputs: + version: + description: 'GStreamer version to install (ignores url and path variables)' + required: false + version-ref: + description: 'GStreamer version ref to install (required if version is specified)' + required: false + s3-download-paths: + description: 'S3 paths to download from (required if version is specified)' + required: false + install-development: + description: 'Install GStreamer development packages (in addition to runtime)' + required: false + default: 'true' runtime-installer-url: description: 'URL of the GStreamer runtime installer' required: false development-installer-url: - description: 'URL of the GStreamer development installer' + description: 'URL of the GStreamer development installer (required if install-development)' required: false runtime-installer-path: description: 'Local path of the GStreamer runtime installer' required: false development-installer-path: - description: 'Local path of the GStreamer development installer' + description: 'Local path of the GStreamer development installer (required if install-development)' required: false outputs: install-dir: @@ -27,96 +40,138 @@ outputs: runs: using: "composite" steps: - # Mutual exclusive -- either set url or path for each. Error if both or neither - # are set. If path is set, verify the file exists. - - if: ${{ inputs.runtime-installer-url && inputs.runtime-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - echo "::error Cannot specify both runtime-installer-url and runtime-installer-path" - exit 1 - - if: ${{ inputs.development-installer-url && inputs.development-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - echo "::error Cannot specify both development-installer-url and development-installer-path" - exit 2 - - if: ${{ !inputs.runtime-installer-url && !inputs.runtime-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - echo "::error Must specify either runtime-installer-url or runtime-installer-path" - exit 3 - - if: ${{ !inputs.development-installer-url && !inputs.development-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - echo "::error Must specify either development-installer-url or development-installer-path" - exit 4 - - if: ${{ inputs.runtime-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - if [[ ! -f "${{ inputs.runtime-installer-path }}" ]]; then - echo "::error Runtime installer path does not exist: ${{ inputs.runtime-installer-path }}" - exit 5 - fi - - if: ${{ inputs.development-installer-path }} - shell: msys2 -eo pipefail {0} - run: | - if [[ ! -f "${{ inputs.development-installer-path }}" ]]; then - echo "::error Development installer path does not exist: ${{ inputs.development-installer-path }}" - exit 6 - fi + - if: runner.os == 'Windows' + id: setup-msys2 + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + path-type: inherit + release: false + - if: ${{ runner.os != 'Windows' }} shell: msys2 -eo pipefail {0} run: | echo "::error This action is only supported on Windows at this time; pull requests welcome!" exit 7 - # If the URL is provided, download the file, set path. - # This will error out if the file is missing. - - if: ${{ !inputs.runtime-installer-path }} - id: download-runtime - name: Download GStreamer Runtime File + - name: Determine installation method(s) from inputs and output path(s) to install files + id: resolve shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} + env: + S3_DOWNLOAD_PATHS: ${{ inputs.s3-download-paths }} run: | - if [ "${{ runner.os }}" == "Windows" ]; then - runtime_path="$(pwd)\gstreamer-runtime.msi" + runner_os="${{ runner.os }}" + version="${{ inputs.version}}" + version_ref="${{ inputs.version-ref }}" + include_development="${{ inputs.install-development }}" + runtime_installer_url="${{ inputs.runtime-installer-url }}" + development_installer_url="${{ inputs.development-installer-url }}" + runtime_installer_path="${{ inputs.runtime-installer-path }}" + development_installer_path="${{ inputs.development-installer-path }}" + + # output vars for later steps + runtime_path="" + development_path="" + result=0 + + s3_find_installer() { + local paths="${1:?Download paths are required as an array}" + local version="${2:?Version is required}" + local uri="" + + for download_path in ${paths}; do + local bucket=$(echo ${download_path} | cut -d/ -f3) + local prefix=$(echo ${download_path} | cut -d/ -f4-) + + file_path=$(aws s3api list-objects-v2 --bucket "${bucket}" --prefix "${prefix}/" --query 'sort_by(Contents, &LastModified)[].Key' | jq -r 'if type!="array" then [] else . end | sort_by(length) | map(select(contains("'"${version}"'"))) | .[0]') + if [ "null" != "${file_path}" ]; then + uri="s3://${bucket}/${file_path}" + fi + done + echo "${uri}" + } + + download_installer() { + local uri="${1}" + local file_name=$(echo "${uri}" | rev | cut -d/ -f1 | rev) + local local_path="$(pwd)/${file_name}" + local result=1 + + if [ -n "${uri}" ]; then + if [ "$(echo "${1}" | cut -c1-5)" = "s3://" ]; then + aws s3 cp --no-progress "${uri}" "${local_path}" &>> download.log + result=$? + else + curl -L -O "${uri}" -o "$local_path" &>> download.log + result=$? + fi + fi + + if [ $result -eq 0 ]; then + echo "${local_path}" + else + echo "" + fi + } + + version_prefix="" + if [ ${runner_os} == 'Windows' ]; then + version_prefix="msvc-x86_64-" fi - curl -L -O "${{ inputs.runtime-installer-url }}" -o "$runtime_path" - echo "runtime-path=$runtime_path" | tee -a $GITHUB_OUTPUT - echo "::notice title=GStreamer runtime installer downloaded from ${{ inputs.runtime-installer-url }}" - - if: ${{ !inputs.development-installer-path }} - id: download-development - name: Download GStreamer Development File - shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} - run: | - if [ "${{ runner.os }}" == "Windows" ]; then - development_path="$(pwd)\gstreamer-development.msi" + + version="${version_prefix}${version}" + if [ -n "${version_ref}" ]; then + version="${version}-${version_ref}" fi - curl -L -O "${{ inputs.development-installer-url }}" -o "$development_path" - echo "development-path=$development_path" | tee -a $GITHUB_OUTPUT - echo "::notice title=GStreamer development installer downloaded from ${{ inputs.development-installer-url }}" - # Set the local paths favoring if the download happened - - id: choose-paths - shell: ${{ runner.os == 'Windows' && 'msys2 -eo pipefail {0}' || 'bash' }} - run: | - # If the download happened, use the downloaded path, else use the input path - runtime="${{ steps.download-runtime.outputs.runtime-path }}" - if [ -z "$runtime" ]; then - runtime="${{ inputs.runtime-installer-path }}" + if [ -f "${runtime_installer_path}" ]; then + runtime_path="${runtime_installer_path}" + elif [ -n "${runtime_installer_url}" ]; then + runtime_path=$(download_installer "${runtime_installer_url}") + else + runtime_installer_url=$(s3_find_installer "${S3_DOWNLOAD_PATHS[@]}" "${version}") + runtime_path=$(download_installer "${runtime_installer_url}") fi - development="${{ steps.download-development.outputs.development-path }}" - if [ -z "$development" ]; then - development="${{ inputs.development-installer-path }}" + + if [ -n "${runtime_path}" ]; then + if [ ${runner_os} == 'Windows' ]; then + runtime_path=$(cygpath -aw "${runtime_path}") + fi + echo "runtime=$runtime_path" | tee -a $GITHUB_OUTPUT + else + echo "::error No GStreamer runtime installer found" + result=1 fi - echo "runtime=$runtime" | tee -a $GITHUB_OUTPUT - echo "development=$development" | tee -a $GITHUB_OUTPUT + + if [ "true" == "${include_development}" ]; then + if [ -f "${development_installer_path}" ]; then + development_path="${development_installer_path}" + elif [ -n "${development_installer_url}" ]; then + development_path=$(download_installer "${development_installer_url}") + else + development_installer_url=$(s3_find_installer "${S3_DOWNLOAD_PATHS[@]}" "devel-${version}") + development_path=$(download_installer "${development_installer_url}") + fi + + if [ -n "${development_path}" ]; then + if [ ${runner_os} == 'Windows' ]; then + development_path=$(cygpath -aw "${development_path}") + fi + echo "development=$development_path" | tee -a $GITHUB_OUTPUT + else + echo "::error No GStreamer development installer found" + result=1 + fi + fi + exit $result - if: ${{ runner.os == 'Windows' }} name: Install GStreamer id: install-gstreamer shell: powershell env: - RUNTIME_MSI: ${{ steps.choose-paths.outputs.runtime }} - DEVELOPMENT_MSI: ${{ steps.choose-paths.outputs.development }} + RUNTIME_MSI: ${{ steps.resolve.outputs.runtime }} + DEVELOPMENT_MSI: ${{ steps.resolve.outputs.development }} run: | $GSTREAMER_INSTALL_DIR="${{ runner.temp }}\gstreamer" $GSTREAMER_ROOT="$GSTREAMER_INSTALL_DIR\1.0\msvc_x86_64" @@ -131,13 +186,23 @@ runs: Write-Output "::notice Using runtime: $Env:RUNTIME_MSI" Write-Output "::notice Using development: $Env:DEVELOPMENT_MSI" - if ("$Env:RUNTIME_MSI" -ne "") { + if (Test-Path -Path "$Env:RUNTIME_MSI") { Write-Output "::notice title=Installing GStreamer Runtime" - Start-Process msiexec -ArgumentList '/i `"$RUNTIME_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn' -NoNewWindow -PassThru -Wait + $proc = Start-Process msiexec -ArgumentList "/i `"$Env:RUNTIME_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn /l*vx runtime-install.log" -NoNewWindow -PassThru -Wait + if ($proc.ExitCode -ne 0) { + Write-Output "::error title=GStreamer Runtime installation failed with exit code $($proc.ExitCode)" + Get-Content runtime-install.log + exit $proc.ExitCode + } - if ("$Env:DEVELOPMENT_MSI" -ne "") { + if (Test-Path -Path "$Env:DEVELOPMENT_MSI") { Write-Output "::notice title=Installing GStreamer Development" - Start-Process msiexec -ArgumentList '/i `"$DEVELOPMENT_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn' -NoNewWindow -PassThru -Wait + $proc = Start-Process msiexec -ArgumentList "/i `"$Env:DEVELOPMENT_MSI`" ADDLOCAL=ALL INSTALLDIR=`"$GSTREAMER_INSTALL_DIR`" /qn /l*vx development-install.log" -NoNewWindow -PassThru -Wait + if ($proc.ExitCode -ne 0) { + Write-Output "::error title=GStreamer Development installation failed with exit code $($proc.ExitCode)" + Get-Content development-install.log + exit $proc.ExitCode + } } } else { Write-Output "::error title=GStreamer is not installed." From fd8ad6709a54d96b38ccbbc01ec0b86502ea9206 Mon Sep 17 00:00:00 2001 From: Thomas Goodwin Date: Tue, 13 May 2025 14:09:01 -0400 Subject: [PATCH 3/6] chore(github): change runner definition --- .github/workflows/windows.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e967e9508..f02576622 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -18,9 +18,9 @@ on: required: false type: choice options: - - '["floor-to-ceiling-windows-latest-runner"]' + - '["windows-2022-16core"]' - '["self-hosted", "Windows", "X64", "cerbero"]' - default: '["floor-to-ceiling-windows-latest-runner"]' + default: '["windows-2022-16core"]' force-build: description: 'Force the build' required: false @@ -35,7 +35,7 @@ on: jobs: build: name: Build GStreamer Windows - ${{ (contains(inputs.runner, 'GitHub-hosted') || inputs.runner == '') && 'GitHub-hosted' || 'Self-hosted' }} - runs-on: ${{ fromJSON(inputs.runner || '["floor-to-ceiling-windows-latest-runner"]') }} + runs-on: ${{ fromJSON(inputs.runner || '["windows-2022-16core"]') }} steps: - name: Get GStreamer installer S3 prefix id: s3-prefix From 4fe795e41a155f6f11131f61fc3d9f0e27392775 Mon Sep 17 00:00:00 2001 From: Jeffery Wilson Date: Mon, 27 Oct 2025 11:57:29 +0000 Subject: [PATCH 4/6] cerbero: Add Windows 2025 Server as supported --- cerbero/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cerbero/utils/__init__.py b/cerbero/utils/__init__.py index 3e89344d3..1ee187a0d 100644 --- a/cerbero/utils/__init__.py +++ b/cerbero/utils/__init__.py @@ -432,6 +432,7 @@ def system_info(): '10': DistroVersion.WINDOWS_10, '11': DistroVersion.WINDOWS_11, '2022Server': DistroVersion.WINDOWS_11, + '2025Server': DistroVersion.WINDOWS_11, } if win32_ver in dmap: distro_version = dmap[win32_ver] From 7c6856cf0610d5b42d22d2706f76166382c0ea1c Mon Sep 17 00:00:00 2001 From: "L. E. Segovia" Date: Sat, 26 Jul 2025 13:46:00 -0300 Subject: [PATCH 5/6] hacks: Fix realpath hack not supporting the strict kwarg Part-of: --- cerbero/hacks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cerbero/hacks.py b/cerbero/hacks.py index daf62efc8..867e68ac3 100644 --- a/cerbero/hacks.py +++ b/cerbero/hacks.py @@ -84,8 +84,8 @@ def abspath(path): return oldabspath(path).replace('\\', '/') -def realpath(path): - return oldrealpath(path).replace('\\', '/') +def realpath(path, **kwargs): + return oldrealpath(path, **kwargs).replace('\\', '/') def relpath(path, start=None): From 44a08c50c40f756d03af64ccecb3707bf4599f11 Mon Sep 17 00:00:00 2001 From: Jeffery Wilson Date: Mon, 27 Oct 2025 12:53:31 +0000 Subject: [PATCH 6/6] ci: force python <3.14 --- .github/actions/setup-gstreamer/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-gstreamer/action.yml b/.github/actions/setup-gstreamer/action.yml index 70c0b4723..6a9bd0cf4 100644 --- a/.github/actions/setup-gstreamer/action.yml +++ b/.github/actions/setup-gstreamer/action.yml @@ -63,7 +63,8 @@ runs: uses: actions/setup-python@v5 with: # cerbero requires >=3.10 - python-version: '>=3.10' + # 3.14 breaks cerbero without https://gitlab.freedesktop.org/gstreamer/cerbero/-/commit/4b2072e5c80c8f84a3db65fa2235057b6d96e5a2 + python-version: '>=3.10 <3.14' - if: runner.os == 'Windows' id: setup-msys2