Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,24 @@ jobs:
mkdir -p ${{ github.workspace }}/vcpkg-archives/
make -f vcpkg-makefile BUILD_ARCH=${{ matrix.arch }}

- name: Collect VCPKG error artifacts (on failure)
if: failure()
run: |
source ${{ github.workspace }}/ci/vcpkg-error-handler.sh
show_vcpkg_errors ${{ github.workspace }}/build-osx ${{ github.workspace }}/vcpkg-error-artifacts
continue-on-error: true

- name: Upload VCPKG error artifacts (on failure)
if: failure()
uses: actions/upload-artifact@v5.0.0
with:
name: vcpkg-errors-osx-${{ matrix.arch }}-${{ github.run_number }}
path: |
vcpkg-error-artifacts/
vcpkg-error-*.tar.gz
retention-days: 30
continue-on-error: true

- name: Compile HDF
run: |
make -f gdal-makefile hdf BUILD_ARCH=${{ matrix.arch }}
Expand Down
24 changes: 24 additions & 0 deletions .github/workflows/unix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,30 @@ jobs:
chmod +x '${{ github.workspace }}/ci/extract-container-artifacts.sh'
'${{ github.workspace }}/ci/extract-container-artifacts.sh' ${{ matrix.arch }} --formats --package-build --metadata --nuget

- name: Extract error artifacts (on failure)
if: failure()
run: |
# Try to extract error artifacts from failed containers
docker ps -a --format "table {{.ID}}\t{{.Image}}\t{{.Status}}" | grep "Exited" | grep "gdal.netcore" | head -1 | awk '{print $1}' | while read container_id; do
if [ -n "$container_id" ]; then
echo "Extracting error artifacts from container: $container_id"
docker cp "$container_id:/build/vcpkg-error-artifacts" ./unix-error-artifacts/ 2>/dev/null || true
docker cp "$container_id:/build/vcpkg-error-*.tar.gz" ./ 2>/dev/null || true
fi
done
continue-on-error: true

- name: Upload VCPKG error artifacts (on failure)
if: failure()
uses: actions/upload-artifact@v5.0.0
with:
name: vcpkg-errors-unix-${{ matrix.arch }}-${{ github.run_number }}
path: |
unix-error-artifacts/
vcpkg-error-*.tar.gz
retention-days: 30
continue-on-error: true

- name: Store metadata as artifact
uses: actions/upload-artifact@v5.0.0
with:
Expand Down
37 changes: 33 additions & 4 deletions ci/Dockerfile.unix
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,42 @@ RUN apt-get update && apt-get install -y \
curl zip unzip tar git wget \
pkg-config \
ninja-build \
autoconf libtool autoconf-archive libudev-dev \
libtool autoconf-archive libudev-dev \
python3 python3-pip \
libhdf4-dev libnetcdf-dev linux-libc-dev \
swig patchelf gettext bison flex \
# required for gdal
default-libmysqlclient-dev && rm -rf /var/lib/apt/lists/* && apt-get clean
default-libmysqlclient-dev && \
rm -rf /var/lib/apt/lists/* && apt-get clean

# Install GCC 11 from Debian bookworm
RUN echo 'deb http://deb.debian.org/debian bookworm main' > /etc/apt/sources.list.d/bookworm.list && \
echo 'Package: *\nPin: release a=bookworm\nPin-Priority: 50\n\nPackage: gcc-11 g++-11 gcc-11-base libgcc-s1\nPin: release a=bookworm\nPin-Priority: 500' > /etc/apt/preferences.d/bookworm && \
apt-get update && \
apt-get install -y gcc-11 g++-11 && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \
rm /etc/apt/sources.list.d/bookworm.list /etc/apt/preferences.d/bookworm && \
apt-get update && \
rm -rf /var/lib/apt/lists/* && apt-get clean

# Install newer autoconf (2.71) from source since Debian 11 has older version
RUN cd /tmp && \
wget https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz && \
tar -xzf autoconf-2.71.tar.gz && \
cd autoconf-2.71 && \
./configure --prefix=/usr/local && \
make && make install && \
cd / && rm -rf /tmp/autoconf-2.71* && \
hash -r

# install cmake latest
RUN pip install cmake --upgrade

ARG DOTNET_VERSION=9.0
ARG DOTNET_INSTALL_DIR=/build/ci/cache/.dotnet

ENV PATH="${PATH}:/usr/share/dotnet"
ENV PATH="/usr/local/bin:/usr/local/gcc-11/bin:${PATH}:/usr/share/dotnet"
# install dotnet with cache mount
RUN --mount=type=cache,target=/tmp/dotnet-cache \
/usr/share/dotnet/dotnet --info || curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Channel $DOTNET_VERSION -InstallDir /usr/share/dotnet
Expand Down Expand Up @@ -72,13 +94,15 @@ RUN --mount=type=cache,target=/build/ci/cache/vcpkg-archives \
FROM vcpkg-stage AS vcpkg-packages
COPY shared/patch /build/shared/patch
COPY unix/vcpkg-makefile /build/unix/vcpkg-makefile
COPY ci/vcpkg-error-handler.sh /build/ci/vcpkg-error-handler.sh
RUN chmod +x /build/ci/vcpkg-error-handler.sh

# Install vcpkg packages with cache
RUN --mount=type=cache,target=/build/ci/cache/vcpkg-archives \
--mount=type=cache,target=/build/build-unix/vcpkg/buildtrees \
--mount=type=cache,target=/build/build-unix/vcpkg/downloads \
set -a && . /tmp/gdal-netcore-env && set +a; \
make -f vcpkg-makefile $(cat /tmp/gdal-netcore-arch)
make -f vcpkg-makefile $(cat /tmp/gdal-netcore-arch) || { /build/ci/vcpkg-error-handler.sh /build/build-unix /build/vcpkg-error-artifacts; exit 1; }

# Stage: Build dependencies (HDF, PROJ)
FROM vcpkg-packages AS deps-build
Expand Down Expand Up @@ -120,6 +144,9 @@ FROM gdalbuild AS package-stage
RUN set -a && . /tmp/gdal-netcore-env && set +a; \
make -f collect-deps-makefile $(cat /tmp/gdal-netcore-arch)

# Create vcpkg-error-artifacts directory to ensure COPY always succeeds
RUN mkdir -p /build/vcpkg-error-artifacts

ARG PRERELEASE
ARG BUILD_NUMBER_TAIL
# Copy only what's needed for packaging - ordered by change frequency (least to most)
Expand All @@ -137,4 +164,6 @@ COPY --from=package-stage /build/package-build/ /build/package-build/
COPY --from=package-stage /build/tests/gdal-formats /build/tests/gdal-formats
# copy nuget packages from build
COPY --from=package-stage /build/nuget /build/nuget
# copy error artifacts if they exist
COPY --from=package-stage /build/vcpkg-error-artifacts /build/vcpkg-error-artifacts
ENTRYPOINT ["bash"]
118 changes: 118 additions & 0 deletions ci/vcpkg-error-handler.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
#!/bin/bash

# VCPKG Error Handler Script
# This script provides detailed error reporting when VCPKG package installations fail
#
# Usage:
# source vcpkg-error-handler.sh
# show_vcpkg_errors [build_dir] [output_dir]
#
# OR as executable:
# ./vcpkg-error-handler.sh [build_dir] [output_dir]
#
# Arguments:
# build_dir - Optional. The build directory path (default: /build/build-unix)
# output_dir - Optional. The output directory for artifacts (default: ./vcpkg-error-artifacts)

show_vcpkg_errors() {
local build_dir="${1:-/build/build-unix}"
local output_dir="${2:-$(pwd)/vcpkg-error-artifacts}"
local vcpkg_installed="${build_dir}/vcpkg/installed"
local vcpkg_buildtrees="${build_dir}/vcpkg/buildtrees"

echo "=== VCPKG BUILD FAILED ===" >&2
echo "Using build directory: $build_dir" >&2
echo "Output directory for artifacts: $output_dir" >&2

# Create output directory for error artifacts
mkdir -p "$output_dir"

echo "Searching for issue_body.md files..." >&2

# Search for issue_body.md files in the installed directory
local issue_files=()
if [[ -d "$vcpkg_installed" ]]; then
while IFS= read -r -d '' issue_file; do
issue_files+=("$issue_file")
echo "=== VCPKG ERROR DETAILS: $issue_file ===" >&2
cat "$issue_file" >&2
echo "=== END ERROR DETAILS ===" >&2
done < <(find "$vcpkg_installed" -name "issue_body.md" -type f -print0 2>/dev/null)
else
echo "VCPKG installed directory not found: $vcpkg_installed" >&2
fi

echo "Searching for build logs..." >&2

# Search for recent build logs in buildtrees directory
local log_files=()
if [[ -d "$vcpkg_buildtrees" ]]; then
local count=0
while IFS= read -r -d '' log_file && [[ $count -lt 10 ]]; do
log_files+=("$log_file")
echo "=== Recent build log: $log_file ===" >&2
tail -50 "$log_file" >&2
echo "=== END LOG ===" >&2
((count++))
done < <(find "$vcpkg_buildtrees" -name "*.log" -type f -mtime -1 -print0 2>/dev/null)
else
echo "VCPKG buildtrees directory not found: $vcpkg_buildtrees" >&2
fi

# Create error artifact if we found any error files
if [[ ${#issue_files[@]} -gt 0 ]] || [[ ${#log_files[@]} -gt 0 ]]; then
echo "Creating error artifact..." >&2

# Copy issue_body.md files
for issue_file in "${issue_files[@]}"; do
local rel_path="${issue_file#$vcpkg_installed/}"
local dest_dir="$output_dir/issue_bodies/$(dirname "$rel_path")"
mkdir -p "$dest_dir"
cp "$issue_file" "$dest_dir/"
done

# Copy recent build logs
for log_file in "${log_files[@]}"; do
local rel_path="${log_file#$vcpkg_buildtrees/}"
local dest_dir="$output_dir/build_logs/$(dirname "$rel_path")"
mkdir -p "$dest_dir"
cp "$log_file" "$dest_dir/"
done

# Create a summary file
cat > "$output_dir/error_summary.txt" << EOF
VCPKG Build Error Summary
Generated: $(date)
Build Directory: $build_dir

Issue Body Files Found: ${#issue_files[@]}
$(printf '%s\n' "${issue_files[@]}")

Build Log Files Found: ${#log_files[@]}
$(printf '%s\n' "${log_files[@]}")

System Information:
OS: $(uname -a)
$(if command -v sw_vers >/dev/null 2>&1; then sw_vers; fi)
EOF

# Create compressed artifact
local artifact_name="vcpkg-error-$(date +%Y%m%d-%H%M%S).tar.gz"
(cd "$(dirname "$output_dir")" && tar -czf "$artifact_name" "$(basename "$output_dir")")
echo "Error artifact created: $(dirname "$output_dir")/$artifact_name" >&2

# Also output the path for CI systems to use
echo "VCPKG_ERROR_ARTIFACT_PATH=$(dirname "$output_dir")/$artifact_name" >> "${GITHUB_ENV:-/dev/null}" 2>/dev/null || true
echo "VCPKG_ERROR_ARTIFACT_DIR=$output_dir" >> "${GITHUB_ENV:-/dev/null}" 2>/dev/null || true
else
echo "No error files found to create artifact" >&2
fi
}

# Export the function so it can be used when sourced
export -f show_vcpkg_errors

# If script is executed directly (not sourced), call the function with provided arguments
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
show_vcpkg_errors "$@"
fi
2 changes: 1 addition & 1 deletion osx/gdal-makefile
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ configure_gdal:
-DHDF4_df_LIBRARY_RELEASE=$(BUILD_ROOT)/hdf-build/lib/libhdf.dylib \
-DHDF4_mfhdf_LIBRARY_RELEASE=$(BUILD_ROOT)/hdf-build/lib/libmfhdf.dylib \
-DHDF4_COMPONENTS="df;mfhdf;szip" \
-DOpenEXR_HALF_LIBRARY=$(VCPKG_INSTALLED_DYNAMIC)/lib/libImath-3_1.dylib \
-DOpenEXR_HALF_LIBRARY=$(VCPKG_INSTALLED_DYNAMIC)/lib/libImath-3_2.dylib \
-DPROJ_INCLUDE_DIR=$(PROJ_BUILD)/include \
-DPROJ_LIBRARY_RELEASE=$(PROJ_BUILD)/lib/libproj.dylib \
-DGDAL_FIND_PACKAGE_PROJ_MODE=MODULE \
Expand Down
51 changes: 44 additions & 7 deletions osx/vcpkg-makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,20 @@ ifneq ($(filter $(REP),$(VCPKG_REQUIRE_OSX)),'')
@if [[ "$(LW)" == *"-force"* ]] ; then \
$(MAKE) -f vcpkg-makefile upgrade; \
$(VCPKG) remove $(REP):$(VCPKG_RID) --recurse || exit 1; \
$(VCPKG) install $(REP):$(VCPKG_RID) --recurse --clean-after-build || exit 1; \
$(VCPKG) install $(REP):$(VCPKG_RID) --recurse --clean-after-build || { \
echo "$(TO) ERROR: Failed to install $(REP):$(VCPKG_RID)" >&2; \
if [ -f $(ROOTDIR_)/ci/vcpkg-error-handler.sh ]; then \
chmod +x $(ROOTDIR_)/ci/vcpkg-error-handler.sh; \
$(ROOTDIR_)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOTDIR_)/vcpkg-error-artifacts; \
else \
echo "$(TO) Searching for issue_body.md for $(REP)..." >&2; \
find $(VCPKG_ROOT)/installed -path "*/$(REP)*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \
echo "$(TO) Found issue file: $$issue_file" >&2; \
cat "$$issue_file" >&2; \
done; \
fi; \
exit 1; \
}; \
else \
echo "To rebuild add {key}-force"; \
$(VCPKG) list; \
Expand Down Expand Up @@ -108,20 +121,44 @@ patch_vcpkg_triplets:

install_packages:
@for pack in $(VCPKG_REQUIRE_OSX); do \
echo "$(TO) Installing $$pack:$(VCPKG_RID)..."; \
MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID) $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || \
{ \
echo "Error occurred while installing $$pack"; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log; \
echo "$(TO) ERROR: Failed to install $$pack:$(VCPKG_RID)" >&2; \
if [ -f $(ROOTDIR_)/ci/vcpkg-error-handler.sh ]; then \
chmod +x $(ROOTDIR_)/ci/vcpkg-error-handler.sh; \
$(ROOTDIR_)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOTDIR_)/vcpkg-error-artifacts; \
else \
echo "$(TO) Searching for issue_body.md for $$pack..." >&2; \
find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \
echo "$(TO) Found issue file: $$issue_file" >&2; \
cat "$$issue_file" >&2; \
done; \
echo "$(TO) Checking build logs..." >&2; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log 2>/dev/null || true; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log 2>/dev/null || true; \
fi; \
exit 1; \
}; \
done
@for pack in $(VCPKG_REQUIRE_OSX_DYNAMIC); do \
echo "$(TO) Installing $$pack:$(VCPKG_RID)-dynamic..."; \
MAKELEVEL=0 $(VCPKG) install $$pack:$(VCPKG_RID)-dynamic $(VCPKG_PARAMS_INSTALL) $(VCPKG_ENSURE_INSTALLED) || \
{ \
echo "Error occurred while installing $$pack"; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log; \
echo "$(TO) ERROR: Failed to install $$pack:$(VCPKG_RID)-dynamic" >&2; \
if [ -f $(ROOTDIR_)/ci/vcpkg-error-handler.sh ]; then \
chmod +x $(ROOTDIR_)/ci/vcpkg-error-handler.sh; \
$(ROOTDIR_)/ci/vcpkg-error-handler.sh $(BUILD_ROOT) $(ROOTDIR_)/vcpkg-error-artifacts; \
else \
echo "$(TO) Searching for issue_body.md for $$pack..." >&2; \
find $(VCPKG_ROOT)/installed -path "*/$$pack*/issue_body.md" -type f 2>/dev/null | while read -r issue_file; do \
echo "$(TO) Found issue file: $$issue_file" >&2; \
cat "$$issue_file" >&2; \
done; \
echo "$(TO) Checking build logs..." >&2; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*err.log 2>/dev/null || true; \
cat $(VCPKG_ROOT)/buildtrees/$$pack/*out.log 2>/dev/null || true; \
fi; \
exit 1; \
}; \
done
Expand Down
2 changes: 1 addition & 1 deletion shared/GdalCore.opt
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ PROJ_COMMIT_VER=$(PROJ_VERSION)

VCPKG_ROOT=$(BUILD_ROOT)/vcpkg
VCPKG_REPO=https://github.com/microsoft/vcpkg.git
VCPKG_COMMIT_VER=2025.06.13
VCPKG_COMMIT_VER=2025.10.17

# base requirements for all runtimes
VCPKG_REQUIRE=geos "tiff[zstd,zip,jpeg,tools,lzma,cxx,webp]" "curl[tool,openssl]" "poppler[cairo,cms,zlib,glib,curl,private-api]"
Expand Down
12 changes: 5 additions & 7 deletions unix/gdal-makefile
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ configure_gdal:
-DCSHARP_APPLICATION_VERSION=$(GDAL_CSHARP_VERSION) \
-DBUILD_JAVA_BINDINGS=OFF \
-DBUILD_SHARED_LIBS=ON \
-DOpenEXR_HALF_LIBRARY=$(VCPKG_INSTALLED_DYNAMIC)/lib/libImath-3_1.so \
-DOpenEXR_HALF_LIBRARY=$(VCPKG_INSTALLED_DYNAMIC)/lib/libImath-3_2.so \
-DCMAKE_PREFIX_PATH=$(VCPKG_INSTALLED_DYNAMIC) \
-DCMAKE_INSTALL_PREFIX=$(BUILD_ROOT)/gdal-build \
-DHDF4_INCLUDE_DIR=$(BUILD_ROOT)/hdf-build/include \
Expand Down Expand Up @@ -183,14 +183,12 @@ build_%:
@echo "$(TO) $(LW) was built successfully!"

# Persist C# bindings generated by GDAL build (from cmake temp) into install layer
# We need only C# sources and .so wrappers from current dir and gdal, ogr, osr, const directories
@if [ "$(LW)" = "gdal" ]; then \
mkdir -p $(GDAL_BUILD)/swig/csharp; \
if [ -d "$(GDAL_CMAKE_TMP)/swig/csharp" ]; then \
cp -rfv $(GDAL_CMAKE_TMP)/swig/csharp/* $(GDAL_BUILD)/swig/csharp/ && \
echo "$(TO) Bindings copied successfully" \
else \
echo "$(TO) WARNING: $(GDAL_CMAKE_TMP)/swig/csharp not found!"; \
fi; \
cd $(GDAL_CMAKE_TMP)/swig/csharp && \
find . gdal ogr osr const -type f \( -name "*.so" -o -name "*.cs" \) ! -path "*/bin/*" ! -path "*/obj/*" \
-exec sh -c 'mkdir -p "$(GDAL_BUILD)/swig/csharp/$$(dirname "{}")" && cp -v "{}" "$(GDAL_BUILD)/swig/csharp/{}"' \; ; \
fi;


Expand Down
Loading