From 24f1e30cd9723bbd396058d163bd764cf4f7727b Mon Sep 17 00:00:00 2001 From: Nox-MSFT Date: Thu, 4 Dec 2025 23:44:18 +0000 Subject: [PATCH 1/4] Enable Ubuntu 24.04 LTS support with GCC 13 compatibility Major Changes: - Add Ubuntu 24.04 platform support with GCC 13.3.0 - Disable Delivery Optimization (not available on 24.04) - Add Azure Storage SDK GCC 12+ compatibility patch - Update documentation for all supported platforms Build System Changes: - scripts/install-deps.sh: GCC version detection and conditional patching * Skip gcc-8 installation on Ubuntu 24.04+ * Auto-detect GCC version and apply Azure Storage SDK patch for GCC >= 12 * Skip DO client installation on Ubuntu 24.04 * Change workspace directory to .workspace/ * Fix file ownership issues - scripts/build.sh: Ubuntu 24.04 detection * Auto-detect Ubuntu 24.04 and disable DO build * Update workspace path to .workspace/ CMake Changes: - CMakeLists.txt: Add ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION option - src/extensions/content_downloaders/CMakeLists.txt: Conditional DO build - packages/CMakeLists.txt: Platform-specific package dependencies - cmake/agentRules.cmake: Conditional DO SDK linking Package Changes: - packages/debian/postinst: Smart downloader registration * Register DO downloader if available, otherwise curl Patches: - scripts/patches/azure-storage-sdk-base64-cstdint.patch: New patch * Fixes missing includes for GCC 12+ * Applied to Azure Storage SDK base64.hpp, base64.cpp, uuid.hpp Documentation: - README.md: Updated platform support table * Added Ubuntu 20.04, Debian 11/12 platforms * Added platform-specific notes for DO availability * Fixed Quick Start command to include unit tests flag - docs/agent-reference/how-to-build-agent-code.md: Comprehensive rewrite * Expanded platform support table with all distributions * Build dependencies table with LKG versions * Runtime dependencies section * Detailed per-dependency build instructions * GCC 12+ patch documentation with automatic detection * Platform-specific build notes for each OS version * Troubleshooting section Tested: - Full build successful on Ubuntu 24.04 with GCC 13.3.0 - All 148 unit tests passed (100%) - Debian package created: deviceupdate-agent_1.2.0_amd64.deb - curl content downloader registered as primary downloader --- .gitignore | 3 + CMakeLists.txt | 2 + README.md | 87 ++-- cmake/agentRules.cmake | 30 +- .../how-to-build-agent-code.md | 427 +++++++++++++++++- packages/CMakeLists.txt | 8 +- packages/debian/postinst | 12 +- scripts/build.sh | 9 +- scripts/install-deps.sh | 50 +- .../azure-storage-sdk-base64-cstdint.patch | 30 ++ .../content_downloaders/CMakeLists.txt | 4 +- 11 files changed, 569 insertions(+), 93 deletions(-) create mode 100644 scripts/patches/azure-storage-sdk-base64-cstdint.patch diff --git a/.gitignore b/.gitignore index b1aaf78af..6cb6f083a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ ### VisualStudio 2017 .vs/ +### Temporary workspace files +.workspace/ + ### GDB debugger .gdbinit .gdb_history diff --git a/CMakeLists.txt b/CMakeLists.txt index 03bbceb6d..ae768665a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -517,6 +517,8 @@ option (ADUC_ENABLE_E2E_TESTING "Enable e2e test pipeline settings" OFF) option (ADUC_ENABLE_SRVC_E2E_TESTING "Enable service side test agent settings" OFF) option (ADUC_ROOTKEY_PKG_DOWNLOAD_WITH_CURL "Use Curl to download the Rootkey Package instead of DeliveryOptimization agent" OFF) +option (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION + "Build with Delivery Optimization support" ON) ### End CMake Options diff --git a/README.md b/README.md index 09bdc8b64..0d5776867 100644 --- a/README.md +++ b/README.md @@ -1,73 +1,60 @@ -# What is Device Update for IoT Hub? +# Device Update for IoT Hub -Device Update for IoT Hub is a service that enables you to deploy over-the-air updates (OTA) for your IoT devices. +Device Update for IoT Hub is an end-to-end platform for deploying over-the-air updates (OTA) to your IoT devices—from tiny sensors to gateway-level devices. -Device Update for IoT Hub is an end-to-end platform that customers can use to publish, distribute, and manage over-the-air updates for everything from tiny sensors to gateway-level devices. +## Key Features -Device Update for IoT Hub also provides controls on how to manage the deployment updates so you are always in control of when and how devices are updated. Device Update for IoT Hub also provides reporting capabilities so you are always up to date on the state of your devices via integration with IoT Hub. +* **Integrated Management** - Update management UX integrated with Azure IoT Hub +* **Controlled Rollout** - Gradual update deployment through device grouping and scheduling +* **Flexible APIs** - Programmatic APIs for automation and custom portal experiences +* **Fleet Visibility** - At-a-glance compliance and status views across heterogeneous device fleets +* **Resilient Updates** - Support for A/B updates with seamless rollback +* **Access Control** - Subscription and role-based access controls via Azure portal +* **Offline Support** - On-premise content cache and Nested Edge support for disconnected devices +* **Comprehensive Reporting** - Detailed update management and reporting tools -Device Update for IoT Hub features provide a powerful and flexible experience, including: +## Supported Platforms -* Update management UX integrated with Azure IoT Hub -* Gradual update rollout through device grouping and update scheduling controls -* Programmatic APIs to enable automation and custom portal experiences -* At-a-glance update compliance and status views across heterogenous device fleets -* Support for resilient device updates (A/B) to deliver seamless rollback -* Subscription and role-based access controls available through the Azure.com portal -* On-premise content cache and Nested Edge support to enable updating cloud disconnected devices -* Detailed update management and reporting tools - -## Reference agent - -| Build | Status | -|------------------- |--------| -| Ubuntu 22.04 AMD64 | [![Ubuntu 22.04 Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main)| - -## Getting started - -* [Device Update for IoT Hub](https://aka.ms/iot-hub-device-update-docs) -* [Getting Started with Device Update Agent](./docs/agent-reference) -* More details on building the agent here: [How to build agent code](./docs/agent-reference/how-to-build-agent-code.md) +| Platform | Status | Notes | +|------------------- |--------|-------| +| Ubuntu 20.04 AMD64 | ✓ Supported | With Delivery Optimization | +| Ubuntu 22.04 AMD64 | [![Ubuntu 22.04 Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main)| Primary development platform | +| Ubuntu 24.04 AMD64 | ✓ Supported | curl downloader only (DO not available) | +| Debian 11 (Bullseye) AMD64 | ✓ Supported | With Delivery Optimization | +| Debian 12 (Bookworm) AMD64 | ✓ Supported | With Delivery Optimization | ## Quick Start -### Build and Install +For users familiar with the build process: ```sh ./scripts/install-deps.sh -a ./scripts/build.sh -c -u --build-packages -cd out -sudo cmake --build . --target install +sudo apt install ./out/deviceupdate-agent_*.deb ``` -### Incremental Build +> **Note**: For detailed build instructions, troubleshooting, and advanced options, see [How to Build the Agent](./docs/agent-reference/how-to-build-agent-code.md) -```sh -cd out -ninja -``` +## Documentation -### Run Tests +* **[Device Update for IoT Hub Documentation](https://aka.ms/iot-hub-device-update-docs)** - Official service documentation +* **[Getting Started with the Agent](./docs/agent-reference)** - Agent overview and reference +* **[Building the Agent](./docs/agent-reference/how-to-build-agent-code.md)** - Detailed build instructions +* **[Running the Agent](./docs/agent-reference/how-to-run-agent.md)** - Deployment and configuration +* **[Troubleshooting Guide](./docs/how-to-troubleshoot-guide.md)** - Common issues and solutions -```sh -cd out -ctest -``` +## Contributing -or, alternatively: +This project welcomes contributions and suggestions. See [CONTRIBUTING.md](./CONTRIBUTING.md) for details. -```sh -ninja test -``` +## Security -### Run tests under valgrind memcheck +See [SECURITY.md](./SECURITY.md) for information on reporting security issues. -Ensure /usr/bin/valgrind is a valid symlink -e.g. `sudo ln -s /opt/valgrind.3.19.0/bin/valgrind /usr/bin/valgrind` +## License -```sh -cd out -ctest -T memcheck -``` +This project is licensed under the MIT License - see [LICENSE](./LICENSE) for details. + +## Support -Results will be in `out/Testing/Temporary/MemoryChecker.*.log` +For support options, see [SUPPORT.md](./SUPPORT.md). diff --git a/cmake/agentRules.cmake b/cmake/agentRules.cmake index 490f7e83a..33e07f85f 100644 --- a/cmake/agentRules.cmake +++ b/cmake/agentRules.cmake @@ -64,21 +64,23 @@ function (target_link_digital_twin_client target scope) endfunction (target_link_digital_twin_client) function (target_link_dosdk target scope) - if (NOT WIN32) - find_package (deliveryoptimization_sdk CONFIG REQUIRED) - target_link_libraries (${target} ${scope} Microsoft::deliveryoptimization) - else () - if (CMAKE_GENERATOR_PLATFORM STREQUAL "") - set (DO_CLIENT_INSTALL_FOLDER "${VCPKG_INSTALLED_DIR}/do-client") + if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + if (NOT WIN32) + find_package (deliveryoptimization_sdk CONFIG REQUIRED) + target_link_libraries (${target} ${scope} Microsoft::deliveryoptimization) else () - # Different install folder if cross-compiling - see build.ps1 - set (DO_CLIENT_INSTALL_FOLDER - "${VCPKG_INSTALLED_DIR}/do-client-${CMAKE_GENERATOR_PLATFORM}") - endif () + if (CMAKE_GENERATOR_PLATFORM STREQUAL "") + set (DO_CLIENT_INSTALL_FOLDER "${VCPKG_INSTALLED_DIR}/do-client") + else () + # Different install folder if cross-compiling - see build.ps1 + set (DO_CLIENT_INSTALL_FOLDER + "${VCPKG_INSTALLED_DIR}/do-client-${CMAKE_GENERATOR_PLATFORM}") + endif () - target_include_directories (${target} ${scope} ${DO_CLIENT_INSTALL_FOLDER}/sdk-cpp/include) - target_link_directories (${target} PUBLIC - ${DO_CLIENT_INSTALL_FOLDER}/cmake/sdk-cpp/${CMAKE_BUILD_TYPE}) - target_link_libraries (${target} ${scope} deliveryoptimization) + target_include_directories (${target} ${scope} ${DO_CLIENT_INSTALL_FOLDER}/sdk-cpp/include) + target_link_directories (${target} PUBLIC + ${DO_CLIENT_INSTALL_FOLDER}/cmake/sdk-cpp/${CMAKE_BUILD_TYPE}) + target_link_libraries (${target} ${scope} deliveryoptimization) + endif () endif () endfunction (target_link_dosdk) diff --git a/docs/agent-reference/how-to-build-agent-code.md b/docs/agent-reference/how-to-build-agent-code.md index d6e8fe9c4..f4a9499f0 100644 --- a/docs/agent-reference/how-to-build-agent-code.md +++ b/docs/agent-reference/how-to-build-agent-code.md @@ -1,30 +1,264 @@ # How To Build the Device Update Agent -Take a look at [dependencies](how-to-build-agent-code.md#dependencies-of-device-update-agent) before you get started. You can build the Device Update agent as a standlone solution or integrate it in your existing application or solution. +This guide provides detailed instructions for building the Device Update for IoT Hub agent from source. -- [Dependencies](how-to-build-agent-code.md#dependencies-of-device-update-agent) -- [As a standalone solution](how-to-build-agent-code.md#as-a-standalone-solution) +## Table of Contents -## Dependencies of Device Update Agent +- [Supported Platforms](#supported-platforms) +- [Build Dependencies](#build-dependencies) +- [Runtime Dependencies](#runtime-dependencies) +- [Quick Start](#quick-start) +- [Building the Agent](#building-the-device-update-agent-for-linux) +- [Building Dependencies from Source](#building-dependencies-from-source) +- [Testing](#build-and-run-the-unit-tests) +- [Packaging](#build-the-debian-package) +- [Installation](#install-the-device-update-agent) +- [Platform-Specific Build Notes](#platform-specific-build-notes) +- [Advanced Options](#build-options-for-mqtt-and-mqtt-over-websockets-iothub-transport-providers) +- [Troubleshooting](#troubleshooting) -### Required Dependencies +## Supported Platforms -- Azure IoT C SDK -- Delivery Optimization SDK -- Azure Blob Storage File Upload Utility -- IotHub Device Update Delta +The Device Update agent has been tested and verified on the following platforms: -### Azure IoT C SDK +| OS Distribution | Version | Architecture | Status | Build Status | Notes | +|----------------|---------|--------------|--------|--------------|-------| +| Ubuntu | 20.04 LTS | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Ubuntu | 22.04 LTS | AMD64 | ✓ Fully Supported | [![Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main) | Primary development platform | +| Ubuntu | 24.04 LTS | AMD64 | ✓ Fully Supported | - | curl downloader only, [see notes](#ubuntu-2404-lts-specifics) | +| Ubuntu | 18.04 LTS | AMD64 | ✗ Not Supported | - | End of support | +| Debian | 11 (Bullseye) | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Debian | 12 (Bookworm) | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Debian | 10 (Buster) | AMD64 | ✗ Not Supported | - | End of support | -Use the [Azure IoT C -SDK](https://github.com/Azure/azure-iot-sdk-c) -to connect to IoT Hub and call Azure IoT Plug and Play APIs. +## Build Dependencies -### Delivery Optimization +The following table lists all build-time dependencies with their Last Known Good (LKG) versions and platform availability. -The [Delivery Optimization -SDK](https://github.com/microsoft/do-client) -provides a robust way for the client to download an update. +**Note:** Dependencies marked with an asterisk (*) require special attention. Click the link for details about patches, limitations, or special build instructions. + +### Core Build Tools + +| Tool | Minimum Version | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | Source | +|------|----------------|--------------|--------------|--------------|-----------|-----------|--------| +| GCC | 10.0 | gcc-10 | gcc-10/11 | gcc-13 | gcc-10 | gcc-12 | System package | +| CMake | 3.10 | 3.16.3 | 3.22.1 | 3.28.3 | 3.18.4 | 3.25.1 | System package or [built from source](#installing-dependencies) | +| Ninja | 1.10+ | 1.10.0 | 1.10.2 | 1.11.1 | 1.10.1 | 1.11.1 | System package | +| Git | 2.0+ | 2.25.1 | 2.34.1 | 2.43.0 | 2.30.2 | 2.39.2 | System package | +| pkg-config | - | ✓ | ✓ | ✓ | ✓ | ✓ | System package | + +### Core Dependencies Built from Source + +| Dependency | LKG Version/Tag | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | Notes | +|------------|----------------|--------------|--------------|--------------|-----------|-----------|-------| +| Azure IoT C SDK | `LTS_08_2023` | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-azure-iot-c-sdk) | +| Azure Storage SDK for C++ | `azure-core_1.6.0` | ✓ | ✓ | ✓* | ✓ | ✓* | [*Requires GCC 12+ patch](#building-azure-storage-sdk-for-c) | +| Delivery Optimization SDK | `main` (latest) | ✓ | ✓ | ✗ | ✓ | ✓ | [Not available on Ubuntu 24.04](#building-delivery-optimization-sdk) | +| Catch2 | `v3.8.0` | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-catch2) (unit tests only) | +| Parson | Latest | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-parson) | +| Microsoft Delta Download Handler | Submodule | ✓ | ✓ | ✓ | ✓ | ✓ | Built with agent | + +### System Package Dependencies + +| Package | Purpose | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | +|---------|---------|--------------|--------------|--------------|-----------|-----------| +| libcurl4-openssl-dev | HTTP/HTTPS client | ✓ Required | ✓ Required | ✓ Required | ✓ Required | ✓ Required | +| libssl-dev | TLS/Crypto | ✓ (1.1.1) | ✓ (3.0) | ✓ (3.0) | ✓ (1.1.1) | ✓ (3.0) | +| uuid-dev | UUID generation | ✓ | ✓ | ✓ | ✓ | ✓ | +| zlib1g-dev | Compression | ✓ | ✓ | ✓ | ✓ | ✓ | + +### Optional Development Tools + +| Tool | Purpose | Installation | +|------|---------|--------------| +| clang-format | Code formatting | `sudo apt install clang-format` | +| cmake-format | CMake formatting | `sudo apt install python3-pip && sudo pip3 install cmake-format` | +| valgrind | Memory testing | `sudo apt install valgrind` (3.19+ recommended) | +| shellcheck | Shell script linting | Via install-deps.sh | + +## Runtime Dependencies + +The following dependencies are required to run the Device Update agent on deployed devices. + +### Core Runtime Requirements + +| Dependency | Minimum Version | Package Name (Ubuntu) | Purpose | +|------------|----------------|----------------------|---------| +| systemd | 237+ | systemd | Daemon management | +| libssl | 1.1+ | libssl3 (22.04+), libssl1.1 (20.04) | TLS/crypto operations | +| libcurl | 7.58+ | libcurl4 | HTTP client | +| curl (binary) | 7.58+ | curl | Content download (24.04), rootkey download | + +### Platform-Specific Runtime Dependencies + +**Ubuntu 20.04, 22.04, Debian 11, Debian 12:** +- `deliveryoptimization-agent` >= 1.0.0 (primary downloader) +- `libdeliveryoptimization` >= 1.0.0 +- `curl` (fallback downloader) + +**Ubuntu 24.04:** +- `curl` (primary downloader) +- Note: Delivery Optimization not available + +## Quick Start + +For most users, building the agent is straightforward: + +```sh +# Install all dependencies +./scripts/install-deps.sh -a + +# Build the agent with unit tests and create Debian package +./scripts/build.sh -c -u --build-packages + +# Install the package +sudo apt install ./out/deviceupdate-agent_*.deb +``` + +For incremental builds after the initial build: + +```sh +cd out +ninja +``` + +To run tests: + +```sh +cd out +ctest +# or +ninja test +``` + +## Building Dependencies from Source + +The `install-deps.sh` script automates building dependencies, but this section documents the process, required versions, and any patches needed for each dependency. + +### Building Azure IoT C SDK + +**Repository:** [https://github.com/Azure/azure-iot-sdk-c](https://github.com/Azure/azure-iot-sdk-c) +**LKG Version:** `LTS_08_2023` branch +**Build Location:** `.workspace/azure-iot-sdk-c` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) + +The Azure IoT C SDK provides the connectivity layer to Azure IoT Hub and implements the Azure IoT Plug and Play APIs. + +**Patches Required:** None + +**Build Options:** +- MQTT transport (`use_mqtt=ON`) +- MQTT over WebSockets (`use_wsio=ON`) +- Both are built by default + +**Known Issues:** None + +--- + +### Building Azure Storage SDK for C++ + +**Repository:** [https://github.com/Azure/azure-sdk-for-cpp](https://github.com/Azure/azure-sdk-for-cpp) +**LKG Version:** `azure-core_1.6.0` tag +**Build Location:** `.workspace/azure_storage_sdk_dir` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04*, Debian 11, 12*) + +The Azure Storage SDK for C++ provides blob storage operations for file uploads. + +#### GCC 12+ Compatibility Patch + +**Affected Platforms:** Platforms using GCC 12 or later +- Ubuntu 24.04 (GCC 13) +- Debian 12 (GCC 12) +- Any custom build environment with GCC 12+ + +**Issue:** GCC 12 and later removed implicit standard library includes that previous versions provided. The Azure Storage SDK compilation fails with errors like: +- `error: 'uint8_t' does not name a type` +- `error: 'uint16_t' was not declared in this scope` + +**Patch File:** `scripts/patches/azure-storage-sdk-base64-cstdint.patch` + +**Affected Files:** +- `sdk/core/azure-core/inc/azure/core/internal/cryptography/base64.hpp` +- `sdk/core/azure-core/src/cryptography/base64.cpp` +- `sdk/core/azure-core/inc/azure/core/uuid.hpp` + +**Solution:** The patch adds explicit `#include ` directives to the affected files. + +**Automatic Application:** +The `install-deps.sh` script automatically detects the GCC version during the Azure Storage SDK build process. If GCC version is 12 or later, the patch is applied automatically. Platforms with GCC 10 or 11 do not require or receive this patch. + +**Manual Application:** +```sh +cd .workspace/azure_storage_sdk_dir +git apply ../../scripts/patches/azure-storage-sdk-base64-cstdint.patch +``` + +**Verification:** +After patching, the SDK compiles successfully with GCC 13 without warnings or errors. + +--- + +### Building Delivery Optimization SDK + +**Repository:** [https://github.com/microsoft/do-client](https://github.com/microsoft/do-client) +**LKG Version:** `main` branch (latest) +**Build Location:** `.workspace/do` +**Platforms:** Ubuntu 20.04, 22.04, Debian 11, 12 + +The Delivery Optimization SDK provides robust, peer-to-peer content distribution for update downloads. + +**Availability:** +- ✓ **Ubuntu 20.04:** Fully supported +- ✓ **Ubuntu 22.04:** Fully supported +- ✓ **Debian 11:** Fully supported +- ✓ **Debian 12:** Fully supported +- ✗ **Ubuntu 24.04:** **NOT AVAILABLE** - Package not maintained for this version + +**Alternative on Ubuntu 24.04:** +The agent automatically uses the curl content downloader (`libcurl_content_downloader.so`) as a replacement. The build system detects Ubuntu 24.04 and configures accordingly: +- Sets `ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF` +- Registers curl downloader as primary content downloader +- Package dependencies exclude DO packages + +**Build Command (Ubuntu 20.04/22.04, Debian 11/12):** +```sh +./scripts/install-deps.sh --install-do +``` + +**Patches Required:** None + +**Known Limitations on Ubuntu 24.04:** +- No peer-to-peer download optimization +- All content downloads via direct HTTP/HTTPS using curl +- Functionally equivalent but may use more bandwidth in fleet scenarios + +--- + +### Building Catch2 + +**Repository:** [https://github.com/catchorg/Catch2](https://github.com/catchorg/Catch2) +**LKG Version:** `v3.8.0` tag +**Build Location:** `.workspace/catch2` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) +**Purpose:** Unit testing framework + +**Patches Required:** None + +**Note:** Only required for building and running unit tests. Not needed for production builds. + +--- + +### Building Parson + +**Repository:** [https://github.com/kgabis/parson](https://github.com/kgabis/parson) +**LKG Version:** Latest from master +**Build Location:** `.workspace/parson` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) +**Purpose:** Lightweight JSON parser + +**Patches Required:** None + +**Known Issues:** None ## Building the Device Update Agent for Linux @@ -288,6 +522,165 @@ After building the Debian package using `build.sh --build-packages`, do: sudo apt install ./out/{PKG_NAME}.deb ``` +## Platform-Specific Build Notes + +This section contains important platform-specific information, limitations, and workarounds. + +### Ubuntu 20.04 LTS Specifics + +**GCC Version:** 10.3.0 (default) +**CMake Version:** 3.16.3 (system), can use newer from install-deps.sh + +**Delivery Optimization:** +- Fully supported +- Installed as system packages + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Ubuntu 22.04 LTS Specifics + +**GCC Version:** 11.x (default) or 10.x +**CMake Version:** 3.22.1 (system) + +**Delivery Optimization:** +- Fully supported +- Primary development and CI/CD platform + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Ubuntu 24.04 LTS Specifics + +**GCC Version:** 13.3.0 (system default) +**CMake Version:** 3.28.3 (system) + +**Key Changes from Previous Versions:** +- GCC 8/9/10 installation automatically skipped +- Uses system default GCC 13 compiler +- Build system auto-detects Ubuntu 24.04 + +**Delivery Optimization:** +- **NOT AVAILABLE** - Package not maintained for Ubuntu 24.04 +- Build automatically disables DO via `ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF` +- `build.sh` detects Ubuntu 24.04 and configures accordingly + +**Content Downloader:** +- Primary: `libcurl_content_downloader.so` +- Automatically registered during package installation +- No Delivery Optimization fallback available + +**Compiler Compatibility:** +- GCC 13 requires Azure Storage SDK patch (applied automatically by install-deps.sh when GCC >= 12) +- See [Building Azure Storage SDK for C++](#building-azure-storage-sdk-for-c) for details + +**Package Dependencies:** +- Debian package depends on `curl` only +- No `deliveryoptimization-agent` or `libdeliveryoptimization` dependencies + +**Known Limitations:** +- No peer-to-peer download optimization +- All downloads are direct HTTP/HTTPS via curl + +--- + +### Debian 11 (Bullseye) Specifics + +**GCC Version:** 10.2.1 (default) +**CMake Version:** 3.18.4 (system) + +**Delivery Optimization:** +- Fully supported +- Installed via install-deps.sh + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Debian 12 (Bookworm) Specifics + +**GCC Version:** 12.2.0 (default) +**CMake Version:** 3.25.1 (system) + +**Delivery Optimization:** +- Fully supported +- Installed via install-deps.sh + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Compiler Compatibility:** +- GCC 12 requires Azure Storage SDK patch (applied automatically by install-deps.sh when GCC >= 12) +- See [Building Azure Storage SDK for C++](#building-azure-storage-sdk-for-c) for details + +**Known Issues:** None + +--- + +## Troubleshooting + +### Build Failures + +#### Azure Storage SDK Compilation Errors (GCC 12+) + +If you encounter errors related to `uint8_t` or missing `` includes when building the Azure Storage SDK, ensure you're using the latest version of `install-deps.sh` which includes automatic GCC version detection and patch application for GCC 12+. This affects: +- Ubuntu 24.04 (GCC 13) +- Debian 12 (GCC 12) +- Any custom environment using GCC 12 or later + +The patch is only applied when the build system detects GCC version 12 or higher. + +#### Permission Issues + +If you encounter permission errors with files in `.workspace/`, the build scripts will automatically fix ownership. If issues persist, manually correct ownership: + +```sh +sudo chown -R $(id -un):$(id -gn) .workspace/ +``` + +#### Symlink Issues + +If cmake or shellcheck symlinks are broken, re-run the install script: + +```sh +./scripts/install-deps.sh --install-packages +``` + +### Valgrind Setup + +To run tests under valgrind memcheck, ensure valgrind 3.19.0 or later is installed: + +```sh +# Verify valgrind is accessible +which valgrind +# If not found, create symlink to your valgrind installation +sudo ln -s /opt/valgrind.3.19.0/bin/valgrind /usr/bin/valgrind +``` + +Then run memcheck: + +```sh +cd out +ctest -T memcheck +``` + +Results will be in `out/Testing/Temporary/MemoryChecker.*.log` + ## Run Device Update Agent Run Device Update Agent by following these [instructions](./how-to-run-agent.md) diff --git a/packages/CMakeLists.txt b/packages/CMakeLists.txt index ef5406682..8cf93453b 100644 --- a/packages/CMakeLists.txt +++ b/packages/CMakeLists.txt @@ -43,8 +43,12 @@ set (CPACK_DEBIAN_PACKAGE_SECTION "admin") # If remove deliveryoptimization-agent from dependencies, preinst script must be updated accordingly. # See https://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps -set (CPACK_DEBIAN_PACKAGE_DEPENDS "deliveryoptimization-agent (>= 1.0.0), libdeliveryoptimization (>= 1.0.0), curl") -set (CPACK_DEBIAN_PACKAGE_SUGGESTS "deliveryoptimization-plugin-apt") +if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + set (CPACK_DEBIAN_PACKAGE_DEPENDS "deliveryoptimization-agent (>= 1.0.0), libdeliveryoptimization (>= 1.0.0), curl") + set (CPACK_DEBIAN_PACKAGE_SUGGESTS "deliveryoptimization-plugin-apt") +else () + set (CPACK_DEBIAN_PACKAGE_DEPENDS "curl") +endif () # Use dpkg-shlibdeps to generate better package dependency list. set (CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) diff --git a/packages/debian/postinst b/packages/debian/postinst index 1f5e38759..695cca3e1 100644 --- a/packages/debian/postinst +++ b/packages/debian/postinst @@ -37,6 +37,7 @@ adu_swupdate_handler_2_file=libmicrosoft_swupdate_2.so adu_example_component_enumerator_file=libcontoso_component_enumerator.so adu_delivery_optimization_downloader_file=libdeliveryoptimization_content_downloader.so +adu_curl_content_downloader_file=libcurl_content_downloader.so adu_delta_download_handler_file=libmicrosoft_delta_download_handler.so @@ -230,7 +231,16 @@ register_reference_extensions() { $adu_bin_path -l 2 --extension-type componentEnumerator --register-extension $adu_extensions_sources_dir/$adu_example_component_enumerator_file echo "Register content downloader extension..." - $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file + if [ -f "$adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file" ]; then + echo "Registering DO content downloader..." + $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file + elif [ -f "$adu_extensions_sources_dir/$adu_curl_content_downloader_file" ]; then + echo "Registering curl content downloader..." + $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_curl_content_downloader_file + else + echo "ERROR: No content downloader found!" + exit 1 + fi echo "Register delta download handler..." $adu_bin_path -l 2 --extension-type downloadHandler --extension-id "microsoft/delta:1" --register-extension $adu_extensions_sources_dir/$adu_delta_download_handler_file diff --git a/scripts/build.sh b/scripts/build.sh index 6cfc36071..05c0586f3 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -44,7 +44,7 @@ declare -a static_analysis_tools=() log_lib="zlog" install_prefix=/usr/local install_adu=false -work_folder=/tmp +work_folder="$root_dir/.workspace" cmake_dir_path="${work_folder}/deviceupdate-cmake" rootkeypkg_curl=false @@ -438,6 +438,13 @@ CMAKE_OPTIONS=( "-DCMAKE_INSTALL_PREFIX=$install_prefix" ) +# Disable DO on Ubuntu 24.04 and newer +if [[ $OS == "ubuntu" && $VER == "24.04" ]]; then + echo "Disabling Delivery Optimization for Ubuntu 24.04" + CMAKE_OPTIONS+=("-DADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF") + CMAKE_OPTIONS+=("-DADUC_ROOTKEY_PKG_DOWNLOAD_WITH_CURL=ON") +fi + if [[ $major_version != "" ]]; then CMAKE_OPTIONS+=("-DADUC_VERSION_MAJOR=$major_version") fi diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index fe0dc3bfe..495d0cd95 100755 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -35,7 +35,9 @@ install_packages=false install_packages_only=false # The folder where source code will be placed # for building and installing from source. -DEFAULT_WORKFOLDER=/tmp +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +repo_root="$(cd "$script_dir/.." > /dev/null 2>&1 && pwd)" +DEFAULT_WORKFOLDER="$repo_root/.workspace" work_folder=$DEFAULT_WORKFOLDER keep_source_code=false use_ssh=false @@ -65,7 +67,6 @@ install_cmake_version="$supported_cmake_version" cmake_force_source=false cmake_prefix="$work_folder" cmake_installer_dir="" -cmake_dir_symlink="/tmp/deviceupdate-cmake" cmake_bin="cmake" install_shellcheck=false @@ -79,6 +80,9 @@ default_do_ref=develop install_do=false do_ref=$default_do_ref +# CMake symlink location +cmake_dir_symlink="$repo_root/.workspace/deviceupdate-cmake" + # catch2 build # # used for dependencies like catch2 that will find system default @@ -140,7 +144,7 @@ print_help() { echo "--install-packages-only Indicates that only packages should be installed and that dependencies should not be installed from source." echo "" echo "-f, --work-folder Specifies the folder where source code will be cloned or downloaded." - echo " Default is /tmp." + echo " Default is [repo-root]/.workspace/." echo "-k, --keep-source-code Indicates that source code should not be deleted after install from work_folder." echo "" echo "--use-ssh Use ssh URLs to clone instead of https URLs." @@ -190,6 +194,11 @@ do_install_aduc_packages() { $SUDO apt-get install --yes gcc-12 g++-12 || return catch2_cc=/usr/bin/gcc-12 catch2_cxx=/usr/bin/g++-12 + elif [[ $OS == "Ubuntu" && $VER == "24.04" ]]; then + # Ubuntu 24.04 and newer have a recent enough default gcc, so we don't need to install a specific version + echo "Using system default gcc for Ubuntu 24.04+" + catch2_cc=/usr/bin/gcc + catch2_cxx=/usr/bin/g++ else $SUDO apt-get install --yes gcc-8 g++-8 || return catch2_cc=/usr/bin/gcc-8 @@ -438,6 +447,13 @@ do_install_do_release_tarball() { do_install_do() { echo "Installing DO ..." + + # Skip DO installation on Ubuntu 24.04 and newer + if [[ $OS == "Ubuntu" && $VER == "24.04" ]]; then + echo "Skipping DO installation on Ubuntu 24.04 (not supported)" + return 0 + fi + local do_dir=$work_folder/do if [[ -d $do_dir ]]; then $SUDO rm -rf $do_dir || return @@ -517,6 +533,25 @@ do_install_azure_storage_sdk() { git checkout tags/$azure_storage_sdk_tag_ref + # Apply patch to fix missing cstdint include for GCC 12+ (Ubuntu 24.04, Debian 12) + # Check GCC version and apply patch only if GCC >= 12 + local gcc_version + gcc_version=$(gcc -dumpversion | cut -d. -f1) + + if [[ $gcc_version -ge 12 ]]; then + local patch_file="$script_dir/patches/azure-storage-sdk-base64-cstdint.patch" + if [[ -f $patch_file ]]; then + echo "Detected GCC $gcc_version (>= 12), applying patch to fix base64.cpp compilation issue..." + git apply "$patch_file" || { + warn "Failed to apply patch, build may fail on GCC $gcc_version" + } + else + warn "Patch file not found at $patch_file, build may fail on GCC $gcc_version" + fi + else + echo "GCC $gcc_version detected, patch not needed (only required for GCC >= 12)" + fi + local azure_storage_sdk_cmake_options="" if [[ $keep_source_code == "true" ]]; then @@ -865,12 +900,13 @@ while [[ $1 != "" ]]; do shift done -# setup workfolder if different from default -if [[ $work_folder != "$DEFAULT_WORKFOLDER" ]]; then +# Always setup workfolder with proper ownership, especially for .workspace in repo +if [[ ! -d $work_folder ]]; then mkdir -pv "$work_folder" || $ret - $SUDO chown "$(id -un)":"$(id -gn)" "$work_folder" || $ret - chmod ug+rwx,o= "$work_folder" || $ret fi +# Ensure the work folder has the correct owner (the user running the script, not root) +$SUDO chown "$(id -un)":"$(id -gn)" "$work_folder" || $ret +$SUDO chmod ug+rwx,o= "$work_folder" || $ret if [[ -d $du_test_data_dir_path ]]; then $SUDO rm -r $du_test_data_dir_path diff --git a/scripts/patches/azure-storage-sdk-base64-cstdint.patch b/scripts/patches/azure-storage-sdk-base64-cstdint.patch new file mode 100644 index 000000000..b5ecd4b3e --- /dev/null +++ b/scripts/patches/azure-storage-sdk-base64-cstdint.patch @@ -0,0 +1,30 @@ +--- a/sdk/core/azure-core/inc/azure/core/base64.hpp ++++ b/sdk/core/azure-core/inc/azure/core/base64.hpp +@@ -10,6 +10,7 @@ + + #pragma once + ++#include + #include + #include + #include +--- a/sdk/core/azure-core/src/base64.cpp ++++ b/sdk/core/azure-core/src/base64.cpp +@@ -3,6 +3,7 @@ + + #include "azure/core/base64.hpp" + #include "azure/core/platform.hpp" ++#include + + #include + #include +--- a/sdk/core/azure-core/inc/azure/core/uuid.hpp ++++ b/sdk/core/azure-core/inc/azure/core/uuid.hpp +@@ -10,6 +10,7 @@ + + #include "azure/core/platform.hpp" + ++#include + #include + #include + diff --git a/src/extensions/content_downloaders/CMakeLists.txt b/src/extensions/content_downloaders/CMakeLists.txt index d9978f00d..fdbb226d4 100644 --- a/src/extensions/content_downloaders/CMakeLists.txt +++ b/src/extensions/content_downloaders/CMakeLists.txt @@ -1,4 +1,6 @@ cmake_minimum_required (VERSION 3.5) add_subdirectory (curl_downloader) -add_subdirectory (deliveryoptimization_downloader) +if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + add_subdirectory (deliveryoptimization_downloader) +endif () From 1b09c6da9acea980803a2869e0100771a85da0be Mon Sep 17 00:00:00 2001 From: Nox-MSFT <55153324+Nox-MSFT@users.noreply.github.com> Date: Fri, 12 Dec 2025 13:08:33 -0800 Subject: [PATCH 2/4] Enable Ubuntu 24.04 build in GitHub workflow --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fa6eb531a..45c0d64da 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04"] + os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04", "ubuntu:22.04"] arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} container: From 8fb4cd19af287667f20f2969955462141ecf470c Mon Sep 17 00:00:00 2001 From: Nox-MSFT <55153324+Nox-MSFT@users.noreply.github.com> Date: Fri, 12 Dec 2025 13:23:27 -0800 Subject: [PATCH 3/4] Update Ubuntu version in Docker build matrix --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 45c0d64da..8f0cd7e4d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04", "ubuntu:22.04"] + os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04", "ubuntu:24.04"] arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} container: From de61c09ee2a9bf6501ee078276df50f8ee7037ff Mon Sep 17 00:00:00 2001 From: Nox-MSFT Date: Mon, 15 Dec 2025 16:49:23 +0000 Subject: [PATCH 4/4] fix(install-deps): Ensure 'file' utility is available for Ubuntu 24.04+ to support CPack packaging --- scripts/install-deps.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index 26a784108..711623829 100755 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -97,7 +97,7 @@ catch2_cc="" catch2_cxx="" # Dependencies packages -aduc_packages=('git' 'make' 'build-essential' 'cmake' 'ninja-build' 'libcurl4-openssl-dev' 'libssl-dev' 'uuid-dev' 'lsb-release' 'curl' 'wget' 'pkg-config' 'libxml2-dev') +aduc_packages=('git' 'make' 'build-essential' 'cmake' 'ninja-build' 'libcurl4-openssl-dev' 'libssl-dev' 'uuid-dev' 'lsb-release' 'curl' 'wget' 'pkg-config' 'libxml2-dev' 'file') static_analysis_packages=('clang' 'clang-tidy' 'cppcheck') compiler_packages=('gcc' 'g++') @@ -230,9 +230,20 @@ do_install_aduc_packages() { $SUDO apt-get install --yes "${aduc_packages[@]}" || return + # For Ubuntu 24.04+, ensure the 'file' utility is installed (may be needed by CPack) + OS=$(lsb_release --short --id) + if [[ $OS == "Ubuntu" ]]; then + # Parse version to check if 24.04 or later + VER_MAJOR=$(echo $VER | cut -d. -f1) + VER_MINOR=$(echo $VER | cut -d. -f2) + if [[ $VER_MAJOR -gt 24 ]] || [[ $VER_MAJOR -eq 24 && $VER_MINOR -ge 4 ]]; then + echo "Ensuring 'file' utility is available for Ubuntu 24.04+" + $SUDO apt-get install --yes file || echo "Warning: Could not install 'file' package" + fi + fi + # The latest version of gcc available on Debian is gcc-6. We install that version if we are # building for Debian, otherwise we install gcc-8 for Ubuntu. - OS=$(lsb_release --short --id) if [[ $OS == "Debian" && $VER == "9" ]]; then $SUDO apt-get install --yes gcc-6 g++-6 || return catch2_cc=/usr/bin/gcc-6