diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fa6eb531a..8f0cd7e4d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,7 +7,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04"] + os: ["debian:12", "debian:11", "ubuntu:22.04", "ubuntu:20.04", "ubuntu:24.04"] arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} container: diff --git a/.gitignore b/.gitignore index b1aaf78af..6cb6f083a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ ### VisualStudio 2017 .vs/ +### Temporary workspace files +.workspace/ + ### GDB debugger .gdbinit .gdb_history diff --git a/CMakeLists.txt b/CMakeLists.txt index 29cecf0fe..7b0d85774 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -517,6 +517,8 @@ option (ADUC_ENABLE_E2E_TESTING "Enable e2e test pipeline settings" OFF) option (ADUC_ENABLE_SRVC_E2E_TESTING "Enable service side test agent settings" OFF) option (ADUC_ROOTKEY_PKG_DOWNLOAD_WITH_CURL "Use Curl to download the Rootkey Package instead of DeliveryOptimization agent" OFF) +option (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION + "Build with Delivery Optimization support" ON) option (ADUC_ENABLE_CONSOLE_LOG "Enable console logging" ON) option (ADUC_ENABLE_FILE_LOG "Enable file logging" ON) diff --git a/README.md b/README.md index f0797e018..583e3cbcc 100644 --- a/README.md +++ b/README.md @@ -1,113 +1,61 @@ -# What is Device Update for IoT Hub? +# Device Update for IoT Hub -Device Update for IoT Hub is a service that enables you to deploy over-the-air updates (OTA) for your IoT devices. +Device Update for IoT Hub is an end-to-end platform for deploying over-the-air updates (OTA) to your IoT devices—from tiny sensors to gateway-level devices. -Device Update for IoT Hub is an end-to-end platform that customers can use to publish, distribute, and manage over-the-air updates for everything from tiny sensors to gateway-level devices. +## Key Features -Device Update for IoT Hub also provides controls on how to manage the deployment updates so you are always in control of when and how devices are updated. Device Update for IoT Hub also provides reporting capabilities so you are always up to date on the state of your devices via integration with IoT Hub. +* **Integrated Management** - Update management UX integrated with Azure IoT Hub +* **Controlled Rollout** - Gradual update deployment through device grouping and scheduling +* **Flexible APIs** - Programmatic APIs for automation and custom portal experiences +* **Fleet Visibility** - At-a-glance compliance and status views across heterogeneous device fleets +* **Resilient Updates** - Support for A/B updates with seamless rollback +* **Access Control** - Subscription and role-based access controls via Azure portal +* **Offline Support** - On-premise content cache and Nested Edge support for disconnected devices +* **Comprehensive Reporting** - Detailed update management and reporting tools -Device Update for IoT Hub features provide a powerful and flexible experience, including: +## Supported Platforms -* Update management UX integrated with Azure IoT Hub -* Gradual update rollout through device grouping and update scheduling controls -* Programmatic APIs to enable automation and custom portal experiences -* At-a-glance update compliance and status views across heterogenous device fleets -* Support for resilient device updates (A/B) to deliver seamless rollback -* Subscription and role-based access controls available through the Azure.com portal -* On-premise content cache and Nested Edge support to enable updating cloud disconnected devices -* Detailed update management and reporting tools - -## Reference agent - -| Build | Status | -|------------------- |--------| -| Ubuntu 22.04 AMD64 | [![Ubuntu 22.04 Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main)| - -## Getting started - -* [Device Update for IoT Hub](https://aka.ms/iot-hub-device-update-docs) -* [Getting Started with Device Update Agent](./docs/agent-reference) -* More details on building the agent here: [How to build agent code](./docs/agent-reference/how-to-build-agent-code.md) +| Platform | Status | Notes | +|------------------- |--------|-------| +| Ubuntu 20.04 AMD64 | ✓ Supported | With Delivery Optimization | +| Ubuntu 22.04 AMD64 | [![Ubuntu 22.04 Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main)| Primary development platform | +| Ubuntu 24.04 AMD64 | ✓ Supported | curl downloader only (DO not available) | +| Debian 11 (Bullseye) AMD64 | ✓ Supported | With Delivery Optimization | +| Debian 12 (Bookworm) AMD64 | ✓ Supported | With Delivery Optimization | ## Quick Start -### Build and Install +For users familiar with the build process: ```sh ./scripts/install-deps.sh -a ./scripts/build.sh -c -u --build-packages -cd out -sudo cmake --build . --target install +sudo apt install ./out/deviceupdate-agent_*.deb ``` -### Incremental Build +> **Note**: For detailed build instructions, troubleshooting, and advanced options, see [How to Build the Agent](./docs/agent-reference/how-to-build-agent-code.md) -```sh -cd out -ninja -``` +## Documentation -### Run Tests +* **[Device Update for IoT Hub Documentation](https://aka.ms/iot-hub-device-update-docs)** - Official service documentation +* **[Getting Started with the Agent](./docs/agent-reference)** - Agent overview and reference +* **[Building the Agent](./docs/agent-reference/how-to-build-agent-code.md)** - Detailed build instructions +* **[Running the Agent](./docs/agent-reference/how-to-run-agent.md)** - Deployment and configuration +* **[Troubleshooting Guide](./docs/how-to-troubleshoot-guide.md)** - Common issues and solutions -```sh -cd out -ctest -``` +## Contributing -or, alternatively: +This project welcomes contributions and suggestions. See [CONTRIBUTING.md](./CONTRIBUTING.md) for details. -```sh -ninja test -``` +## Security -### Run tests under valgrind memcheck +See [SECURITY.md](./SECURITY.md) for information on reporting security issues. -To install Valgrind: -```sh -# Install from apt (Ubuntu 22.04 or newer recommended) -sudo apt-get install valgrind +## License -# Or use install-deps.sh -./scripts/install-deps.sh --install-valgrind apt +This project is licensed under the MIT License - see [LICENSE](./LICENSE) for details. -# Build from source (version 3.23.0) -./scripts/install-deps.sh --install-valgrind source -``` +## Support -Ensure /usr/bin/valgrind is a valid symlink -e.g. `sudo ln -s /opt/valgrind.3.19.0/bin/valgrind /usr/bin/valgrind` +For support options, see [SUPPORT.md](./SUPPORT.md). -```sh -cd out -ctest -T memcheck -``` - -Results will be in `out/Testing/Temporary/MemoryChecker.*.log` - -#### Run specific tests under valgrind - -```sh -cd out -# Run a specific test -ctest -R -T memcheck - -# Run tests matching a pattern -ctest -R ".*device_properties.*" -T memcheck - -# Run with verbose output -ctest -R -T memcheck -V -``` - -#### Run individual test binary directly with valgrind - -```sh -# Run with full leak check -valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes ./path/to/test_binary - -# With verbose output and log file -valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes \ - --verbose --log-file=valgrind-output.log ./path/to/test_binary - -# With child process tracking -valgrind --leak-check=full --trace-children=yes ./path/to/test_binary -``` diff --git a/cmake/agentRules.cmake b/cmake/agentRules.cmake index 490f7e83a..33e07f85f 100644 --- a/cmake/agentRules.cmake +++ b/cmake/agentRules.cmake @@ -64,21 +64,23 @@ function (target_link_digital_twin_client target scope) endfunction (target_link_digital_twin_client) function (target_link_dosdk target scope) - if (NOT WIN32) - find_package (deliveryoptimization_sdk CONFIG REQUIRED) - target_link_libraries (${target} ${scope} Microsoft::deliveryoptimization) - else () - if (CMAKE_GENERATOR_PLATFORM STREQUAL "") - set (DO_CLIENT_INSTALL_FOLDER "${VCPKG_INSTALLED_DIR}/do-client") + if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + if (NOT WIN32) + find_package (deliveryoptimization_sdk CONFIG REQUIRED) + target_link_libraries (${target} ${scope} Microsoft::deliveryoptimization) else () - # Different install folder if cross-compiling - see build.ps1 - set (DO_CLIENT_INSTALL_FOLDER - "${VCPKG_INSTALLED_DIR}/do-client-${CMAKE_GENERATOR_PLATFORM}") - endif () + if (CMAKE_GENERATOR_PLATFORM STREQUAL "") + set (DO_CLIENT_INSTALL_FOLDER "${VCPKG_INSTALLED_DIR}/do-client") + else () + # Different install folder if cross-compiling - see build.ps1 + set (DO_CLIENT_INSTALL_FOLDER + "${VCPKG_INSTALLED_DIR}/do-client-${CMAKE_GENERATOR_PLATFORM}") + endif () - target_include_directories (${target} ${scope} ${DO_CLIENT_INSTALL_FOLDER}/sdk-cpp/include) - target_link_directories (${target} PUBLIC - ${DO_CLIENT_INSTALL_FOLDER}/cmake/sdk-cpp/${CMAKE_BUILD_TYPE}) - target_link_libraries (${target} ${scope} deliveryoptimization) + target_include_directories (${target} ${scope} ${DO_CLIENT_INSTALL_FOLDER}/sdk-cpp/include) + target_link_directories (${target} PUBLIC + ${DO_CLIENT_INSTALL_FOLDER}/cmake/sdk-cpp/${CMAKE_BUILD_TYPE}) + target_link_libraries (${target} ${scope} deliveryoptimization) + endif () endif () endfunction (target_link_dosdk) diff --git a/docs/agent-reference/how-to-build-agent-code.md b/docs/agent-reference/how-to-build-agent-code.md index a534275e8..d2e87a550 100644 --- a/docs/agent-reference/how-to-build-agent-code.md +++ b/docs/agent-reference/how-to-build-agent-code.md @@ -1,30 +1,264 @@ # How To Build the Device Update Agent -Take a look at [dependencies](how-to-build-agent-code.md#dependencies-of-device-update-agent) before you get started. You can build the Device Update agent as a standlone solution or integrate it in your existing application or solution. +This guide provides detailed instructions for building the Device Update for IoT Hub agent from source. -- [Dependencies](how-to-build-agent-code.md#dependencies-of-device-update-agent) -- [As a standalone solution](how-to-build-agent-code.md#as-a-standalone-solution) +## Table of Contents -## Dependencies of Device Update Agent +- [Supported Platforms](#supported-platforms) +- [Build Dependencies](#build-dependencies) +- [Runtime Dependencies](#runtime-dependencies) +- [Quick Start](#quick-start) +- [Building the Agent](#building-the-device-update-agent-for-linux) +- [Building Dependencies from Source](#building-dependencies-from-source) +- [Testing](#build-and-run-the-unit-tests) +- [Packaging](#build-the-debian-package) +- [Installation](#install-the-device-update-agent) +- [Platform-Specific Build Notes](#platform-specific-build-notes) +- [Advanced Options](#build-options-for-mqtt-and-mqtt-over-websockets-iothub-transport-providers) +- [Troubleshooting](#troubleshooting) -### Required Dependencies +## Supported Platforms -- Azure IoT C SDK -- Delivery Optimization SDK -- Azure Blob Storage File Upload Utility -- IotHub Device Update Delta +The Device Update agent has been tested and verified on the following platforms: -### Azure IoT C SDK +| OS Distribution | Version | Architecture | Status | Build Status | Notes | +|----------------|---------|--------------|--------|--------------|-------| +| Ubuntu | 20.04 LTS | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Ubuntu | 22.04 LTS | AMD64 | ✓ Fully Supported | [![Build Status](https://dev.azure.com/azure-device-update/adu-linux-client/_apis/build/status/Azure.iot-hub-device-update?branchName=main)](https://dev.azure.com/azure-device-update/adu-linux-client/_build/latest?definitionId=27&branchName=main) | Primary development platform | +| Ubuntu | 24.04 LTS | AMD64 | ✓ Fully Supported | - | curl downloader only, [see notes](#ubuntu-2404-lts-specifics) | +| Ubuntu | 18.04 LTS | AMD64 | ✗ Not Supported | - | End of support | +| Debian | 11 (Bullseye) | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Debian | 12 (Bookworm) | AMD64 | ✓ Supported | - | With Delivery Optimization | +| Debian | 10 (Buster) | AMD64 | ✗ Not Supported | - | End of support | -Use the [Azure IoT C -SDK](https://github.com/Azure/azure-iot-sdk-c) -to connect to IoT Hub and call Azure IoT Plug and Play APIs. +## Build Dependencies -### Delivery Optimization +The following table lists all build-time dependencies with their Last Known Good (LKG) versions and platform availability. -The [Delivery Optimization -SDK](https://github.com/microsoft/do-client) -provides a robust way for the client to download an update. +**Note:** Dependencies marked with an asterisk (*) require special attention. Click the link for details about patches, limitations, or special build instructions. + +### Core Build Tools + +| Tool | Minimum Version | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | Source | +|------|----------------|--------------|--------------|--------------|-----------|-----------|--------| +| GCC | 10.0 | gcc-10 | gcc-10/11 | gcc-13 | gcc-10 | gcc-12 | System package | +| CMake | 3.10 | 3.16.3 | 3.22.1 | 3.28.3 | 3.18.4 | 3.25.1 | System package or [built from source](#installing-dependencies) | +| Ninja | 1.10+ | 1.10.0 | 1.10.2 | 1.11.1 | 1.10.1 | 1.11.1 | System package | +| Git | 2.0+ | 2.25.1 | 2.34.1 | 2.43.0 | 2.30.2 | 2.39.2 | System package | +| pkg-config | - | ✓ | ✓ | ✓ | ✓ | ✓ | System package | + +### Core Dependencies Built from Source + +| Dependency | LKG Version/Tag | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | Notes | +|------------|----------------|--------------|--------------|--------------|-----------|-----------|-------| +| Azure IoT C SDK | `LTS_08_2023` | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-azure-iot-c-sdk) | +| Azure Storage SDK for C++ | `azure-core_1.6.0` | ✓ | ✓ | ✓* | ✓ | ✓* | [*Requires GCC 12+ patch](#building-azure-storage-sdk-for-c) | +| Delivery Optimization SDK | `main` (latest) | ✓ | ✓ | ✗ | ✓ | ✓ | [Not available on Ubuntu 24.04](#building-delivery-optimization-sdk) | +| Catch2 | `v3.8.0` | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-catch2) (unit tests only) | +| Parson | Latest | ✓ | ✓ | ✓ | ✓ | ✓ | [Build instructions](#building-parson) | +| Microsoft Delta Download Handler | Submodule | ✓ | ✓ | ✓ | ✓ | ✓ | Built with agent | + +### System Package Dependencies + +| Package | Purpose | Ubuntu 20.04 | Ubuntu 22.04 | Ubuntu 24.04 | Debian 11 | Debian 12 | +|---------|---------|--------------|--------------|--------------|-----------|-----------| +| libcurl4-openssl-dev | HTTP/HTTPS client | ✓ Required | ✓ Required | ✓ Required | ✓ Required | ✓ Required | +| libssl-dev | TLS/Crypto | ✓ (1.1.1) | ✓ (3.0) | ✓ (3.0) | ✓ (1.1.1) | ✓ (3.0) | +| uuid-dev | UUID generation | ✓ | ✓ | ✓ | ✓ | ✓ | +| zlib1g-dev | Compression | ✓ | ✓ | ✓ | ✓ | ✓ | + +### Optional Development Tools + +| Tool | Purpose | Installation | +|------|---------|--------------| +| clang-format | Code formatting | `sudo apt install clang-format` | +| cmake-format | CMake formatting | `sudo apt install python3-pip && sudo pip3 install cmake-format` | +| valgrind | Memory testing | `sudo apt install valgrind` (3.19+ recommended) | +| shellcheck | Shell script linting | Via install-deps.sh | + +## Runtime Dependencies + +The following dependencies are required to run the Device Update agent on deployed devices. + +### Core Runtime Requirements + +| Dependency | Minimum Version | Package Name (Ubuntu) | Purpose | +|------------|----------------|----------------------|---------| +| systemd | 237+ | systemd | Daemon management | +| libssl | 1.1+ | libssl3 (22.04+), libssl1.1 (20.04) | TLS/crypto operations | +| libcurl | 7.58+ | libcurl4 | HTTP client | +| curl (binary) | 7.58+ | curl | Content download (24.04), rootkey download | + +### Platform-Specific Runtime Dependencies + +**Ubuntu 20.04, 22.04, Debian 11, Debian 12:** +- `deliveryoptimization-agent` >= 1.0.0 (primary downloader) +- `libdeliveryoptimization` >= 1.0.0 +- `curl` (fallback downloader) + +**Ubuntu 24.04:** +- `curl` (primary downloader) +- Note: Delivery Optimization not available + +## Quick Start + +For most users, building the agent is straightforward: + +```sh +# Install all dependencies +./scripts/install-deps.sh -a + +# Build the agent with unit tests and create Debian package +./scripts/build.sh -c -u --build-packages + +# Install the package +sudo apt install ./out/deviceupdate-agent_*.deb +``` + +For incremental builds after the initial build: + +```sh +cd out +ninja +``` + +To run tests: + +```sh +cd out +ctest +# or +ninja test +``` + +## Building Dependencies from Source + +The `install-deps.sh` script automates building dependencies, but this section documents the process, required versions, and any patches needed for each dependency. + +### Building Azure IoT C SDK + +**Repository:** [https://github.com/Azure/azure-iot-sdk-c](https://github.com/Azure/azure-iot-sdk-c) +**LKG Version:** `LTS_08_2023` branch +**Build Location:** `.workspace/azure-iot-sdk-c` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) + +The Azure IoT C SDK provides the connectivity layer to Azure IoT Hub and implements the Azure IoT Plug and Play APIs. + +**Patches Required:** None + +**Build Options:** +- MQTT transport (`use_mqtt=ON`) +- MQTT over WebSockets (`use_wsio=ON`) +- Both are built by default + +**Known Issues:** None + +--- + +### Building Azure Storage SDK for C++ + +**Repository:** [https://github.com/Azure/azure-sdk-for-cpp](https://github.com/Azure/azure-sdk-for-cpp) +**LKG Version:** `azure-core_1.6.0` tag +**Build Location:** `.workspace/azure_storage_sdk_dir` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04*, Debian 11, 12*) + +The Azure Storage SDK for C++ provides blob storage operations for file uploads. + +#### GCC 12+ Compatibility Patch + +**Affected Platforms:** Platforms using GCC 12 or later +- Ubuntu 24.04 (GCC 13) +- Debian 12 (GCC 12) +- Any custom build environment with GCC 12+ + +**Issue:** GCC 12 and later removed implicit standard library includes that previous versions provided. The Azure Storage SDK compilation fails with errors like: +- `error: 'uint8_t' does not name a type` +- `error: 'uint16_t' was not declared in this scope` + +**Patch File:** `scripts/patches/azure-storage-sdk-base64-cstdint.patch` + +**Affected Files:** +- `sdk/core/azure-core/inc/azure/core/internal/cryptography/base64.hpp` +- `sdk/core/azure-core/src/cryptography/base64.cpp` +- `sdk/core/azure-core/inc/azure/core/uuid.hpp` + +**Solution:** The patch adds explicit `#include ` directives to the affected files. + +**Automatic Application:** +The `install-deps.sh` script automatically detects the GCC version during the Azure Storage SDK build process. If GCC version is 12 or later, the patch is applied automatically. Platforms with GCC 10 or 11 do not require or receive this patch. + +**Manual Application:** +```sh +cd .workspace/azure_storage_sdk_dir +git apply ../../scripts/patches/azure-storage-sdk-base64-cstdint.patch +``` + +**Verification:** +After patching, the SDK compiles successfully with GCC 13 without warnings or errors. + +--- + +### Building Delivery Optimization SDK + +**Repository:** [https://github.com/microsoft/do-client](https://github.com/microsoft/do-client) +**LKG Version:** `main` branch (latest) +**Build Location:** `.workspace/do` +**Platforms:** Ubuntu 20.04, 22.04, Debian 11, 12 + +The Delivery Optimization SDK provides robust, peer-to-peer content distribution for update downloads. + +**Availability:** +- ✓ **Ubuntu 20.04:** Fully supported +- ✓ **Ubuntu 22.04:** Fully supported +- ✓ **Debian 11:** Fully supported +- ✓ **Debian 12:** Fully supported +- ✗ **Ubuntu 24.04:** **NOT AVAILABLE** - Package not maintained for this version + +**Alternative on Ubuntu 24.04:** +The agent automatically uses the curl content downloader (`libcurl_content_downloader.so`) as a replacement. The build system detects Ubuntu 24.04 and configures accordingly: +- Sets `ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF` +- Registers curl downloader as primary content downloader +- Package dependencies exclude DO packages + +**Build Command (Ubuntu 20.04/22.04, Debian 11/12):** +```sh +./scripts/install-deps.sh --install-do +``` + +**Patches Required:** None + +**Known Limitations on Ubuntu 24.04:** +- No peer-to-peer download optimization +- All content downloads via direct HTTP/HTTPS using curl +- Functionally equivalent but may use more bandwidth in fleet scenarios + +--- + +### Building Catch2 + +**Repository:** [https://github.com/catchorg/Catch2](https://github.com/catchorg/Catch2) +**LKG Version:** `v3.8.0` tag +**Build Location:** `.workspace/catch2` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) +**Purpose:** Unit testing framework + +**Patches Required:** None + +**Note:** Only required for building and running unit tests. Not needed for production builds. + +--- + +### Building Parson + +**Repository:** [https://github.com/kgabis/parson](https://github.com/kgabis/parson) +**LKG Version:** Latest from master +**Build Location:** `.workspace/parson` +**Platforms:** All supported (Ubuntu 20.04, 22.04, 24.04, Debian 11, 12) +**Purpose:** Lightweight JSON parser + +**Patches Required:** None + +**Known Issues:** None ## Building the Device Update Agent for Linux @@ -362,6 +596,165 @@ After building the Debian package using `build.sh --build-packages`, do: sudo apt install ./out/{PKG_NAME}.deb ``` +## Platform-Specific Build Notes + +This section contains important platform-specific information, limitations, and workarounds. + +### Ubuntu 20.04 LTS Specifics + +**GCC Version:** 10.3.0 (default) +**CMake Version:** 3.16.3 (system), can use newer from install-deps.sh + +**Delivery Optimization:** +- Fully supported +- Installed as system packages + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Ubuntu 22.04 LTS Specifics + +**GCC Version:** 11.x (default) or 10.x +**CMake Version:** 3.22.1 (system) + +**Delivery Optimization:** +- Fully supported +- Primary development and CI/CD platform + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Ubuntu 24.04 LTS Specifics + +**GCC Version:** 13.3.0 (system default) +**CMake Version:** 3.28.3 (system) + +**Key Changes from Previous Versions:** +- GCC 8/9/10 installation automatically skipped +- Uses system default GCC 13 compiler +- Build system auto-detects Ubuntu 24.04 + +**Delivery Optimization:** +- **NOT AVAILABLE** - Package not maintained for Ubuntu 24.04 +- Build automatically disables DO via `ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF` +- `build.sh` detects Ubuntu 24.04 and configures accordingly + +**Content Downloader:** +- Primary: `libcurl_content_downloader.so` +- Automatically registered during package installation +- No Delivery Optimization fallback available + +**Compiler Compatibility:** +- GCC 13 requires Azure Storage SDK patch (applied automatically by install-deps.sh when GCC >= 12) +- See [Building Azure Storage SDK for C++](#building-azure-storage-sdk-for-c) for details + +**Package Dependencies:** +- Debian package depends on `curl` only +- No `deliveryoptimization-agent` or `libdeliveryoptimization` dependencies + +**Known Limitations:** +- No peer-to-peer download optimization +- All downloads are direct HTTP/HTTPS via curl + +--- + +### Debian 11 (Bullseye) Specifics + +**GCC Version:** 10.2.1 (default) +**CMake Version:** 3.18.4 (system) + +**Delivery Optimization:** +- Fully supported +- Installed via install-deps.sh + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Known Issues:** None + +--- + +### Debian 12 (Bookworm) Specifics + +**GCC Version:** 12.2.0 (default) +**CMake Version:** 3.25.1 (system) + +**Delivery Optimization:** +- Fully supported +- Installed via install-deps.sh + +**Content Downloader:** +- Primary: Delivery Optimization +- Fallback: curl + +**Compiler Compatibility:** +- GCC 12 requires Azure Storage SDK patch (applied automatically by install-deps.sh when GCC >= 12) +- See [Building Azure Storage SDK for C++](#building-azure-storage-sdk-for-c) for details + +**Known Issues:** None + +--- + +## Troubleshooting + +### Build Failures + +#### Azure Storage SDK Compilation Errors (GCC 12+) + +If you encounter errors related to `uint8_t` or missing `` includes when building the Azure Storage SDK, ensure you're using the latest version of `install-deps.sh` which includes automatic GCC version detection and patch application for GCC 12+. This affects: +- Ubuntu 24.04 (GCC 13) +- Debian 12 (GCC 12) +- Any custom environment using GCC 12 or later + +The patch is only applied when the build system detects GCC version 12 or higher. + +#### Permission Issues + +If you encounter permission errors with files in `.workspace/`, the build scripts will automatically fix ownership. If issues persist, manually correct ownership: + +```sh +sudo chown -R $(id -un):$(id -gn) .workspace/ +``` + +#### Symlink Issues + +If cmake or shellcheck symlinks are broken, re-run the install script: + +```sh +./scripts/install-deps.sh --install-packages +``` + +### Valgrind Setup + +To run tests under valgrind memcheck, ensure valgrind 3.19.0 or later is installed: + +```sh +# Verify valgrind is accessible +which valgrind +# If not found, create symlink to your valgrind installation +sudo ln -s /opt/valgrind.3.19.0/bin/valgrind /usr/bin/valgrind +``` + +Then run memcheck: + +```sh +cd out +ctest -T memcheck +``` + +Results will be in `out/Testing/Temporary/MemoryChecker.*.log` + ## Run Device Update Agent Run Device Update Agent by following these [instructions](./how-to-run-agent.md) diff --git a/packages/CMakeLists.txt b/packages/CMakeLists.txt index ef5406682..8cf93453b 100644 --- a/packages/CMakeLists.txt +++ b/packages/CMakeLists.txt @@ -43,8 +43,12 @@ set (CPACK_DEBIAN_PACKAGE_SECTION "admin") # If remove deliveryoptimization-agent from dependencies, preinst script must be updated accordingly. # See https://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps -set (CPACK_DEBIAN_PACKAGE_DEPENDS "deliveryoptimization-agent (>= 1.0.0), libdeliveryoptimization (>= 1.0.0), curl") -set (CPACK_DEBIAN_PACKAGE_SUGGESTS "deliveryoptimization-plugin-apt") +if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + set (CPACK_DEBIAN_PACKAGE_DEPENDS "deliveryoptimization-agent (>= 1.0.0), libdeliveryoptimization (>= 1.0.0), curl") + set (CPACK_DEBIAN_PACKAGE_SUGGESTS "deliveryoptimization-plugin-apt") +else () + set (CPACK_DEBIAN_PACKAGE_DEPENDS "curl") +endif () # Use dpkg-shlibdeps to generate better package dependency list. set (CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) diff --git a/packages/debian/postinst b/packages/debian/postinst index 1f5e38759..695cca3e1 100644 --- a/packages/debian/postinst +++ b/packages/debian/postinst @@ -37,6 +37,7 @@ adu_swupdate_handler_2_file=libmicrosoft_swupdate_2.so adu_example_component_enumerator_file=libcontoso_component_enumerator.so adu_delivery_optimization_downloader_file=libdeliveryoptimization_content_downloader.so +adu_curl_content_downloader_file=libcurl_content_downloader.so adu_delta_download_handler_file=libmicrosoft_delta_download_handler.so @@ -230,7 +231,16 @@ register_reference_extensions() { $adu_bin_path -l 2 --extension-type componentEnumerator --register-extension $adu_extensions_sources_dir/$adu_example_component_enumerator_file echo "Register content downloader extension..." - $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file + if [ -f "$adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file" ]; then + echo "Registering DO content downloader..." + $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_delivery_optimization_downloader_file + elif [ -f "$adu_extensions_sources_dir/$adu_curl_content_downloader_file" ]; then + echo "Registering curl content downloader..." + $adu_bin_path -l 2 --extension-type contentDownloader --register-extension $adu_extensions_sources_dir/$adu_curl_content_downloader_file + else + echo "ERROR: No content downloader found!" + exit 1 + fi echo "Register delta download handler..." $adu_bin_path -l 2 --extension-type downloadHandler --extension-id "microsoft/delta:1" --register-extension $adu_extensions_sources_dir/$adu_delta_download_handler_file diff --git a/scripts/build.sh b/scripts/build.sh index 5bc563a9e..ab91c44aa 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -46,7 +46,7 @@ declare -a static_analysis_tools=() log_lib="zlog" install_prefix=/usr/local install_adu=false -work_folder=/tmp +work_folder="$root_dir/.workspace" cmake_dir_path="${work_folder}/deviceupdate-cmake" rootkeypkg_curl=false @@ -450,6 +450,13 @@ CMAKE_OPTIONS=( "-DCMAKE_INSTALL_PREFIX=$install_prefix" ) +# Disable DO on Ubuntu 24.04 and newer +if [[ $OS == "ubuntu" && $VER == "24.04" ]]; then + echo "Disabling Delivery Optimization for Ubuntu 24.04" + CMAKE_OPTIONS+=("-DADUC_BUILD_WITH_DELIVERY_OPTIMIZATION=OFF") + CMAKE_OPTIONS+=("-DADUC_ROOTKEY_PKG_DOWNLOAD_WITH_CURL=ON") +fi + if [[ $major_version != "" ]]; then CMAKE_OPTIONS+=("-DADUC_VERSION_MAJOR=$major_version") fi diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index 6b88eca60..711623829 100755 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -35,7 +35,9 @@ install_packages=false install_packages_only=false # The folder where source code will be placed # for building and installing from source. -DEFAULT_WORKFOLDER=/tmp +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +repo_root="$(cd "$script_dir/.." > /dev/null 2>&1 && pwd)" +DEFAULT_WORKFOLDER="$repo_root/.workspace" work_folder=$DEFAULT_WORKFOLDER keep_source_code=false use_ssh=false @@ -65,7 +67,6 @@ install_cmake_version="$supported_cmake_version" cmake_force_source=false cmake_prefix="$work_folder" cmake_installer_dir="" -cmake_dir_symlink="/tmp/deviceupdate-cmake" cmake_bin="cmake" install_shellcheck=false @@ -84,6 +85,9 @@ default_do_ref=develop install_do=false do_ref=$default_do_ref +# CMake symlink location +cmake_dir_symlink="$repo_root/.workspace/deviceupdate-cmake" + # catch2 build # # used for dependencies like catch2 that will find system default @@ -93,7 +97,7 @@ catch2_cc="" catch2_cxx="" # Dependencies packages -aduc_packages=('git' 'make' 'build-essential' 'cmake' 'ninja-build' 'libcurl4-openssl-dev' 'libssl-dev' 'uuid-dev' 'lsb-release' 'curl' 'wget' 'pkg-config' 'libxml2-dev') +aduc_packages=('git' 'make' 'build-essential' 'cmake' 'ninja-build' 'libcurl4-openssl-dev' 'libssl-dev' 'uuid-dev' 'lsb-release' 'curl' 'wget' 'pkg-config' 'libxml2-dev' 'file') static_analysis_packages=('clang' 'clang-tidy' 'cppcheck') compiler_packages=('gcc' 'g++') @@ -149,7 +153,7 @@ print_help() { echo "--install-packages-only Indicates that only packages should be installed and that dependencies should not be installed from source." echo "" echo "-f, --work-folder Specifies the folder where source code will be cloned or downloaded." - echo " Default is /tmp." + echo " Default is [repo-root]/.workspace/." echo "-k, --keep-source-code Indicates that source code should not be deleted after install from work_folder." echo "" echo "--use-ssh Use ssh URLs to clone instead of https URLs." @@ -226,9 +230,20 @@ do_install_aduc_packages() { $SUDO apt-get install --yes "${aduc_packages[@]}" || return + # For Ubuntu 24.04+, ensure the 'file' utility is installed (may be needed by CPack) + OS=$(lsb_release --short --id) + if [[ $OS == "Ubuntu" ]]; then + # Parse version to check if 24.04 or later + VER_MAJOR=$(echo $VER | cut -d. -f1) + VER_MINOR=$(echo $VER | cut -d. -f2) + if [[ $VER_MAJOR -gt 24 ]] || [[ $VER_MAJOR -eq 24 && $VER_MINOR -ge 4 ]]; then + echo "Ensuring 'file' utility is available for Ubuntu 24.04+" + $SUDO apt-get install --yes file || echo "Warning: Could not install 'file' package" + fi + fi + # The latest version of gcc available on Debian is gcc-6. We install that version if we are # building for Debian, otherwise we install gcc-8 for Ubuntu. - OS=$(lsb_release --short --id) if [[ $OS == "Debian" && $VER == "9" ]]; then $SUDO apt-get install --yes gcc-6 g++-6 || return catch2_cc=/usr/bin/gcc-6 @@ -244,6 +259,11 @@ do_install_aduc_packages() { $SUDO apt-get install --yes gcc-12 g++-12 || return catch2_cc=/usr/bin/gcc-12 catch2_cxx=/usr/bin/g++-12 + elif [[ $OS == "Ubuntu" && $VER == "24.04" ]]; then + # Ubuntu 24.04 and newer have a recent enough default gcc, so we don't need to install a specific version + echo "Using system default gcc for Ubuntu 24.04+" + catch2_cc=/usr/bin/gcc + catch2_cxx=/usr/bin/g++ else $SUDO apt-get install --yes gcc-8 g++-8 || return catch2_cc=/usr/bin/gcc-8 @@ -492,6 +512,13 @@ do_install_do_release_tarball() { do_install_do() { echo "Installing DO ..." + + # Skip DO installation on Ubuntu 24.04 and newer + if [[ $OS == "Ubuntu" && $VER == "24.04" ]]; then + echo "Skipping DO installation on Ubuntu 24.04 (not supported)" + return 0 + fi + local do_dir=$work_folder/do if [[ -d $do_dir ]]; then $SUDO rm -rf $do_dir || return @@ -571,6 +598,25 @@ do_install_azure_storage_sdk() { git checkout tags/$azure_storage_sdk_tag_ref + # Apply patch to fix missing cstdint include for GCC 12+ (Ubuntu 24.04, Debian 12) + # Check GCC version and apply patch only if GCC >= 12 + local gcc_version + gcc_version=$(gcc -dumpversion | cut -d. -f1) + + if [[ $gcc_version -ge 12 ]]; then + local patch_file="$script_dir/patches/azure-storage-sdk-base64-cstdint.patch" + if [[ -f $patch_file ]]; then + echo "Detected GCC $gcc_version (>= 12), applying patch to fix base64.cpp compilation issue..." + git apply "$patch_file" || { + warn "Failed to apply patch, build may fail on GCC $gcc_version" + } + else + warn "Patch file not found at $patch_file, build may fail on GCC $gcc_version" + fi + else + echo "GCC $gcc_version detected, patch not needed (only required for GCC >= 12)" + fi + local azure_storage_sdk_cmake_options="" if [[ $keep_source_code == "true" ]]; then @@ -931,12 +977,13 @@ while [[ $1 != "" ]]; do shift done -# setup workfolder if different from default -if [[ $work_folder != "$DEFAULT_WORKFOLDER" ]]; then +# Always setup workfolder with proper ownership, especially for .workspace in repo +if [[ ! -d $work_folder ]]; then mkdir -pv "$work_folder" || $ret - $SUDO chown "$(id -un)":"$(id -gn)" "$work_folder" || $ret - chmod ug+rwx,o= "$work_folder" || $ret fi +# Ensure the work folder has the correct owner (the user running the script, not root) +$SUDO chown "$(id -un)":"$(id -gn)" "$work_folder" || $ret +$SUDO chmod ug+rwx,o= "$work_folder" || $ret if [[ -d $du_test_data_dir_path ]]; then $SUDO rm -r $du_test_data_dir_path diff --git a/scripts/patches/azure-storage-sdk-base64-cstdint.patch b/scripts/patches/azure-storage-sdk-base64-cstdint.patch new file mode 100644 index 000000000..b5ecd4b3e --- /dev/null +++ b/scripts/patches/azure-storage-sdk-base64-cstdint.patch @@ -0,0 +1,30 @@ +--- a/sdk/core/azure-core/inc/azure/core/base64.hpp ++++ b/sdk/core/azure-core/inc/azure/core/base64.hpp +@@ -10,6 +10,7 @@ + + #pragma once + ++#include + #include + #include + #include +--- a/sdk/core/azure-core/src/base64.cpp ++++ b/sdk/core/azure-core/src/base64.cpp +@@ -3,6 +3,7 @@ + + #include "azure/core/base64.hpp" + #include "azure/core/platform.hpp" ++#include + + #include + #include +--- a/sdk/core/azure-core/inc/azure/core/uuid.hpp ++++ b/sdk/core/azure-core/inc/azure/core/uuid.hpp +@@ -10,6 +10,7 @@ + + #include "azure/core/platform.hpp" + ++#include + #include + #include + diff --git a/src/extensions/content_downloaders/CMakeLists.txt b/src/extensions/content_downloaders/CMakeLists.txt index d9978f00d..fdbb226d4 100644 --- a/src/extensions/content_downloaders/CMakeLists.txt +++ b/src/extensions/content_downloaders/CMakeLists.txt @@ -1,4 +1,6 @@ cmake_minimum_required (VERSION 3.5) add_subdirectory (curl_downloader) -add_subdirectory (deliveryoptimization_downloader) +if (ADUC_BUILD_WITH_DELIVERY_OPTIMIZATION) + add_subdirectory (deliveryoptimization_downloader) +endif ()